diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b9ce73086..9af43d4c1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,81 +2,141 @@ name: packages on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+' + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+" # Dry-run only workflow_dispatch: schedule: - - cron: '0 13 * * SUN' + - cron: "0 13 * * SUN" + +defaults: + run: + shell: bash -e {0} + +env: + PYTHON_VERSION: "3.11" + PACKAGE: "param" jobs: + waiting_room: + name: Waiting Room + runs-on: ubuntu-latest + needs: [conda_build, pip_install] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + environment: + name: publish + steps: + - run: echo "All builds have finished, have been approved, and ready to publish" + + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 + conda_build: - name: Build Conda Packages - runs-on: 'ubuntu-latest' + name: Build Conda + needs: [pixi_lock] + runs-on: "ubuntu-latest" + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 + with: + environments: "build" + download-data: false + install: false + - name: conda build + run: pixi run -e build build-conda + - uses: actions/upload-artifact@v4 + if: always() + with: + name: conda + path: dist/*.tar.bz2 + if-no-files-found: error + + conda_publish: + name: Publish Conda + runs-on: ubuntu-latest + needs: [conda_build, waiting_room] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') defaults: run: - shell: bash -l {0} - env: - CHANS_DEV: "-c pyviz/label/dev" - PKG_TEST_PYTHON: "--test-python=py37" - PYTHON_VERSION: "3.9" - CHANS: "-c pyviz" - CONDA_UPLOAD_TOKEN: ${{ secrets.CONDA_UPLOAD_TOKEN }} + shell: bash -el {0} steps: - - uses: actions/checkout@v4 - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - uses: actions/setup-python@v5 + - uses: actions/download-artifact@v4 with: - python-version: "3.9" + name: conda + path: dist/ + - name: Set environment variables + run: | + echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "CONDA_FILE=$(ls dist/*.conda)" >> $GITHUB_ENV - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" - - name: Set output - id: vars - run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT + channel: "conda-forge" - name: conda setup run: | - conda update --name base conda - conda install anaconda-client conda-build - pip install hatch - - name: conda build - run: | - VERSION=`hatch version` conda build conda.recipe/ + conda install -y anaconda-client - name: conda dev upload - if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) + if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') run: | - anaconda --token $CONDA_UPLOAD_TOKEN upload --user pyviz --label=dev $(VERSION=`hatch version` conda build --output conda.recipe) + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE - name: conda main upload - if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) + if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) run: | - anaconda --token $CONDA_UPLOAD_TOKEN upload --user pyviz --label=dev --label=main $(VERSION=`hatch version` conda build --output conda.recipe) + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + pip_build: - name: Build PyPI Packages - runs-on: 'ubuntu-latest' - defaults: - run: - shell: bash -l {0} + name: Build PyPI + needs: [pixi_lock] + runs-on: "ubuntu-latest" + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 + with: + environments: "build" + download-data: false + install: false + - name: Build package + run: pixi run -e build build-pip + - uses: actions/upload-artifact@v4 + if: always() + with: + name: pip + path: dist/ + if-no-files-found: error + + pip_install: + name: Install PyPI + runs-on: "ubuntu-latest" + needs: [pip_build] steps: - - uses: actions/checkout@v4 - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - uses: actions/setup-python@v5 with: - python-version: "3.9" - - name: env setup - run: | - python -m pip install --upgrade pip - python -m pip install build - - name: pip build - run: | - python -m build - - name: Publish package to PyPI - if: github.event_name == 'push' + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + - name: Install package + run: python -m pip install dist/*.whl + - name: Import package + run: python -c "import $PACKAGE; print($PACKAGE.__version__)" + + pip_publish: + name: Publish PyPI + runs-on: ubuntu-latest + needs: [pip_build, waiting_room] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: user: ${{ secrets.PPU }} password: ${{ secrets.PPP }} - packages_dir: dist/ + repository-url: "https://upload.pypi.org/legacy/" diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index d2073f6eb..418f8f0e0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -2,50 +2,71 @@ name: docs on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+' + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+" workflow_dispatch: inputs: target: - description: 'Site to build and deploy' + description: "Site to build and deploy" type: choice options: - - dev - - main - - dryrun + - dev + - main + - dryrun required: true default: dryrun schedule: - - cron: '0 13 * * SUN' + - cron: "0 13 * * SUN" + +defaults: + run: + shell: bash -e {0} + +env: + DISPLAY: ":99.0" jobs: - build_docs: - name: Documentation - runs-on: 'ubuntu-latest' + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 + + docs_build: + name: Build Documentation + needs: [pixi_lock] + runs-on: "ubuntu-latest" timeout-minutes: 120 - defaults: - run: - shell: bash -l {0} steps: - - uses: actions/checkout@v4 - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - uses: actions/setup-python@v5 + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 + with: + environments: docs + - name: Build documentation + run: pixi run -e docs docs-build + - uses: actions/upload-artifact@v4 + if: always() with: - python-version: '3.9' + name: docs + if-no-files-found: error + path: builtdocs - name: Set output id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - name: graphviz - run: sudo apt install graphviz graphviz-dev - - name: env setup - run: | - python -m pip install --upgrade pip - python -m pip install hatch - - name: build docs - run: hatch -v run docs:build + + docs_publish: + name: Publish Documentation + runs-on: "ubuntu-latest" + needs: [docs_build] + steps: + - uses: actions/download-artifact@v4 + with: + name: docs + path: builtdocs/ + - name: Set output + id: vars + run: echo "tag=${{ needs.docs_build.outputs.tag }}" >> $GITHUB_OUTPUT - name: Deploy dev uses: peaceiris/actions-gh-pages@v4 if: | diff --git a/.github/workflows/nightly_lock.yaml b/.github/workflows/nightly_lock.yaml new file mode 100644 index 000000000..695590b58 --- /dev/null +++ b/.github/workflows/nightly_lock.yaml @@ -0,0 +1,25 @@ +name: nightly_lock +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +env: + PACKAGE: "param" + +jobs: + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 + - name: Upload lock-file to S3 + if: '!github.event.pull_request.head.repo.fork' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" + run: | + zip $(date +%Y-%m-%d).zip pixi.lock pixi.toml + aws s3 cp ./$(date +%Y-%m-%d).zip s3://assets.holoviz.org/lock/$PACKAGE/ diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0423bb56e..3bc4cc8ec 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,71 +5,128 @@ on: - main pull_request: branches: - - '*' + - "*" workflow_dispatch: schedule: - - cron: '0 13 * * SUN' + - cron: "0 13 * * SUN" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash -e {0} + +env: + DISPLAY: ":99.0" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COV: "--cov=./param --cov-report=xml" + jobs: pre_commit: - name: Run pre-commit hooks - runs-on: 'ubuntu-latest' + name: Run pre-commit + runs-on: "ubuntu-latest" + steps: + - uses: holoviz-dev/holoviz_tasks/pre-commit@v0 + + setup: + name: Setup workflow + runs-on: ubuntu-latest + outputs: + matrix: ${{ env.MATRIX }} + steps: + - name: Set matrix option + run: | + if [[ '${{ github.event_name }}' == 'workflow_dispatch' ]]; then + OPTION=${{ github.event.inputs.target }} + elif [[ '${{ github.event_name }}' == 'schedule' ]]; then + OPTION="full" + elif [[ '${{ github.event_name }}' == 'push' && '${{ github.ref_type }}' == 'tag' ]]; then + OPTION="full" + else + OPTION="default" + fi + echo "MATRIX_OPTION=$OPTION" >> $GITHUB_ENV + - name: Set test matrix with 'default' option + if: env.MATRIX_OPTION == 'default' + run: | + MATRIX=$(jq -nsc '{ + "os": ["ubuntu-latest", "macos-latest", "windows-latest"], + "environment": ["test-38", "test-312", "test-313"] + }') + echo "MATRIX=$MATRIX" >> $GITHUB_ENV + - name: Set test matrix with 'full' option + if: env.MATRIX_OPTION == 'full' + run: | + MATRIX=$(jq -nsc '{ + "os": ["ubuntu-latest", "macos-latest", "windows-latest"], + "environment": ["test-38", "test-39", "test-310", "test-311", "test-312", "test-313"] + }') + echo "MATRIX=$MATRIX" >> $GITHUB_ENV + - name: Set test matrix with 'downstream' option + if: env.MATRIX_OPTION == 'downstream' + run: | + MATRIX=$(jq -nsc '{ + "os": ["ubuntu-latest"], + "environment": ["test-312"] + }') + echo "MATRIX=$MATRIX" >> $GITHUB_ENV + + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 with: - fetch-depth: "1" - - name: set PY - run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - uses: actions/cache@v4 + cache: ${{ github.event.inputs.cache == 'true' || github.event.inputs.cache == '' }} + + unit_test_suite: + name: unit:${{ matrix.environment }}:${{ matrix.os }} + needs: [pre_commit, setup, pixi_lock] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + timeout-minutes: 30 + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 + with: + environments: ${{ matrix.environment }} + - name: Test Unit + run: | + pixi run -e ${{ matrix.environment }} test-unit $COV + - name: Test Examples + run: | + pixi run -e ${{ matrix.environment }} test-example + - uses: codecov/codecov-action@v4 with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - name: pre-commit - uses: pre-commit/action@v3.0.0 - test_suite: - name: Test ${{ matrix.python-version }}, ${{ matrix.platform }} - needs: [pre_commit] - runs-on: ${{ matrix.platform }} + token: ${{ secrets.CODECOV_TOKEN }} + + core_test_suite: + name: core:${{ matrix.environment }}:${{ matrix.os }} + needs: [pre_commit, pixi_lock] + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - platform: ['ubuntu-latest', 'windows-latest', 'macos-latest'] - python-version: ${{ ( github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || ( github.event_name == 'push' && github.ref_type == 'tag' ) ) && fromJSON('["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.9"]') || fromJSON('["3.8", "3.10", "3.12"]') }} + os: ["ubuntu-latest"] + environment: ["test-core", "test-pypy"] timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: "100" - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: env setup + environments: ${{ matrix.environment }} + - name: Test Unit run: | - python -m pip install --upgrade pip - python -m pip install hatch - - name: temp patch pip for blosc2 - if: contains(matrix.platform, 'ubuntu') && matrix.python-version == '3.8' - run: echo "PIP_ONLY_BINARY=blosc2" >> $GITHUB_ENV - - name: run unit tests - run: hatch -v run +py=${{ matrix.python-version }} tests:with_coverage - - name: run examples tests - # A notebook fails on Windows (UNIX path used in an example) - # No need to run these tests for PyPy really - if: contains(matrix.platform, 'ubuntu') && !startsWith(matrix.python-version, 'py') - run: hatch -v run +py=${{ matrix.python-version }} tests_examples:examples - - name: Upload coverage reports to Codecov - if: github.event_name == 'push' || github.event_name == 'pull_request' - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: false - verbose: false - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + pixi run -e ${{ matrix.environment }} test-unit + + result_test_suite: + name: result:test + needs: [unit_test_suite, core_test_suite] + if: always() + runs-on: ubuntu-latest + steps: + - name: check for failures + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: echo job failed && exit 1 diff --git a/.gitignore b/.gitignore index ce9852243..fc5adc442 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ builtdocs/ jupyter_execute/ doc/Reference_Manual/ doc/user_guide/data.pickle -doc/user_guide/output.csv +doc/user_guide/output.* doc/reference/generated # Unit test / Coverage report @@ -32,3 +32,7 @@ param/_version.py # asv benchmark benchmarks/.asv benchmarks/param + +# pixi +.pixi +pixi.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d10dd1dd5..147e37873 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,4 @@ -# This is the configuration for pre-commit, a local framework for managing pre-commit hooks -# Check out the docs at: https://pre-commit.com/ - -default_stages: [commit] +default_stages: [pre-commit] repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 diff --git a/doc/developer_guide.md b/doc/developer_guide.md index 7dead2648..700cb50d0 100644 --- a/doc/developer_guide.md +++ b/doc/developer_guide.md @@ -1,41 +1,189 @@ -# Developer guide +# Developer Guide -## Setup +Welcome. We are so happy you've decided to contribute. -The source code for `Param` is hosted on GitHub. To clone the source repository, issue the following command: +## Setting up a development environment + +This guide describes how to install and configure development environments. + +If you have any problems with the steps here, please reach out in the `dev` channel on [Discord](https://discord.gg/rb6gPXbdAr) or on [Discourse](https://discourse.holoviz.org/). + +## Preliminaries + +### Basic understanding of how to contribute to Open Source + +If this is your first open-source contribution, please study one or more of the below resources. + +- [How to Get Started with Contributing to Open Source | Video](https://youtu.be/RGd5cOXpCQw) +- [Contributing to Open-Source Projects as a New Python Developer | Video](https://youtu.be/jTTf4oLkvaM) +- [How to Contribute to an Open Source Python Project | Blog post](https://www.educative.io/blog/contribue-open-source-python-project) + +### Git + +The Param source code is stored in a [Git](https://git-scm.com) source control repository. The first step to working on Param is to install Git onto your system. There are different ways to do this, depending on whether you use Windows, Mac, or Linux. + +To install Git on any platform, refer to the [Installing Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) section of the [Pro Git Book](https://git-scm.com/book/en/v2). + +To contribute to Param, you will also need [Github account](https://github.com/join) and knowledge of the [_fork and pull request workflow_](https://docs.github.com/en/get-started/quickstart/contributing-to-projects). + +### Pixi + +Developing all aspects of Param requires a wide range of packages in different environments. To make this more manageable, Pixi manages the developer experience. To install Pixi, follow [this guide](https://pixi.sh/latest/#installation). + +#### Glossary + +- Tasks: A task is what can be run with `pixi run `. Tasks can be anything from installing packages to running tests. +- Environments: An environment is a set of packages installed in a virtual environment. Each environment has a name; you can run tasks in a specific environment with the `-e` flag. + For example, `pixi run -e test-core test-unit` will run the `test-unit` task in the `test-core` environment. +- Lock-file: A lock-file is a file that contains all the information about the environments. + +For more information, see the [Pixi documentation](https://pixi.sh/latest/). + +:::{admonition} Note +:class: info + +The first time you run `pixi`, it will create a `.pixi` directory in the source directory. +This directory will contain all the files needed for the virtual environments. +The `.pixi` directory can be large, so it is advised not to put the source directory into a cloud-synced directory. +::: + +## Installing the Project + +### Cloning the Project + +The source code for the Param project is hosted on [GitHub](https://github.com/holoviz/param). The first thing you need to do is clone the repository. + +1. Go to [github.com/holoviz/param](https://github.com/holoviz/param) +2. [Fork the repository](https://docs.github.com/en/get-started/quickstart/contributing-to-projects#forking-a-repository) +3. Run in your terminal: `git clone https://github.com//param` + +The instructions for cloning above created a `param` directory at your file system location. +This `param` directory is the _source checkout_ for the remainder of this document, and your current working directory is this directory. + +### Fetch tags from upstream + +The version number of the package depends on [`git tags`](https://git-scm.com/book/en/v2/Git-Basics-Tagging), so you need to fetch the tags from the upstream repository: ```bash -git clone https://github.com/holoviz/param.git +git remote add upstream https://github.com/holoviz/param.git +git fetch --tags upstream +git push --tags ``` -This will create a `param` directory at your file system location. +## Start developing -`Param` relies on `hatch` to manage the project. Follow the [instructions](https://hatch.pypa.io/latest/install/) to install it. Once installed, run the following command to create the *default* environment and activate it, it contains the dependencies required to develop `Param`: +To start developing, run the following command ```bash -hatch shell +pixi install +``` + +The first time you run it, it will create a `pixi.lock` file with information for all available environments. This command will take a minute or so to run. + +All available tasks can be found by running `pixi task list`, the following sections will give a brief introduction to the most common tasks. + +### Editable install + +It can be advantageous to install Param in [editable mode](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs): + +```bash +pixi run install +``` + +:::{admonition} Note +:class: info + +Currently, this needs to be run for each environment. So, if you want to install in the `test-312` environment, you can add `--environment` / `-e` to the command: + +```bash +pixi run -e test-313 install +``` + +You can find the list of environments in the **pixi.toml** file or via the command `pixi info`. + +::: + +## Linting + +Param uses [pre-commit](https://pre-commit.com/) to apply linting to Param code. Linting can be run for all the files with: + +```bash +pixi run lint +``` + +Linting can also be set up to run automatically with each commit; this is the recommended way because if linting is not passing, the [Continuous Integration](https://en.wikipedia.org/wiki/Continuous_integration) (CI) will also fail. + +```bash +pixi run lint-install ``` ## Testing -The simplest way to run the unit tests is to run the following command: +To help keep Param maintainable, all Pull Requests (PR) with code changes should typically be accompanied by relevant tests. While exceptions may be made for specific circumstances, the default assumption should be that a Pull Request without tests will not be merged. + +There are three types of tasks and five environments related to tests. + +### Unit tests + +Unit tests are usually small tests executed with [pytest](https://docs.pytest.org). They can be found in `param/tests/`. +Unit tests can be run with the `test-unit` task: + +```bash +pixi run test-unit +``` + +The task is available in the following environments: + +1. `test-38`, `test-39`, `test-310`, `test-311`, `test-312`, and `test-313`. +1. `test-core` and `test-pypy` + +Where the first ones have the same environments except for different Python versions. `test-core` only has a core set of dependencies, and `test-pypy` is for testing on PyPy. + +If you haven't set the environment flag in the command, a menu will help you select which one of the environments to use. + +### Example tests + +Param's documentation consists mainly of Jupyter Notebooks. The example tests execute all the notebooks and fail if an error is raised. Example tests are possible thanks to [nbval](https://nbval.readthedocs.io/) and can be found in the `doc/` folder. +Example tests can be run with the following command: ```bash -hatch run tests +pixi run test-example ``` -You can also run the examples tests, i.e. check that the notebooks run without any error, with: +This task has the same environments as the unit tests except for `test-core` and `test-pypy`. + +## Documentation + +The documentation can be built with the command: ```bash -hatch run examples +pixi run docs-build ``` -## Documentation building +## Build -Run the following command to build the documentation: +Param has two build tasks, for building packages for Pip and Conda. ```bash -hatch run docs:build +pixi run build-pip +pixi run build-conda ``` -Once completed, the built site can be found in the `builtdocs` folder. +## Continuous Integration + +Every push to the `main` branch or any PR branch on GitHub automatically triggers a test build with [GitHub Actions](https://github.com/features/actions). + +You can see the list of all current and previous builds at [this URL](https://github.com/holoviz/param/actions) + +### Etiquette + +GitHub Actions provides free build workers for open-source projects. A few considerations will help you be considerate of others needing these limited resources: + +- Run the tests locally before opening or pushing to an opened PR. + +- Group commits to meaningful chunks of work before pushing to GitHub (i.e., don't push on every commit). + +## Useful Links + +- [Dev version of Param Site](https://holoviz-dev.github.io/param) + - Use this to explore new, not yet released features and docs diff --git a/doc/user_guide/Parameter_Types.ipynb b/doc/user_guide/Parameter_Types.ipynb index f0c0d23af..8182e2768 100644 --- a/doc/user_guide/Parameter_Types.ipynb +++ b/doc/user_guide/Parameter_Types.ipynb @@ -717,8 +717,8 @@ "class P(param.Parameterized):\n", " p = param.Path('Parameter_Types.ipynb')\n", " f = param.Filename('Parameter_Types.ipynb')\n", - " d = param.Foldername('lib', search_paths=['/','/usr','/share'])\n", - " o = param.Filename('output.csv', check_exists=False)\n", + " d = param.Foldername('lib', search_paths=['/','/usr','/share'], check_exists=False)\n", + " o = param.Filename('output.txt', check_exists=False)\n", " \n", "p = P()\n", "p.p" @@ -731,7 +731,7 @@ "metadata": {}, "outputs": [], "source": [ - "p.p = '/usr/lib'\n", + "p.p = 'Outputs.ipynb'\n", "p.p" ] }, diff --git a/param/_utils.py b/param/_utils.py index 70b20b140..650632eef 100644 --- a/param/_utils.py +++ b/param/_utils.py @@ -612,3 +612,27 @@ def async_executor(func): task.add_done_callback(_running_tasks.discard) else: event_loop.run_until_complete(func()) + +class _GeneratorIsMeta(type): + def __instancecheck__(cls, inst): + return isinstance(inst, tuple(cls.types())) + + def __subclasscheck__(cls, sub): + return issubclass(sub, tuple(cls.types())) + + def __iter__(cls): + yield from cls.types() + +class _GeneratorIs(metaclass=_GeneratorIsMeta): + @classmethod + def __iter__(cls): + yield from cls.types() + +def gen_types(gen_func): + """ + Decorator which takes a generator function which yields difference types + make it so it can be called with isinstance and issubclass.""" + if not inspect.isgeneratorfunction(gen_func): + msg = "gen_types decorator can only be applied to generator" + raise TypeError(msg) + return type(gen_func.__name__, (_GeneratorIs,), {"types": staticmethod(gen_func)}) diff --git a/param/parameterized.py b/param/parameterized.py index f303a09dd..3e8074b95 100644 --- a/param/parameterized.py +++ b/param/parameterized.py @@ -16,11 +16,12 @@ import logging import numbers import operator -import random import re +import sys import types import typing import warnings +from inspect import getfullargspec # Allow this file to be used standalone if desired, albeit without JSON serialization try: @@ -55,6 +56,7 @@ accept_arguments, iscoroutinefunction, descendents, + gen_types, ) # Ideally setting param_pager would be in __init__.py but param_pager is @@ -72,17 +74,18 @@ param_pager = None -from inspect import getfullargspec - -dt_types = (dt.datetime, dt.date) -_int_types = (int,) +@gen_types +def _dt_types(): + yield dt.datetime + yield dt.date + if np := sys.modules.get("numpy"): + yield np.datetime64 -try: - import numpy as np - dt_types = dt_types + (np.datetime64,) - _int_types = _int_types + (np.integer,) -except: - pass +@gen_types +def _int_types(): + yield int + if np := sys.modules.get("numpy"): + yield np.integer VERBOSE = INFO - 1 logging.addLevelName(VERBOSE, "VERBOSE") @@ -1715,11 +1718,18 @@ class Comparator: type(None): operator.eq, lambda o: hasattr(o, '_infinitely_iterable'): operator.eq, # Time } - equalities.update({dtt: operator.eq for dtt in dt_types}) + gen_equalities = { + _dt_types: operator.eq + } @classmethod def is_equal(cls, obj1, obj2): - for eq_type, eq in cls.equalities.items(): + equals = cls.equalities.copy() + for gen, op in cls.gen_equalities.items(): + for t in gen(): + equals[t] = op + + for eq_type, eq in equals.items(): try: are_instances = isinstance(obj1, eq_type) and isinstance(obj2, eq_type) except TypeError: @@ -3790,6 +3800,9 @@ def pprint(val,imports=None, prefix="\n ", settings=[], elif type(val) in script_repr_reg: rep = script_repr_reg[type(val)](val,imports,prefix,settings) + elif isinstance(val, _no_script_repr): + rep = None + elif isinstance(val, Parameterized) or (type(val) is type and issubclass(val, Parameterized)): rep=val.param.pprint(imports=imports, prefix=prefix+" ", qualify=qualify, unknown_value=unknown_value, @@ -3824,17 +3837,13 @@ def container_script_repr(container,imports,prefix,settings): return rep -def empty_script_repr(*args): # pyflakes:ignore (unused arguments): - return None - -try: +@gen_types +def _no_script_repr(): # Suppress scriptrepr for objects not yet having a useful string representation - import numpy - script_repr_reg[random.Random] = empty_script_repr - script_repr_reg[numpy.random.RandomState] = empty_script_repr - -except ImportError: - pass # Support added only if those libraries are available + if random := sys.modules.get("random"): + yield random.Random + if npr := sys.modules.get("numpy.random"): + yield npr.RandomState def function_script_repr(fn,imports,prefix,settings): diff --git a/param/parameters.py b/param/parameters.py index e65510e43..61f82d8a0 100644 --- a/param/parameters.py +++ b/param/parameters.py @@ -30,11 +30,12 @@ import warnings from collections import OrderedDict +from collections.abc import Iterable from contextlib import contextmanager from .parameterized import ( Parameterized, Parameter, ParameterizedFunction, ParamOverrides, String, - Undefined, get_logger, instance_descriptor, dt_types, + Undefined, get_logger, instance_descriptor, _dt_types, _int_types, _identity_hook ) from ._utils import ( @@ -94,7 +95,7 @@ def guess_param_types(**kwargs): kws = dict(default=v, constant=True) if isinstance(v, Parameter): params[k] = v - elif isinstance(v, dt_types): + elif isinstance(v, _dt_types): params[k] = Date(**kws) elif isinstance(v, bool): params[k] = Boolean(**kws) @@ -109,7 +110,7 @@ def guess_param_types(**kwargs): elif isinstance(v, tuple): if all(_is_number(el) for el in v): params[k] = NumericTuple(**kws) - elif all(isinstance(el, dt_types) for el in v) and len(v)==2: + elif len(v) == 2 and all(isinstance(el, _dt_types) for el in v): params[k] = DateRange(**kws) else: params[k] = Tuple(**kws) @@ -141,7 +142,7 @@ def parameterized_class(name, params, bases=Parameterized): Dynamically create a parameterized class with the given name and the supplied parameters, inheriting from the specified base(s). """ - if not (isinstance(bases, list) or isinstance(bases, tuple)): + if not isinstance(bases, (list, tuple)): bases=[bases] return type(name, tuple(bases), params) @@ -917,14 +918,14 @@ def _validate_value(self, val, allow_None): if self.allow_None and val is None: return - if not isinstance(val, dt_types) and not (allow_None and val is None): + if not isinstance(val, _dt_types) and not (allow_None and val is None): raise ValueError( f"{_validate_error_prefix(self)} only takes datetime and " f"date types, not {type(val)}." ) def _validate_step(self, val, step): - if step is not None and not isinstance(step, dt_types): + if step is not None and not isinstance(step, _dt_types): raise ValueError( f"{_validate_error_prefix(self, 'step')} can only be None, " f"a datetime or date type, not {type(step)}." @@ -1355,7 +1356,7 @@ class DateRange(Range): """ def _validate_bound_type(self, value, position, kind): - if not isinstance(value, dt_types): + if not isinstance(value, _dt_types): raise ValueError( f"{_validate_error_prefix(self)} {position} {kind} can only be " f"None or a date/datetime value, not {type(value)}." @@ -1379,7 +1380,7 @@ def _validate_value(self, val, allow_None): f"not {type(val)}." ) for n in val: - if isinstance(n, dt_types): + if isinstance(n, _dt_types): continue raise ValueError( f"{_validate_error_prefix(self)} only takes date/datetime " @@ -2184,18 +2185,18 @@ def _validate(self, val): def _validate_class_(self, val, class_, is_instance): if (val is None and self.allow_None): return - if isinstance(class_, tuple): - class_name = ('(%s)' % ', '.join(cl.__name__ for cl in class_)) + if (is_instance and isinstance(val, class_)) or (not is_instance and issubclass(val, class_)): + return + + if isinstance(class_, Iterable): + class_name = ('({})'.format(', '.join(cl.__name__ for cl in class_))) else: class_name = class_.__name__ - if is_instance: - if not (isinstance(val, class_)): - raise ValueError( - f"{_validate_error_prefix(self)} value must be an instance of {class_name}, not {val!r}.") - else: - if not (issubclass(val, class_)): - raise ValueError( - f"{_validate_error_prefix(self)} value must be a subclass of {class_name}, not {val}.") + + raise ValueError( + f"{_validate_error_prefix(self)} value must be " + f"{'an instance' if is_instance else 'a subclass'} of {class_name}, not {val!r}." + ) def get_range(self): """ diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 000000000..f693d250b --- /dev/null +++ b/pixi.toml @@ -0,0 +1,138 @@ +[project] +name = "param" +channels = ["conda-forge"] +platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] + +[tasks] +install = 'python -m pip install --no-deps --disable-pip-version-check -e .' + +[activation.env] +PYTHONIOENCODING = "utf-8" + +[environments] +test-38 = ["py38", "test-core", "test", "example", "test-example"] +test-39 = ["py39", "test-core", "test", "example", "test-example"] +test-310 = ["py310", "test-core", "test", "example", "test-example"] +test-311 = ["py311", "test-core", "test", "example", "test-example"] +test-312 = ["py312", "test-core", "test", "example", "test-example"] +test-313 = ["py313", "test-core", "test", "example", "test-example"] +test-core = ["py313", "test-core"] +test-pypy = ["pypy", "test-core", "test-pypy"] +docs = ["py311", "example", "doc"] +build = ["py311", "build"] +lint = ["py311", "lint"] + +[dependencies] +nomkl = "*" +pip = "*" + +[feature.py38.dependencies] +python = "3.8.*" + +[feature.py39.dependencies] +python = "3.9.*" + +[feature.py310.dependencies] +python = "3.10.*" + +[feature.py311.dependencies] +python = "3.11.*" + +[feature.py312.dependencies] +python = "3.12.*" + +[feature.py313.dependencies] +python = "3.13.*" + +[feature.pypy] +platforms = ["linux-64", "osx-64", "win-64"] # not supported for osx-arm64 + +[feature.pypy.dependencies] +pypy = "7.3.*" + +[feature.example.dependencies] +aiohttp = "*" +pandas = "*" +panel = "*" + +# ============================================= +# =================== TESTS =================== +# ============================================= +[feature.test-core.dependencies] +pytest = "*" +pytest-asyncio = "*" +pytest-cov = "*" +pytest-github-actions-annotate-failures = "*" + +[feature.test-core.tasks] +test-unit = 'pytest tests' + +[feature.test.dependencies] +cloudpickle = "*" +ipython = "*" +jsonschema = "*" +nest-asyncio = "*" +numpy = "*" +odfpy = "*" +openpyxl = "*" +pandas = "*" +pyarrow = "*" +pytables = "*" +xlrd = "*" + +[feature.test.target.linux.dependencies] +gmpy2 = "*" + +[feature.test-pypy.dependencies] +cloudpickle = "*" +ipython = "*" +jsonschema = "*" +nest-asyncio = "*" +numpy = "*" +odfpy = "*" +openpyxl = "*" +pandas = "*" +xlrd = "*" + +[feature.test-example.tasks] +test-example = 'pytest -n logical --dist loadscope --nbval-lax doc' + +[feature.test-example.dependencies] +psutil = "*" +pytest-xdist = "*" +nbval = "*" + +# ============================================= +# =================== DOCS ==================== +# ============================================= +[feature.doc] +channels = ["pyviz"] + +[feature.doc.dependencies] +graphviz = "*" +nbsite = ">=0.8.4,<0.9.0" +sphinx-remove-toctrees = "*" + +[feature.doc.tasks.docs-build] +cmd = "sphinx-build -b html doc builtdocs" + +# ============================================= +# ================== BUILD ==================== +# ============================================= +[feature.build.dependencies] +python-build = "*" +conda-build = "*" + +[feature.build.tasks] +build-conda = 'bash scripts/conda/build.sh' +build-pip = 'python -m build .' + +# ============================================= +# =================== LINT ==================== +# ============================================= +[feature.lint.dependencies] +pre-commit = "*" + +[feature.lint.tasks] +lint = 'pre-commit run --all-files' +lint-install = 'pre-commit install' diff --git a/pyproject.toml b/pyproject.toml index fe4a0ebcf..cc4dababe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries", ] @@ -39,13 +40,7 @@ examples = [ "pandas", "panel", ] -doc = [ - "param[examples]", - "nbsite ==0.8.4", - "sphinx-remove-toctrees", -] tests = [ - "coverage[toml]", "pytest", "pytest-asyncio", ] @@ -75,14 +70,8 @@ tests-full = [ "cloudpickle", "nest_asyncio", ] -lint = [ - "flake8", - "pre-commit", -] all = [ "param[tests-full]", - "param[doc]", - "param[lint]", ] [project.urls] @@ -94,6 +83,7 @@ HoloViz = "https://holoviz.org/" [tool.hatch.version] source = "vcs" +raw-options = { version_scheme = "no-guess-dev" } [tool.hatch.build.targets.wheel] include = [ @@ -111,118 +101,15 @@ include = [ [tool.hatch.build.hooks.vcs] version-file = "param/_version.py" -[tool.hatch.envs.default] -dependencies = [ - # Linters - "param[lint]", - # Base tests dependencies - "param[tests]", - # Examples tests dependencies - "param[tests-examples]", - # Deserializatoin dependencies - "param[tests-deser]", - # Additional tests dependencies, not including gmpy as - # it's tricky to install. - "ipython", - "jsonschema", - "numpy", - "pandas", - "cloudpickle", - "nest_asyncio", - # To keep __version__ up-to-date in editable installs - "setuptools_scm", -] -post-install-commands = [ - "python -m pip install pre-commit", - "pre-commit install", -] - -[tool.hatch.envs.default.scripts] -tests = "pytest {args:tests}" -examples = "pytest -n logical --dist loadscope --nbval-lax {args:doc}" - -[tool.hatch.envs.docs] -template = "docs" -features = ["doc"] -python = "3.9" - -[tool.hatch.envs.docs.scripts] -build = [ - "sphinx-build -b html doc builtdocs", -] - -[tool.hatch.envs.tests] -template = "tests" -dependencies = [ - "param[tests]", - "ipython", - "jsonschema", -] - -[[tool.hatch.envs.tests.matrix]] -python = [ - "3.8", - "3.9", - "3.10", - "3.11", - "3.12", - "pypy3.9", -] - -[tool.hatch.envs.tests.scripts] -tests = "pytest {args:tests}" -with_coverage = [ - "coverage run --source=numbergen,param -m pytest -v {args:tests}", - "coverage report", - "coverage xml", -] - -[tool.hatch.envs.tests.overrides] -# Only install these on non PyPy environments -name."^(?!pypy).*".dependencies = [ - "numpy", - "pandas", - "xlrd", - "openpyxl", - "odfpy", - "pyarrow", - "cloudpickle", - "nest_asyncio", -] -# Only install gmpy on Linux on these version -# Only install tables (deser HDF5) on Linux on these version -matrix.python.dependencies = [ - { value = "gmpy", if = ["3.8", "3.9", "3.10"], platform = ["linux"] }, - { value = "tables", if = ["3.8", "3.9", "3.10", "3.11", "3.12"], platform = ["linux"] }, -] - -[tool.hatch.envs.tests_examples] -template = "tests_examples" -dependencies = [ - "param[tests-examples]", - "ipython", - "jsonschema", -] - -[[tool.hatch.envs.tests_examples.matrix]] -python = [ - "3.8", - "3.9", - "3.10", - "3.11", - "3.12", -] - -[tool.hatch.envs.tests_examples.scripts] -examples = "pytest -v -n logical --dist loadscope --nbval-lax {args:doc}" - [tool.pytest.ini_options] +addopts = "--color=yes" python_files = "test*.py" filterwarnings = [ "error", ] xfail_strict = "true" asyncio_mode = "auto" +asyncio_default_fixture_loop_scope="function" [tool.coverage.report] omit = ["param/version.py"] diff --git a/scripts/conda/build.sh b/scripts/conda/build.sh new file mode 100755 index 000000000..566c87a31 --- /dev/null +++ b/scripts/conda/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +PACKAGE="param" + +python -m build -s . + +VERSION=$(python -c "import $PACKAGE; print($PACKAGE._version.__version__)") +export VERSION + +# conda config --env --set conda_build.pkg_format 2 +conda build scripts/conda/recipe --no-anaconda-upload --no-verify -c conda-forge + +mv "$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2" dist diff --git a/conda.recipe/meta.yaml b/scripts/conda/recipe/meta.yaml similarity index 83% rename from conda.recipe/meta.yaml rename to scripts/conda/recipe/meta.yaml index 78a734b94..68450f252 100644 --- a/conda.recipe/meta.yaml +++ b/scripts/conda/recipe/meta.yaml @@ -1,4 +1,4 @@ -{% set pyproject = load_file_data('../pyproject.toml', from_recipe_dir=True) %} +{% set pyproject = load_file_data('../../../pyproject.toml', from_recipe_dir=True) %} {% set buildsystem = pyproject['build-system'] %} {% set project = pyproject['project'] %} @@ -10,11 +10,11 @@ package: version: {{ version }} source: - path: .. + url: ../../../dist/{{ name }}-{{ version }}.tar.gz build: noarch: python - script: {{ PYTHON }} -m pip install . -vv + script: {{ PYTHON }} -m pip install --no-deps -vv . requirements: build: diff --git a/tests/testimports.py b/tests/testimports.py new file mode 100644 index 000000000..f353e0470 --- /dev/null +++ b/tests/testimports.py @@ -0,0 +1,20 @@ +import sys +from subprocess import check_output +from textwrap import dedent + + +def test_no_blocklist_imports(): + check = """\ + import sys + import param + + blocklist = {"numpy", "IPython", "pandas"} + mods = blocklist & set(sys.modules) + + if mods: + print(", ".join(mods), end="") + """ + + output = check_output([sys.executable, '-c', dedent(check)]) + + assert output == b"" diff --git a/tests/testreactive.py b/tests/testreactive.py index e53e56ce9..68e6b9ee3 100644 --- a/tests/testreactive.py +++ b/tests/testreactive.py @@ -590,7 +590,7 @@ def gen(): async def test_reactive_gen_pipe(): def gen(val): yield val+1 - time.sleep(0.05) + time.sleep(0.1) yield val+2 rxv = rx(0) @@ -624,7 +624,7 @@ def gen(i): async def test_reactive_gen_pipe_with_dep(): def gen(value, i): yield value+i+1 - time.sleep(0.05) + time.sleep(0.1) yield value+i+2 irx = rx(0) diff --git a/tests/testutils.py b/tests/testutils.py index e5228d4b6..7fd54b99c 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -1,6 +1,7 @@ import datetime as dt import os +from collections.abc import Iterable from functools import partial import param @@ -8,7 +9,7 @@ from param import guess_param_types, resolve_path from param.parameterized import bothmethod -from param._utils import _is_mutable_container, iscoroutinefunction +from param._utils import _is_mutable_container, iscoroutinefunction, gen_types try: @@ -421,3 +422,20 @@ def test_iscoroutinefunction_asyncgen(): def test_iscoroutinefunction_partial_asyncgen(): pagen = partial(partial(agen)) assert iscoroutinefunction(pagen) + +def test_gen_types(): + @gen_types + def _int_types(): + yield int + + assert isinstance(1, (str, _int_types)) + assert isinstance(5, _int_types) + assert isinstance(5.0, _int_types) is False + + assert issubclass(int, (str, _int_types)) + assert issubclass(int, _int_types) + assert issubclass(float, _int_types) is False + + assert next(iter(_int_types())) is int + assert next(iter(_int_types)) is int + assert isinstance(_int_types, Iterable)