Add experimental code for data fragments. (#282) #1050
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Ubuntu CI | |
| on: | |
| push: | |
| pull_request: | |
| schedule: | |
| - cron: '30 2 * * 1' # Every Monday @ 2h30am UTC | |
| env: | |
| POETRY_VERSION: 1.2.1 | |
| MINIO_SERVER_DOWNLOAD_URL: https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20230907020502.0.0_amd64.deb | |
| MINIO_CLIENT_DOWNLOAD_URL: https://dl.min.io/client/mc/release/linux-amd64/archive/mcli_20230907224855.0.0_amd64.deb | |
| jobs: | |
| check_skip: | |
| runs-on: ubuntu-latest | |
| if: | | |
| !contains(format('{0} {1} {2}', github.event.head_commit.message, github.event.pull_request.title, github.event.pull_request.body), '[skip ci]') | |
| steps: | |
| - run: | | |
| cat <<'MESSAGE' | |
| github.event_name: ${{ toJson(github.event_name) }} | |
| github.event: | |
| ${{ toJson(github.event) }} | |
| MESSAGE | |
| test: | |
| needs: check_skip | |
| runs-on: ubuntu-latest | |
| if: "!contains(github.event.head_commit.message, '[skip ci]')" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10"] | |
| steps: | |
| - name: Create Cache Hash | |
| run: | | |
| export HASH=$(sha256sum <<EOT | cut -c -16 | |
| ${{ env.POETRY_VERSION }} | |
| ${{ env.MINIO_SERVER_DOWNLOAD_URL }} | |
| ${{ env.MINIO_CLIENT_DOWNLOAD_URL }} | |
| EOT | |
| ) | |
| echo "INSTALL_CACHE_HASH=$HASH" >> $GITHUB_ENV | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache Installations | |
| id: cache-installs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.local | |
| key: install-${{ env.INSTALL_CACHE_HASH }}-2 | |
| - name: Install Poetry | |
| if: steps.cache-installs.outputs.cache-hit != 'true' | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - --version ${{ env.POETRY_VERSION }} | |
| - name: Test poetry run | |
| run: poetry --version | |
| - name: Install Minio Server and Client | |
| if: steps.cache-installs.outputs.cache-hit != 'true' | |
| run: | | |
| cd /tmp | |
| wget -q -c ${{ env.MINIO_SERVER_DOWNLOAD_URL }} -O minio.deb | |
| wget -q -c ${{ env.MINIO_CLIENT_DOWNLOAD_URL }} -O mcli.deb | |
| dpkg-deb --extract minio.deb /tmp/minio | |
| dpkg-deb --extract mcli.deb /tmp/mcli | |
| mkdir -p "${HOME}/.local/bin" | |
| echo "${HOME}/.local/bin" >> $GITHUB_PATH | |
| install /tmp/minio/usr/local/bin/minio "${HOME}/.local/bin/minio" | |
| install /tmp/mcli/usr/local/bin/mcli "${HOME}/.local/bin/mc" | |
| - name: Test minio run | |
| run: | | |
| minio --help | |
| mc --help | |
| - name: Checkout source | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install dask-ms base | |
| run: poetry install --extras "testing arrow zarr" | |
| - name: Test dask-ms base | |
| run: poetry run py.test -s -vvv daskms/ | |
| - name: Install dask-ms complete | |
| run: poetry install --extras "testing complete" | |
| - name: Test dask-ms complete | |
| run: poetry run py.test -s -vvv daskms/ | |
| # - name: Debug with tmate on failure | |
| # if: ${{ failure() }} | |
| # uses: mxschmitt/action-tmate@v3 | |
| test_apps: | |
| needs: check_skip | |
| runs-on: ubuntu-latest | |
| if: "!contains(github.event.head_commit.message, '[skip ci]')" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10"] | |
| steps: | |
| - name: Create Cache Hash | |
| run: | | |
| export HASH=$(sha256sum <<EOT | cut -c -16 | |
| ${{ env.POETRY_VERSION }} | |
| EOT | |
| ) | |
| echo "INSTALL_CACHE_HASH=$HASH" >> $GITHUB_ENV | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache Installations | |
| id: cache-installs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.local | |
| key: install-${{ env.INSTALL_CACHE_HASH }}-0 | |
| - name: Install Poetry | |
| if: steps.cache-installs.outputs.cache-hit != 'true' | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - --version ${{ env.POETRY_VERSION }} | |
| - name: Test poetry run | |
| run: poetry --version | |
| - name: Checkout source | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install dask-ms complete | |
| run: poetry install --extras "testing complete" | |
| - name: Test dask-ms applications | |
| run: poetry run py.test -s -vvv --applications daskms/ | |
| # - name: Debug with tmate on failure | |
| # if: ${{ failure() }} | |
| # uses: mxschmitt/action-tmate@v3 | |
| deploy: | |
| needs: [test, test_apps] | |
| runs-on: ubuntu-latest | |
| # Run on a push to a tag or master | |
| if: > | |
| github.event_name == 'push' && | |
| (startsWith(github.event.ref, 'refs/tags') || | |
| github.event.ref == 'refs/heads/master') | |
| steps: | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Install Poetry | |
| uses: abatilo/actions-poetry@v2 | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Checkout source | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - name: Build distributions | |
| run: poetry build | |
| - name: Publish distribution 📦 to Test PyPI | |
| uses: pypa/gh-action-pypi-publish@master | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.test_pypi_token }} | |
| repository_url: https://test.pypi.org/legacy/ | |
| continue-on-error: true | |
| - name: Publish distribution 📦 to PyPI | |
| uses: pypa/gh-action-pypi-publish@master | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.pypi_token }} |