Continuous Integration #1060
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Integration | |
| on: | |
| push: | |
| branches: [ master ] | |
| pull_request: | |
| branches: [ master ] | |
| schedule: | |
| # run this every week at 12 noon on Monday | |
| - cron: '0 12 * * 1' | |
| env: | |
| EXAMPLE_DATA_FILE_VERSION: v5 | |
| # Setting this did not appear to work. Instead will need to | |
| # find/replace when changing it. | |
| # CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe:latest | |
| jobs: | |
| # Run a download step first so that it is in | |
| # the cache, because all the other jobs will be running | |
| # at the same time and so might miss it. | |
| Download_Data: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Cache example data | |
| id: cache-example | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-example-data | |
| with: | |
| path: example.tar.gz | |
| # update this when we change package contents and want | |
| # to force an update | |
| key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }} | |
| - name: Download test data | |
| if: steps.cache-example.outputs.cache-hit != 'true' | |
| run: | | |
| wget -O example.tar.gz "https://portal.nersc.gov/cfs/lsst/txpipe/data/example.tar.gz" | |
| Run_Installer: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-14] | |
| include: | |
| - os: ubuntu-latest | |
| INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget | |
| - os: macos-14 | |
| INSTALL_DEPS: echo nothing to do | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Install wget | |
| run: | | |
| ${{ matrix.INSTALL_DEPS }} | |
| - name: Cache conda environment | |
| id: cache-conda | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-conda-environment | |
| with: | |
| path: ./conda | |
| key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }} | |
| fail-on-cache-miss: false | |
| - name: Run installer | |
| if: steps.cache-conda.outputs.cache-hit != 'true' | |
| run: | | |
| ./bin/install.sh | |
| Unit_Tests: | |
| runs-on: ${{ matrix.os }} | |
| needs: Run_Installer | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-14] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Restore conda environment | |
| id: restore-conda | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-conda-environment | |
| with: | |
| path: ./conda | |
| key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }} | |
| fail-on-cache-miss: true | |
| - name: Test with pytest | |
| run: | | |
| source conda/bin/activate | |
| ceci --version | |
| pytest txpipe | |
| Pipelines: | |
| runs-on: ${{ matrix.os }} | |
| needs: [Run_Installer, Download_Data] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-14] | |
| pipeline: [metadetect, metacal, redmagic, lensfit, metadetect_source_only, mock_shear] | |
| include: | |
| - os: ubuntu-latest | |
| INSTALL_DEPS: echo nothing to do | |
| - os: macos-14 | |
| INSTALL_DEPS: brew update-reset && brew install --cask basictex && eval "$(/usr/libexec/path_helper)" && pdflatex --version | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Install dependencies | |
| run: | | |
| ${{ matrix.INSTALL_DEPS }} | |
| - name: Restore conda environment | |
| id: restore-conda | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-conda-environment | |
| with: | |
| path: ./conda | |
| key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }} | |
| fail-on-cache-miss: true | |
| - name: Restore example data | |
| uses: actions/cache@v4 | |
| with: | |
| path: example.tar.gz | |
| key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }} | |
| fail-on-cache-miss: true | |
| - name: Extract test data | |
| run: | | |
| tar -zxvf example.tar.gz | |
| - name: Run pipeline | |
| run: | | |
| mkdir -p cache/workspaces | |
| source ./conda/bin/activate | |
| # Just for the metadetect source pipeline | |
| # we activate the debug mode with an env var | |
| if [ "${{ matrix.pipeline }}" == "metadetect_source_only" ]; then | |
| export TX_DASK_DEBUG=1 | |
| fi | |
| ceci examples/${{ matrix.pipeline }}/pipeline.yml | |
| if [ "${{ matrix.pipeline }}" == "mock_shear" ]; then | |
| test -f data/example/outputs_mock_shear/binned_shear_catalog.hdf5 | |
| else | |
| test -f data/example/outputs_${{ matrix.pipeline }}/shear_xi_plus.png | |
| fi | |
| - name: Show logs | |
| if: always() | |
| run: | | |
| # always run | |
| cat data/example/logs_${{ matrix.pipeline }}/* | |
| - name: Create output report | |
| if: matrix.os == 'macos-14' | |
| run: | | |
| source ./conda/bin/activate | |
| eval "$(/usr/libexec/path_helper)" | |
| python bin/make_output_report.py data/example/outputs_${{ matrix.pipeline }} ${{ matrix.pipeline }}-report.md | |
| pandoc ${{ matrix.pipeline }}-report.md -o ${{ matrix.pipeline }}-report.pdf | |
| - name: Upload output report | |
| if: matrix.os == 'macos-14' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.pipeline }}-report | |
| path: ${{ matrix.pipeline }}-report.pdf |