From 2781d40d9ba7df995f43da99d17e692d9b9d943b Mon Sep 17 00:00:00 2001 From: Cole Lyman Date: Thu, 1 Aug 2024 14:19:10 -0600 Subject: [PATCH] Cache conda packages in GIthub Actions (#92) (#466) * Try caching conda packages * Try removing mamba version * Fix path to point to correct env file * Remove defaults channel * List package versions * Move ydiff install and change mamba to conda * Pin to a modern pandas version * Remove pip from env file * Pin pip version and add ydiff install * Add back in defaults channel and strict channel priority * Remove strict priority and pip pin * Try caching the env instead of packages * Switch mamba to conda * Update path to cache conda env * Try caching the tests repo * Add missing ) to tests SHA step * Fix cache key and allow different branches to be set up * Refactor deprecated echo command * Move tests repo to different location for proper caching * Fix spelling mistake and try to fix test caching * Add branch to tests cache name * Always save the tests cache * Don't always save the tests cache (doesn't seem to work) * Remove caching of test repo, doesn't actually improve speed * Ident the steps and remove checking out tests repo into separate directory --- .github/envs/test_env.yml | 4 +- .github/workflows/integration_tests.yml | 165 +++++++++++++----------- 2 files changed, 95 insertions(+), 74 deletions(-) diff --git a/.github/envs/test_env.yml b/.github/envs/test_env.yml index 3e4aa121..581caa2b 100644 --- a/.github/envs/test_env.yml +++ b/.github/envs/test_env.yml @@ -13,4 +13,6 @@ dependencies: - pyparsing=2.3.1 - scipy - matplotlib=3.8.4 - - pandas + - pandas>2 + - pip: + - ydiff diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 1724dea4..ab66a06a 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -19,79 +19,98 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" + - name: Setup Conda Env + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: test_env channels: conda-forge,bioconda,defaults auto-activate-base: false - activate-environment: test_env - environment-file: .github/envs/test_env.yml - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y gcc g++ bowtie2 samtools libsys-hostname-long-perl - pip install ydiff - - - name: Create directory for files - run: | - mkdir ../CRISPResso2_copy - cp -r * ../CRISPResso2_copy - - - name: Copy C2_tests repo - uses: actions/checkout@v3 - with: - repository: edilytics/CRISPResso2_tests - # ref: '' # Use this to specify a branch other than master - - - name: Run Basic - run: | - make basic test print - - - name: Run Params - if: success() || failure() - run: | - make params test print - - - name: Run Prime Editor - if: success() || failure() - run: | - make prime-editor test print - - - name: Run Batch - if: success() || failure() - run: | - make batch test print - - - name: Run Pooled - if: success() || failure() - run: | - make pooled test print - - - name: Run Pooled Mixed Mode - if: success() || failure() - run: | - make pooled-mixed-mode test print - - - name: Run Pooled Mixed Mode Demux - if: success() || failure() - run: | - make pooled-mixed-mode-genome-demux test print - - - name: Run Pooled Paired Sim - if: success() || failure() - run: | - make pooled-paired-sim test print - - - name: Run WGS - if: success() || failure() - run: | - make wgs test print - - - name: Run Compare - if: success() || failure() - run: | - make compare test print + use-mamba: true + + - name: Get Date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache Conda Env + id: cache-env + uses: actions/cache@v3 + env: + # Increase this number to reset the cache if envs/test_env.yml hasn't changed + CACHE_NUMBER: 0 + with: + path: /usr/share/miniconda/envs/test_env + key: + conda-${{ runner.os }}--${{ runner.arch }}--${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('.github/envs/test_env.yml') }} + + - name: Update Conda Env + run: | + conda env update -n test_env -f .github/envs/test_env.yml + if: steps.cache-env.outputs.cache-hit != 'true' + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y gcc g++ bowtie2 samtools libsys-hostname-long-perl + conda list + + - name: Create directory for files + run: | + mkdir ../CRISPResso2_copy + cp -r * ../CRISPResso2_copy + + - name: Copy C2_tests repo + uses: actions/checkout@v3 + with: + repository: edilytics/CRISPResso2_tests + # ref: '' # update to specific branch + + - name: Run Basic + run: | + make basic test print + + - name: Run Params + if: success() || failure() + run: | + make params test print + + - name: Run Prime Editor + if: success() || failure() + run: | + make prime-editor test print + + - name: Run Batch + if: success() || failure() + run: | + make batch test print + + - name: Run Pooled + if: success() || failure() + run: | + make pooled test print + + - name: Run Pooled Mixed Mode + if: success() || failure() + run: | + make pooled-mixed-mode test print + + - name: Run Pooled Mixed Mode Demux + if: success() || failure() + run: | + make pooled-mixed-mode-genome-demux test print + + - name: Run Pooled Paired Sim + if: success() || failure() + run: | + make pooled-paired-sim test print + + - name: Run WGS + if: success() || failure() + run: | + make wgs test print + + - name: Run Compare + if: success() || failure() + run: | + make compare test print