From 5d7f30538bcae21e1de6a738f8fe88b36c0244a3 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:22:21 -0700 Subject: [PATCH 01/13] consolidate notebook action yamls to prevent race condition --- .github/workflows/renderJupyter.yaml | 45 ----------- .github/workflows/renderNotebooks.yaml | 105 +++++++++++++++++++++++++ .github/workflows/renderRMarkdown.yaml | 50 ------------ 3 files changed, 105 insertions(+), 95 deletions(-) delete mode 100644 .github/workflows/renderJupyter.yaml create mode 100644 .github/workflows/renderNotebooks.yaml delete mode 100644 .github/workflows/renderRMarkdown.yaml diff --git a/.github/workflows/renderJupyter.yaml b/.github/workflows/renderJupyter.yaml deleted file mode 100644 index 3af81a5..0000000 --- a/.github/workflows/renderJupyter.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Github action that runs all jupyter notebook after pull request -# Jupyter notebooks will be run in the container/jupyter.Dockerfile - -name: render-jupyter-notebook - -on: - pull_request: - types: - - opened - - reopened - - edited - push: - -jobs: - run_notebooks: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - file: container/jupyter.Dockerfile - context: . - push: false # Set to true if you want to push the image to a registry - tags: notebook-runner - - - - name: Run Docker container to compile JupyterNotebook - uses: addnab/docker-run-action@v3 - with: - image: notebook-runner - options: -v ${{ github.workspace }}:/workspace --rm -u root - run: | - notebooks=$(find /workspace -name "*.ipynb") - if [ -n "$notebooks" ]; then - find /workspace -name "*.ipynb" -exec jupyter nbconvert --to html --execute {} \; - find /workspace -name "*.ipynb" -exec jupyter nbconvert --to markdown --execute {} \; - else - echo "No Jupyter notebooks found. Skipping execution." - fi - - - uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml new file mode 100644 index 0000000..53c2e4b --- /dev/null +++ b/.github/workflows/renderNotebooks.yaml @@ -0,0 +1,105 @@ +# Github action that renders R markdown when PR is opened +# All R markdown is run in container/rmd.Dockerfile + +on: + pull_request: + types: [opened, reopened, edited] + push: + workflow_dispatch: + +name: render-notebooks + +jobs: + render-rmarkdown: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + + - name: Check if Dockerfile changed + id: dockerfile-check + run: | + if git diff --name-only HEAD^ HEAD | grep -q 'container/rmd.Dockerfile'; then + echo "dockerfile_changed=true" >> $GITHUB_ENV + else + echo "dockerfile_changed=false" >> $GITHUB_ENV + fi + + - name: Build and push Docker image + if: env.dockerfile_changed == 'true' + uses: docker/build-push-action@v5 + with: + file: container/rmd.Dockerfile + context: . + push: false # Set to true if you want to push the image to a registry + tags: notebook-runner + + - name: Run Docker container to compile assets + uses: addnab/docker-run-action@v3 + with: + image: notebook-runner + options: -v ${{ github.workspace }}:/workspace --rm -u root + run: | + RMD_FILES=$( find /workspace -name "*Rmd" -o -name "*rmd" ) + if [ -n "$RMD_FILES" ]; then + echo $RMD_FILES + for RMD in $RMD_FILES + do + Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' $RMD + done + else + echo "No R markdown found. Skipping execution." + fi + #Checkout again to avoid race condition with parallel-running Jupyter rendering script. + - uses: actions/checkout@v4 + - uses: stefanzweifel/git-auto-commit-action@v5 + + + render_jupyter: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check if Dockerfile changed + id: dockerfile-check + run: | + if git diff --name-only HEAD^ HEAD | grep -q 'container/jupyter.Dockerfile'; then + echo "dockerfile_changed=true" >> $GITHUB_ENV + else + echo "dockerfile_changed=false" >> $GITHUB_ENV + fi + + - name: Build and push Docker image + if: env.dockerfile_changed == 'true' + uses: docker/build-push-action@v5 + with: + file: container/jupyter.Dockerfile + context: . + push: false # Set to true if you want to push the image to a registry + tags: notebook-runner + + + + - name: Run Docker container to compile JupyterNotebook + uses: addnab/docker-run-action@v3 + with: + image: notebook-runner + options: -v ${{ github.workspace }}:/workspace --rm -u root + run: | + notebooks=$(find /workspace -name "*.ipynb") + if [ -n "$notebooks" ]; then + find /workspace -name "*.ipynb" -exec jupyter nbconvert --to html --execute {} \; + find /workspace -name "*.ipynb" -exec jupyter nbconvert --to markdown --execute {} \; + else + echo "No Jupyter notebooks found. Skipping execution." + fi + + - uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/.github/workflows/renderRMarkdown.yaml b/.github/workflows/renderRMarkdown.yaml deleted file mode 100644 index 6e19947..0000000 --- a/.github/workflows/renderRMarkdown.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Github action that renders R markdown when PR is opened -# All R markdown is run in container/rmd.Dockerfile - -on: - pull_request: - types: [opened, reopened, edited] - push: - workflow_dispatch: - -name: render-rmarkdown - -jobs: - render-rmarkdown: - runs-on: ubuntu-latest - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - file: container/rmd.Dockerfile - context: . - push: false # Set to true if you want to push the image to a registry - tags: notebook-runner - - - name: Run Docker container to compile assets - uses: addnab/docker-run-action@v3 - with: - image: notebook-runner - options: -v ${{ github.workspace }}:/workspace --rm -u root - run: | - RMD_FILES=$( find /workspace -name "*Rmd" -o -name "*rmd" ) - if [ -n "$RMD_FILES" ]; then - echo $RMD_FILES - for RMD in $RMD_FILES - do - Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' $RMD - done - else - echo "No R markdown found. Skipping execution." - fi - #Checkout again to avoid race condition with parallel-running Jupyter rendering script. - - uses: actions/checkout@v4 - - uses: stefanzweifel/git-auto-commit-action@v5 From ad88a93e722a5532b32aa4a8e24597bbbed11096 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:28:39 -0700 Subject: [PATCH 02/13] steps to only compile containers when dockerfile changes (hopefully) --- .github/workflows/renderNotebooks.yaml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 53c2e4b..379064c 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -31,19 +31,26 @@ jobs: echo "dockerfile_changed=false" >> $GITHUB_ENV fi + - name: Extract metadata (tags, labels) for Rmd Docker + id: meta_rmd + uses: docker/metadata-action@v5.3.0 + with: + images: bcgsc/long-pog-rmd + - name: Build and push Docker image if: env.dockerfile_changed == 'true' uses: docker/build-push-action@v5 with: file: container/rmd.Dockerfile context: . - push: false # Set to true if you want to push the image to a registry - tags: notebook-runner + push: true + tags: ${{ steps.meta_rmd.outputs.tags }} + labels: ${{ steps.meta_rmd.outputs.labels }} - name: Run Docker container to compile assets uses: addnab/docker-run-action@v3 with: - image: notebook-runner + image: ${{ steps.meta_rmd.outputs.tags }} options: -v ${{ github.workspace }}:/workspace --rm -u root run: | RMD_FILES=$( find /workspace -name "*Rmd" -o -name "*rmd" ) @@ -77,21 +84,28 @@ jobs: echo "dockerfile_changed=false" >> $GITHUB_ENV fi + - name: Extract metadata (tags, labels) for Jupyter Docker + id: meta_jupyter + uses: docker/metadata-action@v5.3.0 + with: + images: bcgsc/long-pog-jupyter + - name: Build and push Docker image if: env.dockerfile_changed == 'true' uses: docker/build-push-action@v5 with: file: container/jupyter.Dockerfile context: . - push: false # Set to true if you want to push the image to a registry - tags: notebook-runner + push: true + tags: ${{ steps.meta_jupyter.outputs.tags }} + labels: ${{ steps.meta_jupyter.outputs.labels }} - name: Run Docker container to compile JupyterNotebook uses: addnab/docker-run-action@v3 with: - image: notebook-runner + image: ${{ steps.meta_jupyter.outputs.tags }} options: -v ${{ github.workspace }}:/workspace --rm -u root run: | notebooks=$(find /workspace -name "*.ipynb") From cfe30f58b958f0f475f0eff5b24a20b7ece8bf90 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:31:39 -0700 Subject: [PATCH 03/13] lint yaml --- .github/workflows/renderNotebooks.yaml | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 379064c..17c0fe6 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -76,8 +76,8 @@ jobs: uses: actions/checkout@v4 - name: Check if Dockerfile changed - id: dockerfile-check - run: | + id: dockerfile-check + run: | if git diff --name-only HEAD^ HEAD | grep -q 'container/jupyter.Dockerfile'; then echo "dockerfile_changed=true" >> $GITHUB_ENV else @@ -85,20 +85,20 @@ jobs: fi - name: Extract metadata (tags, labels) for Jupyter Docker - id: meta_jupyter - uses: docker/metadata-action@v5.3.0 - with: - images: bcgsc/long-pog-jupyter + id: meta_jupyter + uses: docker/metadata-action@v5.3.0 + with: + images: bcgsc/long-pog-jupyter - name: Build and push Docker image - if: env.dockerfile_changed == 'true' - uses: docker/build-push-action@v5 - with: - file: container/jupyter.Dockerfile - context: . - push: true - tags: ${{ steps.meta_jupyter.outputs.tags }} - labels: ${{ steps.meta_jupyter.outputs.labels }} + if: env.dockerfile_changed == 'true' + uses: docker/build-push-action@v5 + with: + file: container/jupyter.Dockerfile + context: . + push: true + tags: ${{ steps.meta_jupyter.outputs.tags }} + labels: ${{ steps.meta_jupyter.outputs.labels }} From fed2767a283cac34e36c79ff4241226417e98ebc Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:38:20 -0700 Subject: [PATCH 04/13] make jobs sequential to avoid race condition --- .github/workflows/renderNotebooks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 17c0fe6..7771ef9 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -70,6 +70,7 @@ jobs: render_jupyter: runs-on: ubuntu-latest + needs: render-rmarkdown steps: - name: Checkout code From d21ccd98efc7c6343ec655c213b72475a7a34f72 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:43:38 -0700 Subject: [PATCH 05/13] add check to only recompile notebooks that have changed --- .github/workflows/renderNotebooks.yaml | 39 ++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 7771ef9..6675961 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -46,7 +46,25 @@ jobs: push: true tags: ${{ steps.meta_rmd.outputs.tags }} labels: ${{ steps.meta_rmd.outputs.labels }} - + + - name: Run Docker container to compile changed R Markdown files + uses: addnab/docker-run-action@v3 + with: + image: ${{ steps.meta_rmd.outputs.tags }} + options: -v ${{ github.workspace }}:/workspace --rm -u root + run: | + CHANGED_RMD_FILES=$(git diff --name-only HEAD^ HEAD -- '*.Rmd' '*.rmd') + if [ -n "$CHANGED_RMD_FILES" ]; then + echo "Changed R Markdown files:" + echo "$CHANGED_RMD_FILES" + for RMD in $CHANGED_RMD_FILES; do + Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' /workspace/$RMD + done + else + echo "No changed R Markdown files found. Skipping execution." + fi + + - name: Run Docker container to compile assets uses: addnab/docker-run-action@v3 with: @@ -108,13 +126,18 @@ jobs: with: image: ${{ steps.meta_jupyter.outputs.tags }} options: -v ${{ github.workspace }}:/workspace --rm -u root - run: | - notebooks=$(find /workspace -name "*.ipynb") - if [ -n "$notebooks" ]; then - find /workspace -name "*.ipynb" -exec jupyter nbconvert --to html --execute {} \; - find /workspace -name "*.ipynb" -exec jupyter nbconvert --to markdown --execute {} \; + + run: | + changed_notebooks=$(git diff --name-only HEAD^ HEAD -- '*.ipynb') + if [ -n "$changed_notebooks" ]; then + echo "Changed notebooks:" + echo "$changed_notebooks" + for notebook in $changed_notebooks; do + jupyter nbconvert --to html --execute /workspace/$notebook + jupyter nbconvert --to markdown --execute /workspace/$notebook + done else - echo "No Jupyter notebooks found. Skipping execution." - fi + echo "No changed Jupyter notebooks found. Skipping execution." + fi - uses: stefanzweifel/git-auto-commit-action@v5 From 317e34b72cd284709451f34abf00bfe595b5d122 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:49:07 -0700 Subject: [PATCH 06/13] lint yaml again --- .github/workflows/renderNotebooks.yaml | 33 ++++++++++++-------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 6675961..0a28037 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -48,21 +48,21 @@ jobs: labels: ${{ steps.meta_rmd.outputs.labels }} - name: Run Docker container to compile changed R Markdown files - uses: addnab/docker-run-action@v3 - with: - image: ${{ steps.meta_rmd.outputs.tags }} - options: -v ${{ github.workspace }}:/workspace --rm -u root - run: | - CHANGED_RMD_FILES=$(git diff --name-only HEAD^ HEAD -- '*.Rmd' '*.rmd') - if [ -n "$CHANGED_RMD_FILES" ]; then - echo "Changed R Markdown files:" - echo "$CHANGED_RMD_FILES" - for RMD in $CHANGED_RMD_FILES; do - Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' /workspace/$RMD - done - else - echo "No changed R Markdown files found. Skipping execution." - fi + uses: addnab/docker-run-action@v3 + with: + image: ${{ steps.meta_rmd.outputs.tags }} + options: -v ${{ github.workspace }}:/workspace --rm -u root + run: | + CHANGED_RMD_FILES=$(git diff --name-only HEAD^ HEAD -- '*.Rmd' '*.rmd') + if [ -n "$CHANGED_RMD_FILES" ]; then + echo "Changed R Markdown files:" + echo "$CHANGED_RMD_FILES" + for RMD in $CHANGED_RMD_FILES; do + Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' /workspace/$RMD + done + else + echo "No changed R Markdown files found. Skipping execution." + fi - name: Run Docker container to compile assets @@ -80,9 +80,6 @@ jobs: done else echo "No R markdown found. Skipping execution." - fi - #Checkout again to avoid race condition with parallel-running Jupyter rendering script. - - uses: actions/checkout@v4 - uses: stefanzweifel/git-auto-commit-action@v5 From 80569f355d4c6855ec9190f43f9f25037af6fce5 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:52:15 -0700 Subject: [PATCH 07/13] fix yaml tab issue that got through linter --- .github/workflows/renderNotebooks.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 0a28037..eb4ff7d 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -52,17 +52,17 @@ jobs: with: image: ${{ steps.meta_rmd.outputs.tags }} options: -v ${{ github.workspace }}:/workspace --rm -u root - run: | - CHANGED_RMD_FILES=$(git diff --name-only HEAD^ HEAD -- '*.Rmd' '*.rmd') - if [ -n "$CHANGED_RMD_FILES" ]; then - echo "Changed R Markdown files:" - echo "$CHANGED_RMD_FILES" - for RMD in $CHANGED_RMD_FILES; do - Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' /workspace/$RMD - done - else - echo "No changed R Markdown files found. Skipping execution." - fi + run: | + CHANGED_RMD_FILES=$(git diff --name-only HEAD^ HEAD -- '*.Rmd' '*.rmd') + if [ -n "$CHANGED_RMD_FILES" ]; then + echo "Changed R Markdown files:" + echo "$CHANGED_RMD_FILES" + for RMD in $CHANGED_RMD_FILES; do + Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' /workspace/$RMD + done + else + echo "No changed R Markdown files found. Skipping execution." + fi - name: Run Docker container to compile assets From cd98a0e1e0210edff548af15c3e5b1db094ab901 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 12:55:25 -0700 Subject: [PATCH 08/13] fix yaml tab issue that got through linter --- .github/workflows/renderNotebooks.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index eb4ff7d..7d14293 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -124,17 +124,17 @@ jobs: image: ${{ steps.meta_jupyter.outputs.tags }} options: -v ${{ github.workspace }}:/workspace --rm -u root - run: | - changed_notebooks=$(git diff --name-only HEAD^ HEAD -- '*.ipynb') - if [ -n "$changed_notebooks" ]; then - echo "Changed notebooks:" - echo "$changed_notebooks" - for notebook in $changed_notebooks; do - jupyter nbconvert --to html --execute /workspace/$notebook - jupyter nbconvert --to markdown --execute /workspace/$notebook - done - else - echo "No changed Jupyter notebooks found. Skipping execution." - fi + run: | + changed_notebooks=$(git diff --name-only HEAD^ HEAD -- '*.ipynb') + if [ -n "$changed_notebooks" ]; then + echo "Changed notebooks:" + echo "$changed_notebooks" + for notebook in $changed_notebooks; do + jupyter nbconvert --to html --execute /workspace/$notebook + jupyter nbconvert --to markdown --execute /workspace/$notebook + done + else + echo "No changed Jupyter notebooks found. Skipping execution." + fi - uses: stefanzweifel/git-auto-commit-action@v5 From 7fc0ae0ba9a02b2480ead1ecb0ffb5ea5556c60d Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 13:17:38 -0700 Subject: [PATCH 09/13] restore missing fi --- .github/workflows/renderNotebooks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 7d14293..2cec687 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -80,6 +80,8 @@ jobs: done else echo "No R markdown found. Skipping execution." + fi + - uses: stefanzweifel/git-auto-commit-action@v5 From 8b976a4c60d8360aaab3cc8df8bcdd44373d84d5 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 13:55:09 -0700 Subject: [PATCH 10/13] separate file change step so Docker pull only happens if files changed --- .github/workflows/renderNotebooks.yaml | 75 +++++++++++++++++--------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 2cec687..6f3a297 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -65,22 +65,34 @@ jobs: fi - - name: Run Docker container to compile assets + - name: Check for changed R Markdown files + id: check_rmd_files + run: | + CHANGED_RMD_FILES=$(git diff --name-only HEAD^ HEAD -- '*.Rmd' '*.rmd') + if [ -n "$CHANGED_RMD_FILES" ]; then + echo "Changed R Markdown files:" + echo "$CHANGED_RMD_FILES" + echo "::set-output name=has_changed::true" + echo "$CHANGED_RMD_FILES" > changed_rmd_files.txt + else + echo "No changed R Markdown files found." + echo "::set-output name=has_changed::false" + + - name: Run Docker container to compile changed R Markdown files + if: steps.check_rmd_files.outputs.has_changed == 'true' uses: addnab/docker-run-action@v3 with: image: ${{ steps.meta_rmd.outputs.tags }} - options: -v ${{ github.workspace }}:/workspace --rm -u root + options: | + -v ${{ github.workspace }}:/workspace \ + -v /tmp/cache:/root/.cache \ + --rm -u root run: | - RMD_FILES=$( find /workspace -name "*Rmd" -o -name "*rmd" ) - if [ -n "$RMD_FILES" ]; then - echo $RMD_FILES - for RMD in $RMD_FILES - do - Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' $RMD - done - else - echo "No R markdown found. Skipping execution." - fi + CHANGED_RMD_FILES=$(cat changed_rmd_files.txt) + for RMD in $CHANGED_RMD_FILES; do + Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' /workspace/$RMD + done + - uses: stefanzweifel/git-auto-commit-action@v5 @@ -120,23 +132,34 @@ jobs: - - name: Run Docker container to compile JupyterNotebook + - name: Check for changed Jupyter Notebook files + id: check_ipynb_files + run: | + CHANGED_IPYNB_FILES=$(git diff --name-only HEAD^ HEAD -- '*.ipynb') + if [ -n "$CHANGED_IPYNB_FILES" ]; then + echo "Changed Jupyter Notebook files:" + echo "$CHANGED_IPYNB_FILES" + echo "::set-output name=has_changed::true" + echo "$CHANGED_IPYNB_FILES" > changed_ipynb_files.txt + else + echo "No changed Jupyter Notebook files found." + echo "::set-output name=has_changed::false" + + - name: Run Docker container to compile changed Jupyter Notebooks + if: steps.check_ipynb_files.outputs.has_changed == 'true' uses: addnab/docker-run-action@v3 with: image: ${{ steps.meta_jupyter.outputs.tags }} - options: -v ${{ github.workspace }}:/workspace --rm -u root - + options: | + -v ${{ github.workspace }}:/workspace \ + -v /tmp/cache:/root/.cache \ + --rm -u root run: | - changed_notebooks=$(git diff --name-only HEAD^ HEAD -- '*.ipynb') - if [ -n "$changed_notebooks" ]; then - echo "Changed notebooks:" - echo "$changed_notebooks" - for notebook in $changed_notebooks; do - jupyter nbconvert --to html --execute /workspace/$notebook - jupyter nbconvert --to markdown --execute /workspace/$notebook - done - else - echo "No changed Jupyter notebooks found. Skipping execution." - fi + CHANGED_IPYNB_FILES=$(cat changed_ipynb_files.txt) + for NOTEBOOK in $CHANGED_IPYNB_FILES; do + jupyter nbconvert --to html --execute /workspace/$NOTEBOOK + jupyter nbconvert --to markdown --execute /workspace/$NOTEBOOK + done + - uses: stefanzweifel/git-auto-commit-action@v5 From 88e9dcb235959960acc2b7913dd3868e009a19bd Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 13:56:57 -0700 Subject: [PATCH 11/13] remove duplicated section --- .github/workflows/renderNotebooks.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index 6f3a297..c343926 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -47,23 +47,6 @@ jobs: tags: ${{ steps.meta_rmd.outputs.tags }} labels: ${{ steps.meta_rmd.outputs.labels }} - - name: Run Docker container to compile changed R Markdown files - uses: addnab/docker-run-action@v3 - with: - image: ${{ steps.meta_rmd.outputs.tags }} - options: -v ${{ github.workspace }}:/workspace --rm -u root - run: | - CHANGED_RMD_FILES=$(git diff --name-only HEAD^ HEAD -- '*.Rmd' '*.rmd') - if [ -n "$CHANGED_RMD_FILES" ]; then - echo "Changed R Markdown files:" - echo "$CHANGED_RMD_FILES" - for RMD in $CHANGED_RMD_FILES; do - Rscript -e 'for (f in commandArgs(TRUE)) if (file.exists(f)) rmarkdown::render(f, output_format = "all")' /workspace/$RMD - done - else - echo "No changed R Markdown files found. Skipping execution." - fi - - name: Check for changed R Markdown files id: check_rmd_files From 9928c7be25e4ea4fc03ab815f171ba23165fa5e2 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 13:58:29 -0700 Subject: [PATCH 12/13] restore missing fi... again --- .github/workflows/renderNotebooks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index c343926..d93dd5b 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -60,6 +60,7 @@ jobs: else echo "No changed R Markdown files found." echo "::set-output name=has_changed::false" + fi - name: Run Docker container to compile changed R Markdown files if: steps.check_rmd_files.outputs.has_changed == 'true' @@ -127,6 +128,7 @@ jobs: else echo "No changed Jupyter Notebook files found." echo "::set-output name=has_changed::false" + fi - name: Run Docker container to compile changed Jupyter Notebooks if: steps.check_ipynb_files.outputs.has_changed == 'true' From fa0ca2eeb356b40a5dd866430625f567be947083 Mon Sep 17 00:00:00 2001 From: koneill Date: Thu, 5 Sep 2024 14:03:45 -0700 Subject: [PATCH 13/13] correct fetch-depth to allow file change check --- .github/workflows/renderNotebooks.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/renderNotebooks.yaml b/.github/workflows/renderNotebooks.yaml index d93dd5b..8c2775a 100644 --- a/.github/workflows/renderNotebooks.yaml +++ b/.github/workflows/renderNotebooks.yaml @@ -88,6 +88,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check if Dockerfile changed id: dockerfile-check