From 63e8e043228f0caff0f11a34819742332c635dab Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Thu, 20 Jul 2023 12:43:12 -0600 Subject: [PATCH 01/39] Add a status badge to the README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 210bb657..d8a49598 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![ExascaleSandboxTests](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yml/badge.svg)](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yml) + ``` This repository is a scientific product and is not official communication of the National Oceanic and Atmospheric Administration, or the United States From a6e86ed3c77c0ab630f08138e24bc29fd5b0c414 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Thu, 20 Jul 2023 15:00:33 -0600 Subject: [PATCH 02/39] Don't use caching in CI when pushing container images to registry --- .github/workflows/docker-slurm.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 59698a4e..2d482b5d 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -106,8 +106,8 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 @@ -116,8 +116,8 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node uses: docker/build-push-action@v4 @@ -126,5 +126,5 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From b88364ffa890415e0ce455dd0fb9f7403ab594d2 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Thu, 20 Jul 2023 19:31:48 -0600 Subject: [PATCH 03/39] Remove use of docker image cache in push steps --- .github/workflows/docker-slurm.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 2d482b5d..2277428f 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -106,8 +106,6 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 @@ -116,8 +114,6 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node uses: docker/build-push-action@v4 @@ -126,5 +122,3 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From 2f4455ed5368fd2a8cdf2517b5c23eeab7120d67 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 21 Jul 2023 09:52:07 -0600 Subject: [PATCH 04/39] Try cache-from at the beginning and cache-to at the end --- .github/workflows/docker-slurm.yml | 89 +++++++++++++++--------------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 2277428f..efcabfa3 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -34,7 +34,7 @@ jobs: load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Build master uses: docker/build-push-action@v4 @@ -45,7 +45,7 @@ jobs: load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Build node uses: docker/build-push-action@v4 @@ -56,48 +56,48 @@ jobs: load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - - - name: Start Slurm cluster containers - run: | - cd docker - export DOCKER_CLIENT_TIMEOUT=600 - export COMPOSE_HTTP_TIMEOUT=600 - docker-compose -f docker-compose.yml up -d --no-build - - - name: Create ssh keys in cluster - run: | - docker exec frontend ssh-keygen -t rsa -f /home/admin/.ssh/id_rsa -N "" - docker exec frontend cp /home/admin/.ssh/id_rsa.pub /home/admin/.ssh/authorized_keys - - - name: Copy test scripts to work directory - run: | - docker exec frontend bash -l -c "mkdir -p work; cd work ; cp ../test/*.py . ; cp ../test/*.f90 ." - - - name: Parsl hello test - run: | - docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_hello.py" - - - name: Parsl/Flux hello MPI test - run: | - docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py" - docker exec frontend bash -l -c "cat work/resource_list.txt" - docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" -# Uncomment this step to enable interactive debugging +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max # - -# name: Debug session -# if: ${{ failure() }} -# uses: mxschmitt/action-tmate@v3 -# timeout-minutes: 60 -# with: -# limit-access-to-actor: true - - - name: Shut down Slurm cluster containers - run: | - cd docker - export DOCKER_CLIENT_TIMEOUT=600 - export COMPOSE_HTTP_TIMEOUT=600 - docker-compose -f docker-compose.yml down +# name: Start Slurm cluster containers +# run: | +# cd docker +# export DOCKER_CLIENT_TIMEOUT=600 +# export COMPOSE_HTTP_TIMEOUT=600 +# docker-compose -f docker-compose.yml up -d --no-build +# - +# name: Create ssh keys in cluster +# run: | +# docker exec frontend ssh-keygen -t rsa -f /home/admin/.ssh/id_rsa -N "" +# docker exec frontend cp /home/admin/.ssh/id_rsa.pub /home/admin/.ssh/authorized_keys +# - +# name: Copy test scripts to work directory +# run: | +# docker exec frontend bash -l -c "mkdir -p work; cd work ; cp ../test/*.py . ; cp ../test/*.f90 ." +# - +# name: Parsl hello test +# run: | +# docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_hello.py" +# - +# name: Parsl/Flux hello MPI test +# run: | +# docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py" +# docker exec frontend bash -l -c "cat work/resource_list.txt" +# docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" +## Uncomment this step to enable interactive debugging +## - +## name: Debug session +## if: ${{ failure() }} +## uses: mxschmitt/action-tmate@v3 +## timeout-minutes: 60 +## with: +## limit-access-to-actor: true +# - +# name: Shut down Slurm cluster containers +# run: | +# cd docker +# export DOCKER_CLIENT_TIMEOUT=600 +# export COMPOSE_HTTP_TIMEOUT=600 +# docker-compose -f docker-compose.yml down - name: Push frontend uses: docker/build-push-action@v4 @@ -106,6 +106,7 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 @@ -114,6 +115,7 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node uses: docker/build-push-action@v4 @@ -122,3 +124,4 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From eda66e9f507b70281321cdd4620d4640fa942311 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 21 Jul 2023 11:02:48 -0600 Subject: [PATCH 05/39] Remove checkout action as it's not needed --- .github/workflows/docker-slurm.yml | 68 +++++++++++++++--------------- docker/docker-compose.yml | 28 ++++++------ 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index efcabfa3..afe73f33 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -11,9 +11,9 @@ jobs: docker: runs-on: ubuntu-latest steps: - - - name: Checkout repository - uses: actions/checkout@v3 +# - +# name: Checkout repository +# uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -24,16 +24,16 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build frontend - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/frontend/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# - +# name: Build frontend +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/frontend/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache # cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Build master @@ -44,8 +44,8 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Build node uses: docker/build-push-action@v4 @@ -55,15 +55,15 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max -# - -# name: Start Slurm cluster containers -# run: | -# cd docker -# export DOCKER_CLIENT_TIMEOUT=600 -# export COMPOSE_HTTP_TIMEOUT=600 -# docker-compose -f docker-compose.yml up -d --no-build + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + - + name: Start Slurm cluster containers + run: | + cd docker + export DOCKER_CLIENT_TIMEOUT=600 + export COMPOSE_HTTP_TIMEOUT=600 + docker-compose -f docker-compose.yml up -d --no-build # - # name: Create ssh keys in cluster # run: | @@ -91,13 +91,13 @@ jobs: ## timeout-minutes: 60 ## with: ## limit-access-to-actor: true -# - -# name: Shut down Slurm cluster containers -# run: | -# cd docker -# export DOCKER_CLIENT_TIMEOUT=600 -# export COMPOSE_HTTP_TIMEOUT=600 -# docker-compose -f docker-compose.yml down + - + name: Shut down Slurm cluster containers + run: | + cd docker + export DOCKER_CLIENT_TIMEOUT=600 + export COMPOSE_HTTP_TIMEOUT=600 + docker-compose -f docker-compose.yml down - name: Push frontend uses: docker/build-push-action@v4 @@ -106,7 +106,7 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 @@ -115,7 +115,7 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node uses: docker/build-push-action@v4 @@ -124,4 +124,4 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 30cf962e..e0903fe1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,18 +1,18 @@ services: - slurmfrontend: - build: - context: ./ - dockerfile: ./frontend/Dockerfile - image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - container_name: frontend - hostname: slurmfrontend - user: admin - volumes: - - shared-vol:/home/admin - - opt-vol:/opt - - ./test:/home/admin/test - ports: - - 8888:8888 +# slurmfrontend: +# build: +# context: ./ +# dockerfile: ./frontend/Dockerfile +# image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest +# container_name: frontend +# hostname: slurmfrontend +# user: admin +# volumes: +# - shared-vol:/home/admin +# - opt-vol:/opt +# - ./test:/home/admin/test +# ports: +# - 8888:8888 slurmmaster: build: context: ./ From 7706d12cbe03b176854d81ace5cf45c5dfa559b2 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 21 Jul 2023 11:15:41 -0600 Subject: [PATCH 06/39] Put checkout action back in --- .github/workflows/docker-slurm.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index afe73f33..d38e5f00 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -11,9 +11,9 @@ jobs: docker: runs-on: ubuntu-latest steps: -# - -# name: Checkout repository -# uses: actions/checkout@v3 + - + name: Checkout repository + uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 From 97a895592e9c2b592e4a6c5b877092bd99221637 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 21 Jul 2023 11:22:40 -0600 Subject: [PATCH 07/39] Don't push frontend for now --- .github/workflows/docker-slurm.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index d38e5f00..1c830096 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -98,15 +98,15 @@ jobs: export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 docker-compose -f docker-compose.yml down - - - name: Push frontend - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/frontend/Dockerfile - push: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# - +# name: Push frontend +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/frontend/Dockerfile +# push: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest +## cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 From 37e81061a09a390b4747b3a6b75083f1b53f0b98 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 21 Jul 2023 11:31:49 -0600 Subject: [PATCH 08/39] Try pulling the caches before trying to use them --- .github/workflows/docker-slurm.yml | 46 +++++++++++++++++------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 1c830096..292f1b49 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -24,17 +24,23 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} -# - -# name: Build frontend -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/frontend/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + - + name: Pull caches + run: | + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true + - + name: Build frontend + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/frontend/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Build master uses: docker/build-push-action@v4 @@ -98,15 +104,15 @@ jobs: export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 docker-compose -f docker-compose.yml down -# - -# name: Push frontend -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/frontend/Dockerfile -# push: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -## cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + - + name: Push frontend + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/frontend/Dockerfile + push: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 From aa6324c9a61a5e8fb20473a15c7954d9f07ec18e Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 21 Jul 2023 14:03:34 -0600 Subject: [PATCH 09/39] Add the missing pieces back in and try again --- .github/workflows/docker-slurm.yml | 52 +++++++++++++++--------------- docker/docker-compose.yml | 28 ++++++++-------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 292f1b49..2729d582 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -70,33 +70,33 @@ jobs: export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 docker-compose -f docker-compose.yml up -d --no-build + - + name: Create ssh keys in cluster + run: | + docker exec frontend ssh-keygen -t rsa -f /home/admin/.ssh/id_rsa -N "" + docker exec frontend cp /home/admin/.ssh/id_rsa.pub /home/admin/.ssh/authorized_keys + - + name: Copy test scripts to work directory + run: | + docker exec frontend bash -l -c "mkdir -p work; cd work ; cp ../test/*.py . ; cp ../test/*.f90 ." + - + name: Parsl hello test + run: | + docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_hello.py" + - + name: Parsl/Flux hello MPI test + run: | + docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py" + docker exec frontend bash -l -c "cat work/resource_list.txt" + docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" +# Uncomment this step to enable interactive debugging # - -# name: Create ssh keys in cluster -# run: | -# docker exec frontend ssh-keygen -t rsa -f /home/admin/.ssh/id_rsa -N "" -# docker exec frontend cp /home/admin/.ssh/id_rsa.pub /home/admin/.ssh/authorized_keys -# - -# name: Copy test scripts to work directory -# run: | -# docker exec frontend bash -l -c "mkdir -p work; cd work ; cp ../test/*.py . ; cp ../test/*.f90 ." -# - -# name: Parsl hello test -# run: | -# docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_hello.py" -# - -# name: Parsl/Flux hello MPI test -# run: | -# docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py" -# docker exec frontend bash -l -c "cat work/resource_list.txt" -# docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" -## Uncomment this step to enable interactive debugging -## - -## name: Debug session -## if: ${{ failure() }} -## uses: mxschmitt/action-tmate@v3 -## timeout-minutes: 60 -## with: -## limit-access-to-actor: true +# name: Debug session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 +# timeout-minutes: 60 +# with: +# limit-access-to-actor: true - name: Shut down Slurm cluster containers run: | diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e0903fe1..30cf962e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,18 +1,18 @@ services: -# slurmfrontend: -# build: -# context: ./ -# dockerfile: ./frontend/Dockerfile -# image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# container_name: frontend -# hostname: slurmfrontend -# user: admin -# volumes: -# - shared-vol:/home/admin -# - opt-vol:/opt -# - ./test:/home/admin/test -# ports: -# - 8888:8888 + slurmfrontend: + build: + context: ./ + dockerfile: ./frontend/Dockerfile + image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + container_name: frontend + hostname: slurmfrontend + user: admin + volumes: + - shared-vol:/home/admin + - opt-vol:/opt + - ./test:/home/admin/test + ports: + - 8888:8888 slurmmaster: build: context: ./ From cba9c8e7db6b65489f652019e82724e561b2ee49 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 11 Aug 2023 15:02:33 -0600 Subject: [PATCH 10/39] Forget about the cache... --- .github/workflows/docker-slurm.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 2729d582..17a3edd6 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -27,9 +27,9 @@ jobs: - name: Pull caches run: | - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest || true - name: Build frontend uses: docker/build-push-action@v4 @@ -39,8 +39,8 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Build master uses: docker/build-push-action@v4 @@ -50,8 +50,8 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Build node uses: docker/build-push-action@v4 @@ -61,8 +61,8 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | From d88e6180ec894ef01becc0c0765f8137b1dbb5c4 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 11 Aug 2023 16:21:25 -0600 Subject: [PATCH 11/39] Back to using cache --- .github/workflows/docker-slurm.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 17a3edd6..9915f78a 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -27,9 +27,9 @@ jobs: - name: Pull caches run: | - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true - name: Build frontend uses: docker/build-push-action@v4 @@ -38,8 +38,8 @@ jobs: file: ./docker/frontend/Dockerfile push: false load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache # cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Build master @@ -49,8 +49,8 @@ jobs: file: ./docker/master/Dockerfile push: false load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache # cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Build node @@ -60,8 +60,8 @@ jobs: file: ./docker/node/Dockerfile push: false load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache # cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers @@ -112,7 +112,7 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 @@ -121,7 +121,7 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node uses: docker/build-push-action@v4 @@ -130,4 +130,4 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From 540636ad49d9f2d23d916b19020b634b4ffe32a9 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 12 Aug 2023 13:38:43 -0600 Subject: [PATCH 12/39] Try using latest as cache --- .github/workflows/docker-slurm.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 9915f78a..e93039f1 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -24,12 +24,12 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull caches - run: | - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true +# - +# name: Pull caches +# run: | +# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true +# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true +# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true - name: Build frontend uses: docker/build-push-action@v4 @@ -38,8 +38,8 @@ jobs: file: ./docker/frontend/Dockerfile push: false load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest # cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Build master @@ -49,8 +49,8 @@ jobs: file: ./docker/master/Dockerfile push: false load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest # cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Build node @@ -60,8 +60,8 @@ jobs: file: ./docker/node/Dockerfile push: false load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest # cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers @@ -112,7 +112,7 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 @@ -121,7 +121,7 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node uses: docker/build-push-action@v4 @@ -130,4 +130,4 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From 457264402dba6b09cacb1052722b03ef18583157 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Mon, 14 Aug 2023 08:34:36 -0600 Subject: [PATCH 13/39] debug --- .github/workflows/docker-slurm.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index e93039f1..ef921368 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -39,8 +39,8 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Build master uses: docker/build-push-action@v4 @@ -50,8 +50,8 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Build node uses: docker/build-push-action@v4 @@ -61,8 +61,8 @@ jobs: push: false load: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | From 5f2dee0a299d39b4a5b6158d6262c6a0529faa73 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Mon, 14 Aug 2023 09:56:53 -0600 Subject: [PATCH 14/39] debug --- .github/workflows/docker-slurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index ef921368..f35a830b 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -69,7 +69,7 @@ jobs: cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker-compose -f docker-compose.yml up -d --no-build + docker-compose -f docker-compose.yml up -d - name: Create ssh keys in cluster run: | From 7352cbb621fba034af44adbbb98fb9be943be0af Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Mon, 14 Aug 2023 10:42:17 -0600 Subject: [PATCH 15/39] debug --- .github/workflows/docker-slurm.yml | 66 +++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index f35a830b..c61b544b 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -30,39 +30,39 @@ jobs: # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true - - - name: Build frontend - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/frontend/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - - - name: Build master - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/master/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - - - name: Build node - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/node/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# - +# name: Build frontend +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/frontend/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# - +# name: Build master +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/master/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# - +# name: Build node +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/node/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | From 1e719f399e9a371ec45f45103d2923d41e255adf Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Mon, 14 Aug 2023 15:19:15 -0600 Subject: [PATCH 16/39] Update flux --- docker/install/install_flux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/install/install_flux.sh b/docker/install/install_flux.sh index 7429bf25..86306d34 100755 --- a/docker/install/install_flux.sh +++ b/docker/install/install_flux.sh @@ -55,8 +55,8 @@ spack add miniconda3%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} spack install # Install flux components -spack add flux-core@0.49.0%${SPACK_ENV_COMPILER} ^python@3.9 ${TARGET_ARCH_OPT} -spack add flux-sched@0.27.0%${SPACK_ENV_COMPILER} ^python@3.9 ${TARGET_ARCH_OPT} +spack add flux-core@0.53.0%${SPACK_ENV_COMPILER} ^python@3.9 ${TARGET_ARCH_OPT} +spack add flux-sched@0.28.0%${SPACK_ENV_COMPILER} ^python@3.9 ${TARGET_ARCH_OPT} spack concretize -f spack install --no-checksum From 8199257c9254aeedebb11e8cb240ecf692abc7fe Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 18 Aug 2023 14:51:50 -0600 Subject: [PATCH 17/39] Add spack env load in flux app --- docker/frontend/Dockerfile | 12 +++++-- docker/install/install_base.sh | 20 ++++++------ docker/install/install_flux.sh | 23 +++++++------- docker/install/install_parsl.sh | 49 +++++++++++++++++++++++++++-- docker/test/parsl_mpi_flux_hello.py | 5 ++- 5 files changed, 84 insertions(+), 25 deletions(-) diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 95ad45e1..9a94f6ca 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -39,11 +39,19 @@ COPY ./install /tmp/install USER admin SHELL ["/bin/bash", "-c"] +# Install the Spack workflow environment +#RUN . /opt/spack/share/spack/setup-env.sh \ +# && cd /tmp/install \ +# && ./install_base.sh + +# Install the Spack workflow environment +RUN . /opt/spack/share/spack/setup-env.sh \ + && cd /tmp/install \ + && ./install_flux.sh + # Install the Spack workflow environment RUN . /opt/spack/share/spack/setup-env.sh \ && cd /tmp/install \ - && ./install_base.sh \ - && ./install_flux.sh \ && ./install_parsl.sh # Set up user shell init diff --git a/docker/install/install_base.sh b/docker/install/install_base.sh index 9ed0d512..93de8ff0 100755 --- a/docker/install/install_base.sh +++ b/docker/install/install_base.sh @@ -46,15 +46,17 @@ spack env activate ${SPACK_ENV_NAME} spack compiler find # Install python tools and miniconda3 -spack add python@3.9%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-pytest%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-pylint%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-flake8%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-mypy%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-black%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-ply%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add miniconda3%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +spack add python@3.9.15%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-pytest%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-pylint%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-flake8%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-mypy%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-black%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-ply%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +spack add miniconda3%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} +spack add flux-core@0.53.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} +spack add flux-sched@0.28.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} spack concretize -spack install +spack install --no-checksum exit 0 diff --git a/docker/install/install_flux.sh b/docker/install/install_flux.sh index 86306d34..cee4115b 100755 --- a/docker/install/install_flux.sh +++ b/docker/install/install_flux.sh @@ -44,19 +44,20 @@ spack env create ${SPACK_ENV_NAME} || true spack env activate ${SPACK_ENV_NAME} # Install python tools -spack add python@3.9%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-pytest%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-pylint%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-flake8%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-mypy%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-black%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add py-ply%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add miniconda3%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack install +#spack add python@3.9.15%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-pip%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-pytest%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-pylint%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-flake8%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-mypy%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-black%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add py-ply%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} +#spack add miniconda3%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} +#spack install # Install flux components -spack add flux-core@0.53.0%${SPACK_ENV_COMPILER} ^python@3.9 ${TARGET_ARCH_OPT} -spack add flux-sched@0.28.0%${SPACK_ENV_COMPILER} ^python@3.9 ${TARGET_ARCH_OPT} +spack add flux-core@0.53.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} +spack add flux-sched@0.28.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} spack concretize -f spack install --no-checksum diff --git a/docker/install/install_parsl.sh b/docker/install/install_parsl.sh index dda7686f..12df9b81 100755 --- a/docker/install/install_parsl.sh +++ b/docker/install/install_parsl.sh @@ -1,7 +1,52 @@ #!/bin/env bash +# These can be customized to suit individual needs +DEFAULT_GCC_VERSION=$(/usr/bin/gcc --version | head -1 | sed -e 's/([^()]*)//g' | awk '{print $2}') # Version of system default gcc +DEFAULT_COMPILER="gcc@${DEFAULT_GCC_VERSION}" # Default system compiler used to build newer gcc + +SPACK_ENV_NAME="flux" # Name of spack environment to create +SPACK_ENV_COMPILER="gcc@9.4.0" # Compiler to use to build the spack environment +TARGET_ARCH_OPT="target=x86_64" # Compiler architecture build target + +################################################################################ +# help # +################################################################################ +help() +{ + # Display help + echo "Installs parsl into flux Spack environment" + echo + echo "Usage: install_parsl.sh" + echo +} + +# Get the location of Spack so we can update permissions +if [[ $(which spack 2>/dev/null) ]]; then + SPACK_DIR=$(dirname $(dirname $(which spack))) +else + echo "Cannot find Spack." + echo + echo "Please install Spack and/or source Spack's environment setup script: .../spack/share/setup-env.sh" + echo + exit 1 +fi + +set -eu + +# Configure spack +spack config add concretizer:unify:true +spack config add concretizer:reuse:true +spack config add config:db_lock_timeout:300 +spack config add config:install_tree:padded_length:128 + +# Configure spack environment +spack env create ${SPACK_ENV_NAME} || true +spack env activate ${SPACK_ENV_NAME} + spack env activate flux -pip install parsl -pip install 'dill @ git+https://github.com/uqfoundation/dill' +spack add py-pip%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} +spack install +python -m pip install parsl +#pip install 'dill @ git+https://github.com/uqfoundation/dill' exit 0 diff --git a/docker/test/parsl_mpi_flux_hello.py b/docker/test/parsl_mpi_flux_hello.py index 0318df05..0fbb3c0e 100755 --- a/docker/test/parsl_mpi_flux_hello.py +++ b/docker/test/parsl_mpi_flux_hello.py @@ -26,7 +26,8 @@ walltime='00:10:00', launcher=SimpleLauncher(), worker_init=''' -# FluxExecutor ignores worker_init +. /opt/spack/share/spack/setup-env.sh +spack env activate flux ''', ), ) @@ -45,6 +46,8 @@ @bash_app def resource_list(): return ''' + . /opt/spack/share/spack/setup-env.sh + spack env activate flux flux resource list > resource_list.txt ''' From c16ad60924d560d57338e17a52e6ebeff8c7a248 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 18 Aug 2023 16:16:41 -0600 Subject: [PATCH 18/39] debug --- .github/workflows/docker-slurm.yml | 86 +++++++++++++++--------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index c61b544b..a2ebb5eb 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -30,39 +30,39 @@ jobs: # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true -# - -# name: Build frontend -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/frontend/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max -# - -# name: Build master -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/master/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max -# - -# name: Build node -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/node/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + - + name: Build frontend + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/frontend/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + - + name: Build master + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/master/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + - + name: Build node + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/node/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | @@ -90,13 +90,13 @@ jobs: docker exec frontend bash -l -c "cat work/resource_list.txt" docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" # Uncomment this step to enable interactive debugging -# - -# name: Debug session -# if: ${{ failure() }} -# uses: mxschmitt/action-tmate@v3 -# timeout-minutes: 60 -# with: -# limit-access-to-actor: true + - + name: Debug session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 60 + with: + limit-access-to-actor: true - name: Shut down Slurm cluster containers run: | @@ -112,7 +112,7 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master uses: docker/build-push-action@v4 @@ -121,7 +121,7 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node uses: docker/build-push-action@v4 @@ -130,4 +130,4 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From e18062d254525b985517e9d6a5580d6f4f92fa49 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 18 Aug 2023 18:57:28 -0600 Subject: [PATCH 19/39] debug --- .github/workflows/docker-slurm.yml | 66 +++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index a2ebb5eb..054b4a50 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -30,39 +30,39 @@ jobs: # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true - - - name: Build frontend - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/frontend/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - - - name: Build master - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/master/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - - - name: Build node - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/node/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# - +# name: Build frontend +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/frontend/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# - +# name: Build master +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/master/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# - +# name: Build node +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/node/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | From 2ece6033f0c376bd7d56289a544c75ce30b4616d Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 18 Aug 2023 20:35:45 -0600 Subject: [PATCH 20/39] debug --- docker/frontend/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 9a94f6ca..1a103632 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -52,7 +52,8 @@ RUN . /opt/spack/share/spack/setup-env.sh \ # Install the Spack workflow environment RUN . /opt/spack/share/spack/setup-env.sh \ && cd /tmp/install \ - && ./install_parsl.sh + && ./install_parsl.sh \ + && ./install_flux.sh # Set up user shell init RUN echo ". /opt/spack/share/spack/setup-env.sh" >> /home/admin/.bash_profile \ From 1ee07ec7f75049dfc0b7f17160e270267feac437 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Fri, 18 Aug 2023 21:57:22 -0600 Subject: [PATCH 21/39] debug --- .github/workflows/docker-slurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 054b4a50..9b6e365a 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -86,7 +86,7 @@ jobs: - name: Parsl/Flux hello MPI test run: | - docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py" + docker exec frontend bash -l -c "spack env activate flux; cd /tmp/install; ./install_flux.sh; cd; cd work ; ./parsl_mpi_flux_hello.py" docker exec frontend bash -l -c "cat work/resource_list.txt" docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" # Uncomment this step to enable interactive debugging From 04bbc1fbb1e57a54c9d1983de468c0ba7ee31e92 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 07:27:31 -0600 Subject: [PATCH 22/39] Debug --- .github/workflows/docker-slurm.yml | 77 ++++++++++++++++-------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 9b6e365a..6d93948b 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -30,46 +30,46 @@ jobs: # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true # docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true -# - -# name: Build frontend -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/frontend/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max -# - -# name: Build master -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/master/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max -# - -# name: Build node -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/node/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + - + name: Build frontend + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/frontend/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + - + name: Build master + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/master/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + - + name: Build node + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/node/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker-compose -f docker-compose.yml up -d + docker-compose -f docker-compose.yml up -d --no-build - name: Create ssh keys in cluster run: | @@ -79,6 +79,10 @@ jobs: name: Copy test scripts to work directory run: | docker exec frontend bash -l -c "mkdir -p work; cd work ; cp ../test/*.py . ; cp ../test/*.f90 ." + - + name: Fix the flux install # This is a workaround to deal with bugs caused by spack/pip conflicts + run: | + docker exec frontend bash -l -c "spack env activate flux; cd /tmp/install; ./install_flux.sh" - name: Parsl hello test run: | @@ -86,7 +90,7 @@ jobs: - name: Parsl/Flux hello MPI test run: | - docker exec frontend bash -l -c "spack env activate flux; cd /tmp/install; ./install_flux.sh; cd; cd work ; ./parsl_mpi_flux_hello.py" + docker exec frontend bash -l -c "cd; cd work ; ./parsl_mpi_flux_hello.py" docker exec frontend bash -l -c "cat work/resource_list.txt" docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" # Uncomment this step to enable interactive debugging @@ -112,6 +116,7 @@ jobs: file: ./docker/frontend/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - name: Push master @@ -121,6 +126,7 @@ jobs: file: ./docker/master/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - name: Push node @@ -130,4 +136,5 @@ jobs: file: ./docker/node/Dockerfile push: true tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From fb06be8c13b1c9cdafe20102de966012a78fe0fe Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 11:16:27 -0600 Subject: [PATCH 23/39] debug --- .github/workflows/docker-slurm.yml | 78 +++++++++++++++--------------- docker/docker-compose.yml | 24 +++++++++ 2 files changed, 63 insertions(+), 39 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 6d93948b..3602dfbe 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -24,52 +24,52 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} -# - -# name: Pull caches -# run: | -# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true -# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true -# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true - - - name: Build frontend - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/frontend/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - - - name: Build master - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/master/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - - name: Build node - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/node/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + name: Pull caches + run: | + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true + docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true +# - +# name: Build frontend +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/frontend/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# - +# name: Build master +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/master/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max +# - +# name: Build node +# uses: docker/build-push-action@v4 +# with: +# context: ./docker +# file: ./docker/node/Dockerfile +# push: false +# load: true +# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest +# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker-compose -f docker-compose.yml up -d --no-build + docker compose -f docker-compose.yml up -d - name: Create ssh keys in cluster run: | diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 30cf962e..56a37645 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,6 +2,14 @@ services: slurmfrontend: build: context: ./ + cache_from: + - type=registry + - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache + - mode=max +# cache_to: +# - type=registry +# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache +# - mode=max dockerfile: ./frontend/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest container_name: frontend @@ -16,6 +24,10 @@ services: slurmmaster: build: context: ./ + cache_from: + - type=registry + - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache + - mode=max dockerfile: ./master/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest container_name: master @@ -34,6 +46,10 @@ services: slurmnode1: build: context: ./ + cache_from: + - type=registry + - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node1 @@ -51,6 +67,10 @@ services: slurmnode2: build: context: ./ + cache_from: + - type=registry + - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node2 @@ -68,6 +88,10 @@ services: slurmnode3: build: context: ./ + cache_from: + - type=registry + - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache + - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node3 From edb5c2c48dd28e8a2bde496a3f7dbbb68a748f39 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 11:19:56 -0600 Subject: [PATCH 24/39] debug --- docker/docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 56a37645..d43c7f44 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -24,10 +24,10 @@ services: slurmmaster: build: context: ./ - cache_from: - - type=registry - - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache - - mode=max +# cache_from: +# - type=registry +# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache +# - mode=max dockerfile: ./master/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest container_name: master From e9174cbf2e574fc8e77ec08a968bf639194b806f Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 11:22:50 -0600 Subject: [PATCH 25/39] debug --- docker/docker-compose.yml | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d43c7f44..485b4375 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,9 +3,7 @@ services: build: context: ./ cache_from: - - type=registry - - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache - - mode=max + - type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max # cache_to: # - type=registry # - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache @@ -46,10 +44,10 @@ services: slurmnode1: build: context: ./ - cache_from: - - type=registry - - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache - - mode=max +# cache_from: +# - type=registry +# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache +# - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node1 @@ -67,10 +65,10 @@ services: slurmnode2: build: context: ./ - cache_from: - - type=registry - - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache - - mode=max +# cache_from: +# - type=registry +# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache +# - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node2 @@ -88,10 +86,10 @@ services: slurmnode3: build: context: ./ - cache_from: - - type=registry - - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache - - mode=max +# cache_from: +# - type=registry +# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache +# - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node3 From f6a0ab46e67c7e599f3969119a8d79d5355029a2 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 11:29:55 -0600 Subject: [PATCH 26/39] debug --- .github/workflows/docker-slurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 3602dfbe..5c15e7b3 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -90,7 +90,7 @@ jobs: - name: Parsl/Flux hello MPI test run: | - docker exec frontend bash -l -c "cd; cd work ; ./parsl_mpi_flux_hello.py" + docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py" docker exec frontend bash -l -c "cat work/resource_list.txt" docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" # Uncomment this step to enable interactive debugging From 2a674d9f244cfc156ea8dceb720c3302c8adfa75 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 11:39:20 -0600 Subject: [PATCH 27/39] debug --- docker/docker-compose.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 485b4375..93e58141 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -4,10 +4,6 @@ services: context: ./ cache_from: - type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max -# cache_to: -# - type=registry -# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache -# - mode=max dockerfile: ./frontend/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest container_name: frontend @@ -22,10 +18,6 @@ services: slurmmaster: build: context: ./ -# cache_from: -# - type=registry -# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache -# - mode=max dockerfile: ./master/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest container_name: master @@ -44,10 +36,6 @@ services: slurmnode1: build: context: ./ -# cache_from: -# - type=registry -# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache -# - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node1 @@ -65,10 +53,6 @@ services: slurmnode2: build: context: ./ -# cache_from: -# - type=registry -# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache -# - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node2 @@ -86,10 +70,6 @@ services: slurmnode3: build: context: ./ -# cache_from: -# - type=registry -# - ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache -# - mode=max dockerfile: ./node/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest container_name: node3 From 73c9b12ebce9b75114ee6d58c38d71e35fffd466 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 11:49:14 -0600 Subject: [PATCH 28/39] debug --- docker/frontend/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 1a103632..efe3c4bd 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -52,7 +52,11 @@ RUN . /opt/spack/share/spack/setup-env.sh \ # Install the Spack workflow environment RUN . /opt/spack/share/spack/setup-env.sh \ && cd /tmp/install \ - && ./install_parsl.sh \ + && ./install_parsl.sh + +# Install the Spack workflow environment again +RUN . /opt/spack/share/spack/setup-env.sh \ + && cd /tmp/install \ && ./install_flux.sh # Set up user shell init From 276d68069119ea4fb4b3c25cffd23824117dca63 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 12:26:18 -0600 Subject: [PATCH 29/39] debug --- .github/workflows/docker-slurm.yml | 78 +++++++++++++++--------------- docker/frontend/Dockerfile | 3 ++ 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 5c15e7b3..ebd049db 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -24,52 +24,52 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull caches - run: | - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true - docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true -# - -# name: Build frontend -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/frontend/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max -# - -# name: Build master -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/master/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max # - -# name: Build node -# uses: docker/build-push-action@v4 -# with: -# context: ./docker -# file: ./docker/node/Dockerfile -# push: false -# load: true -# tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest -# cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max -# cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max +# name: Pull caches +# run: | +# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true +# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true +# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true + - + name: Build frontend + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/frontend/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max + - + name: Build master + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/master/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max + - + name: Build node + uses: docker/build-push-action@v4 + with: + context: ./docker + file: ./docker/node/Dockerfile + push: false + load: true + tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest + cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max + cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker compose -f docker-compose.yml up -d + docker compose -f docker-compose.yml up -d --no-build - name: Create ssh keys in cluster run: | diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index efe3c4bd..669cbf26 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -44,6 +44,9 @@ SHELL ["/bin/bash", "-c"] # && cd /tmp/install \ # && ./install_base.sh +RUN echo "foobar foobar foobar" > /tmp/foobarfoobar \ + && echo "hello cruel world" >> /tmp/foobarfoobar + # Install the Spack workflow environment RUN . /opt/spack/share/spack/setup-env.sh \ && cd /tmp/install \ From b387245ed2910645a1bdd61fca7c9d136b3325be Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 12:26:56 -0600 Subject: [PATCH 30/39] debug --- docker/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 93e58141..fa1ebf9f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,8 +2,8 @@ services: slurmfrontend: build: context: ./ - cache_from: - - type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max +# cache_from: +# - type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max dockerfile: ./frontend/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest container_name: frontend From bf21dc0801f8770a41b2149b8df1098ffe64e813 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 13:43:41 -0600 Subject: [PATCH 31/39] Revert back to not using ghcr for caches. --- .github/workflows/docker-slurm.yml | 72 +----------------------------- 1 file changed, 1 insertion(+), 71 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index ebd049db..d5e01f9b 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -24,52 +24,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} -# - -# name: Pull caches -# run: | -# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache || true -# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache || true -# docker pull ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache || true - - - name: Build frontend - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/frontend/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - - - name: Build master - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/master/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - - - name: Build node - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/node/Dockerfile - push: false - load: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - name: Start Slurm cluster containers run: | cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker compose -f docker-compose.yml up -d --no-build + docker compose -f docker-compose.yml up -d - name: Create ssh keys in cluster run: | @@ -93,7 +54,6 @@ jobs: docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py" docker exec frontend bash -l -c "cat work/resource_list.txt" docker exec frontend bash -l -c "cat work/mpi_apps.hello.out" -# Uncomment this step to enable interactive debugging - name: Debug session if: ${{ failure() }} @@ -108,33 +68,3 @@ jobs: export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 docker-compose -f docker-compose.yml down - - - name: Push frontend - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/frontend/Dockerfile - push: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max - - - name: Push master - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/master/Dockerfile - push: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max - - - name: Push node - uses: docker/build-push-action@v4 - with: - context: ./docker - file: ./docker/node/Dockerfile - push: true - tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest - cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max - cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max From a993a74289abe7b317d124a5c7dbe599c0ed36ec Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 13:45:27 -0600 Subject: [PATCH 32/39] Remove cache from compose yaml --- docker/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fa1ebf9f..30cf962e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,8 +2,6 @@ services: slurmfrontend: build: context: ./ -# cache_from: -# - type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max dockerfile: ./frontend/Dockerfile image: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest container_name: frontend From ad10ece22f1d40c0d7a16da0e533acf4187539cc Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 14:09:13 -0600 Subject: [PATCH 33/39] Cleanup --- docker/frontend/Dockerfile | 14 ++------ docker/install/install_base.sh | 62 --------------------------------- docker/install/install_flux.sh | 12 ------- docker/install/install_parsl.sh | 4 ++- 4 files changed, 6 insertions(+), 86 deletions(-) delete mode 100755 docker/install/install_base.sh diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 669cbf26..72f0f007 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -39,25 +39,17 @@ COPY ./install /tmp/install USER admin SHELL ["/bin/bash", "-c"] -# Install the Spack workflow environment -#RUN . /opt/spack/share/spack/setup-env.sh \ -# && cd /tmp/install \ -# && ./install_base.sh - -RUN echo "foobar foobar foobar" > /tmp/foobarfoobar \ - && echo "hello cruel world" >> /tmp/foobarfoobar - -# Install the Spack workflow environment +# Install the Spack flux environment RUN . /opt/spack/share/spack/setup-env.sh \ && cd /tmp/install \ && ./install_flux.sh -# Install the Spack workflow environment +# Install parsl into the flux environment RUN . /opt/spack/share/spack/setup-env.sh \ && cd /tmp/install \ && ./install_parsl.sh -# Install the Spack workflow environment again +# Re-install the Spack flux environment to rebuild view after pip install of parsl RUN . /opt/spack/share/spack/setup-env.sh \ && cd /tmp/install \ && ./install_flux.sh diff --git a/docker/install/install_base.sh b/docker/install/install_base.sh deleted file mode 100755 index 93de8ff0..00000000 --- a/docker/install/install_base.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/env bash - -# These can be customized to suit individual needs -DEFAULT_GCC_VERSION=$(gcc --version | head -1 | sed -e 's/([^()]*)//g' | awk '{print $2}') # Version of system default gcc -DEFAULT_COMPILER="gcc@${DEFAULT_GCC_VERSION}" # Default system compiler used to build newer gcc - -SPACK_ENV_NAME="base" # Name of spack environment to create -SPACK_ENV_COMPILER="gcc@9.4.0" # Compiler to use to build the spack environment -TARGET_ARCH_OPT="target=x86_64" # Compiler architecture build target - -################################################################################ -# help # -################################################################################ -help() -{ - # Display help - echo "Installs a base Spack environment to use for building exaworks packages." - echo "This includes: gcc, python, pytest, pylint, flake8, mypy, and black." - echo - echo "Usage: install_base.sh" - echo -} - -# Get the location of Spack so we can update permissions -if [[ $(which spack 2>/dev/null) ]]; then - SPACK_DIR=$(dirname $(dirname $(which spack))) -else - echo "Cannot find Spack." - echo - echo "Please install Spack and/or source Spack's environment setup script: .../spack/share/setup-env.sh" - echo - exit 1 -fi - -set -eu - -# Configure spack -spack config add concretizer:unify:true -spack config add concretizer:reuse:true -spack config add config:db_lock_timeout:300 -spack config add config:install_tree:padded_length:128 - -# Create the base environment -spack env create ${SPACK_ENV_NAME} || true -spack env activate ${SPACK_ENV_NAME} -spack compiler find - -# Install python tools and miniconda3 -spack add python@3.9.15%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-pytest%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-pylint%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-flake8%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-mypy%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-black%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-ply%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -spack add miniconda3%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} -spack add flux-core@0.53.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} -spack add flux-sched@0.28.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} -spack concretize -spack install --no-checksum - -exit 0 diff --git a/docker/install/install_flux.sh b/docker/install/install_flux.sh index cee4115b..78e2ae90 100755 --- a/docker/install/install_flux.sh +++ b/docker/install/install_flux.sh @@ -43,18 +43,6 @@ spack config add config:install_tree:padded_length:128 spack env create ${SPACK_ENV_NAME} || true spack env activate ${SPACK_ENV_NAME} -# Install python tools -#spack add python@3.9.15%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-pip%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-pytest%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-pylint%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-flake8%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-mypy%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-black%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add py-ply%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT} -#spack add miniconda3%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} -#spack install - # Install flux components spack add flux-core@0.53.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} spack add flux-sched@0.28.0%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} diff --git a/docker/install/install_parsl.sh b/docker/install/install_parsl.sh index 12df9b81..699e977b 100755 --- a/docker/install/install_parsl.sh +++ b/docker/install/install_parsl.sh @@ -43,10 +43,12 @@ spack config add config:install_tree:padded_length:128 spack env create ${SPACK_ENV_NAME} || true spack env activate ${SPACK_ENV_NAME} +# Re-install pip to update the view and make it reappear in the flux environment spack env activate flux spack add py-pip%${SPACK_ENV_COMPILER} ^python@3.9.15 ${TARGET_ARCH_OPT} spack install + +# Install parsl with pip python -m pip install parsl -#pip install 'dill @ git+https://github.com/uqfoundation/dill' exit 0 From fc0b2570de058769c59a70f3f8022fa4953ee4aa Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sat, 19 Aug 2023 16:37:20 -0600 Subject: [PATCH 34/39] add push --- .github/workflows/docker-slurm.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index d5e01f9b..7d2095d4 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -30,7 +30,7 @@ jobs: cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker compose -f docker-compose.yml up -d + docker compose -f docker-compose.yml up --build -d - name: Create ssh keys in cluster run: | @@ -68,3 +68,9 @@ jobs: export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 docker-compose -f docker-compose.yml down + - + name: Push containers + run: | + docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest + docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest + docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest \ No newline at end of file From 65b1306c4c9ab40bb1877f1a2b573870402743a5 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sun, 20 Aug 2023 09:46:24 -0600 Subject: [PATCH 35/39] debug --- .github/workflows/docker-slurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 7d2095d4..d06e5aca 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -30,7 +30,7 @@ jobs: cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker compose -f docker-compose.yml up --build -d + docker-compose -f docker-compose.yml up --build -d - name: Create ssh keys in cluster run: | From 006a836308c16998c3c10c7641eecce5be26c115 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sun, 20 Aug 2023 12:00:36 -0600 Subject: [PATCH 36/39] debug --- .github/workflows/docker-slurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index d06e5aca..7d2095d4 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -30,7 +30,7 @@ jobs: cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker-compose -f docker-compose.yml up --build -d + docker compose -f docker-compose.yml up --build -d - name: Create ssh keys in cluster run: | From 47cab6e8b5e9653c7f6a636d33b66f7c18780f86 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sun, 20 Aug 2023 14:24:57 -0600 Subject: [PATCH 37/39] debug --- .github/workflows/docker-slurm.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index 7d2095d4..c7c230bd 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -24,6 +24,10 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - + name: Update to avoid docker compose V2 bug + run: | + sudo apt-get update && sudo apt-get upgrade - name: Start Slurm cluster containers run: | From 489078757125efb927f9bc223330bd4bfd14e5aa Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sun, 20 Aug 2023 15:45:28 -0600 Subject: [PATCH 38/39] Revert to old compose --- .github/workflows/docker-slurm.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index c7c230bd..d06e5aca 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -24,17 +24,13 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Update to avoid docker compose V2 bug - run: | - sudo apt-get update && sudo apt-get upgrade - name: Start Slurm cluster containers run: | cd docker export DOCKER_CLIENT_TIMEOUT=600 export COMPOSE_HTTP_TIMEOUT=600 - docker compose -f docker-compose.yml up --build -d + docker-compose -f docker-compose.yml up --build -d - name: Create ssh keys in cluster run: | From ab8d9940472aec44bd84fe735c384c6bf4735cb8 Mon Sep 17 00:00:00 2001 From: Christopher Harrop Date: Sun, 20 Aug 2023 19:45:15 -0600 Subject: [PATCH 39/39] Add carriage return at the end --- .github/workflows/docker-slurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-slurm.yml b/.github/workflows/docker-slurm.yml index d06e5aca..9cca6439 100644 --- a/.github/workflows/docker-slurm.yml +++ b/.github/workflows/docker-slurm.yml @@ -73,4 +73,4 @@ jobs: run: | docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest - docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest \ No newline at end of file + docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest