Remove broken caching and use workaround for broken pip install

.github/workflows/docker-slurm.yml

@@ -24,46 +24,13 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Build frontend
-        uses: docker/build-push-action@v4
-        with:
-          context: ./docker
-          file: ./docker/frontend/Dockerfile
-          push: false
-          load: true
-          tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max
-      -
-        name: Build master
-        uses: docker/build-push-action@v4
-        with:
-          context: ./docker
-          file: ./docker/master/Dockerfile
-          push: false
-          load: true
-          tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max
-      -
-        name: Build node
-        uses: docker/build-push-action@v4
-        with:
-          context: ./docker
-          file: ./docker/node/Dockerfile
-          push: false
-          load: true
-          tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max
       -
         name: Start Slurm cluster containers
         run: |
           cd docker
           export DOCKER_CLIENT_TIMEOUT=600
           export COMPOSE_HTTP_TIMEOUT=600
-          docker-compose -f docker-compose.yml up -d --no-build
+          docker-compose -f docker-compose.yml up --build -d 
       -
         name: Create ssh keys in cluster
         run: |
@@ -73,6 +40,10 @@ jobs:
         name: Copy test scripts to work directory
         run: |
           docker exec frontend bash -l -c "mkdir -p work; cd work ; cp ../test/*.py . ; cp ../test/*.f90 ."
+      -
+        name: Fix the flux install # This is a workaround to deal with bugs caused by spack/pip conflicts
+        run: |
+          docker exec frontend bash -l -c "spack env activate flux; cd /tmp/install; ./install_flux.sh"
       -
         name: Parsl hello test
         run: |
@@ -83,14 +54,13 @@ jobs:
           docker exec frontend bash -l -c "spack env activate flux; cd work ; ./parsl_mpi_flux_hello.py"
           docker exec frontend bash -l -c "cat work/resource_list.txt"
           docker exec frontend bash -l -c "cat work/mpi_apps.hello.out"
-# Uncomment this step to enable interactive debugging
-#      -
-#        name: Debug session
-#        if: ${{ failure() }}
-#        uses: mxschmitt/action-tmate@v3
-#        timeout-minutes: 60
-#        with:
-#          limit-access-to-actor: true
+      -
+        name: Debug session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 60
+        with:
+          limit-access-to-actor: true
       -
         name: Shut down Slurm cluster containers
         run: |
@@ -99,32 +69,8 @@ jobs:
           export COMPOSE_HTTP_TIMEOUT=600
           docker-compose -f docker-compose.yml down
       -
-        name: Push frontend
-        uses: docker/build-push-action@v4
-        with:
-          context: ./docker
-          file: ./docker/frontend/Dockerfile
-          push: true
-          tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:cache,mode=max
-      -
-        name: Push master
-        uses: docker/build-push-action@v4
-        with:
-          context: ./docker
-          file: ./docker/master/Dockerfile
-          push: true
-          tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:cache,mode=max
-      -
-        name: Push node
-        uses: docker/build-push-action@v4
-        with:
-          context: ./docker
-          file: ./docker/node/Dockerfile
-          push: true
-          tags: ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:cache,mode=max
+        name: Push containers
+        run: |
+          docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/frontend:latest
+          docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/master:latest
+          docker push ghcr.io/noaa-gsl/exascaleworkflowsandbox/node:latest

README.md

@@ -1,3 +1,5 @@
+[![ExascaleSandboxTests](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yml/badge.svg)](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yml)
+
 ```
 This repository is a scientific product and is not official communication
 of the National Oceanic and Atmospheric Administration, or the United States

docker/frontend/Dockerfile

@@ -39,13 +39,21 @@ COPY ./install /tmp/install
 USER admin
 SHELL ["/bin/bash", "-c"]
 
-# Install the Spack workflow environment
+# Install the Spack flux environment
+RUN . /opt/spack/share/spack/setup-env.sh \
+ && cd /tmp/install \
+ && ./install_flux.sh
+
+# Install parsl into the flux environment
 RUN . /opt/spack/share/spack/setup-env.sh \
  && cd /tmp/install \
- && ./install_base.sh \
- && ./install_flux.sh \
  && ./install_parsl.sh
 
+# Re-install the Spack flux environment to rebuild view after pip install of parsl
+RUN . /opt/spack/share/spack/setup-env.sh \
+ && cd /tmp/install \
+ && ./install_flux.sh
+
 # Set up user shell init
 RUN echo ". /opt/spack/share/spack/setup-env.sh" >> /home/admin/.bash_profile \
  && echo "source /opt/intel/oneapi/setvars.sh" >> /home/admin/.bash_profile \

docker/install/install_flux.sh

@@ -43,20 +43,9 @@ spack config add config:install_tree:padded_length:128
 spack env create ${SPACK_ENV_NAME} || true
 spack env activate ${SPACK_ENV_NAME}
 
-# Install python tools
-spack add [email protected]%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack add py-pytest%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack add py-pylint%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack add py-flake8%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack add py-mypy%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack add py-black%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack add py-ply%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack add miniconda3%${SPACK_ENV_COMPILER} ${TARGET_ARCH_OPT}
-spack install
-
 # Install flux components
-spack add flux-core@0.49.0%${SPACK_ENV_COMPILER} ^[email protected] ${TARGET_ARCH_OPT}
-spack add flux-sched@0.27.0%${SPACK_ENV_COMPILER} ^[email protected] ${TARGET_ARCH_OPT}
+spack add flux-core@0.53.0%${SPACK_ENV_COMPILER} ^[email protected].15 ${TARGET_ARCH_OPT}
+spack add flux-sched@0.28.0%${SPACK_ENV_COMPILER} ^[email protected].15 ${TARGET_ARCH_OPT}
 spack concretize -f
 spack install --no-checksum
 

docker/install/install_parsl.sh

@@ -1,7 +1,54 @@
 #!/bin/env bash
 
+# These can be customized to suit individual needs
+DEFAULT_GCC_VERSION=$(/usr/bin/gcc --version | head -1 | sed -e 's/([^()]*)//g' | awk '{print $2}')  # Version of system default gcc
+DEFAULT_COMPILER="gcc@${DEFAULT_GCC_VERSION}"  # Default system compiler used to build newer gcc
+
+SPACK_ENV_NAME="flux"            # Name of spack environment to create
+SPACK_ENV_COMPILER="[email protected]"  # Compiler to use to build the spack environment
+TARGET_ARCH_OPT="target=x86_64"  # Compiler architecture build target
+
+################################################################################
+# help                                                                         #
+################################################################################
+help()
+{
+   # Display help
+   echo "Installs parsl into flux Spack environment"
+   echo
+   echo "Usage: install_parsl.sh"
+   echo
+}
+
+# Get the location of Spack so we can update permissions
+if [[ $(which spack 2>/dev/null) ]]; then
+  SPACK_DIR=$(dirname $(dirname $(which spack)))
+else
+  echo "Cannot find Spack."
+  echo
+  echo "Please install Spack and/or source Spack's environment setup script: .../spack/share/setup-env.sh"
+  echo
+  exit 1
+fi
+
+set -eu
+
+# Configure spack
+spack config add concretizer:unify:true
+spack config add concretizer:reuse:true
+spack config add config:db_lock_timeout:300
+spack config add config:install_tree:padded_length:128
+
+# Configure spack environment
+spack env create ${SPACK_ENV_NAME} || true
+spack env activate ${SPACK_ENV_NAME}
+
+# Re-install pip to update the view and make it reappear in the flux environment
 spack env activate flux
-pip install parsl
-pip install 'dill @ git+https://github.com/uqfoundation/dill'
+spack add py-pip%${SPACK_ENV_COMPILER} ^[email protected] ${TARGET_ARCH_OPT}
+spack install
+
+# Install parsl with pip
+python -m pip install parsl
 
 exit 0

docker/test/parsl_mpi_flux_hello.py

@@ -26,7 +26,8 @@
                 walltime='00:10:00',
                 launcher=SimpleLauncher(),
                 worker_init='''
-# FluxExecutor ignores worker_init
+. /opt/spack/share/spack/setup-env.sh
+spack env activate flux
 ''',
             ),
         )
@@ -45,6 +46,8 @@
 @bash_app
 def resource_list():
     return '''
+    . /opt/spack/share/spack/setup-env.sh
+    spack env activate flux
     flux resource list > resource_list.txt
     '''