CI updates (#222)

* update developer.yml * update Intel.yml * update Linux_options.yml * update Linux_versions.yml * update MacOS.yml * update Spack.yml * fix Linux_options.yml * fix Linux_versions.yml * fix MacOS.yml * fix Linux_options.yml * fix Spack.yml * change +shared+pic requirement to conflict, not variant/when * fix Spack.yml: no more extra deps
NOAA-EMC · Jun 4, 2024 · f23ad5b · f23ad5b
1 parent 74c934f
commit f23ad5b
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 261 deletions.
diff --git a/.github/workflows/Intel.yml b/.github/workflows/Intel.yml
@@ -44,5 +44,4 @@ jobs:
         cmake --build w3emc/build --parallel 2 --verbose
 
     - name: "Test w3emc"
-      run: |
-        ctest --test-dir w3emc/build --output-on-failure --rerun-failed --verbose
+      run: ctest --test-dir w3emc/build --output-on-failure --rerun-failed --verbose
diff --git a/.github/workflows/Linux_options.yml b/.github/workflows/Linux_options.yml
@@ -24,80 +24,26 @@ jobs:
 
     steps:
 
-    - name: checkout-bacio
-      uses: actions/checkout@v3
+    - name: "Build dependencies"
+      uses: NOAA-EMC/ci-build-nceplibs@develop
       with:
-        repository: NOAA-EMC/NCEPLIBS-bacio
-        path: bacio
-        ref: v2.6.0
-
-    - name: cache-bacio
-      id: cache-bacio
-      uses: actions/cache@v3
-      with:
-        path: ~/bacio
-        key: bacio-Linux_options_${{ runner.os }}-v2.5.0-${{ matrix.options }}
-
-    - name: build-bacio
-      if: steps.cache-bacio.outputs.cache-hit != 'true'
-      run: |
-        set -x
-        cd bacio
-        mkdir build && cd build
-        if [ "${{ matrix.options }}" = "-DBUILD_SHARED_LIBS=ON" ]
-        then
-           cmake -DCMAKE_INSTALL_PREFIX=~/bacio -DBUILD_SHARED_LIBS=ON ..
-        else
-           cmake -DCMAKE_INSTALL_PREFIX=~/bacio ..
-        fi
-        make -j2 VERBOSE=1
-        make install
-
-    - name: checkout-bufr
-      uses: actions/checkout@v3
-      with:
-        repository: NOAA-EMC/NCEPLIBS-bufr
-        path: bufr
-        ref: bufr_v12.0.0
-
-    - name: cache-bufr
-      id: cache-bufr
-      uses: actions/cache@v3
-      with:
-        path: ~/bufr
-        key: bufr-Linux_options_${{ runner.os }}-v12.0.0-${{ matrix.options }}
-
-    - name: build-bufr
-      if: steps.cache-bufr.outputs.cache-hit != 'true'
-      run: |
-        set -x
-        cd bufr
-        mkdir build && cd build
-        if [ "${{ matrix.options }}" = "-DBUILD_SHARED_LIBS=ON" ]
-        then
-           cmake -DCMAKE_INSTALL_PREFIX=~/bufr -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON ..
-        else
-           cmake -DCMAKE_INSTALL_PREFIX=~/bufr -DBUILD_TESTING=OFF ..
-        fi
-        make -j2 VERBOSE=1
-        make install
+        bacio-version: v2.6.0
+        bacio-cmake-args: -DBUILD_TESTING=OFF ${{ matrix.options == '-DBUILD_SHARED_LIBS=ON' && matrix.options || '' }}
+        bufr-version: bufr_v12.0.0
+        bufr-cmake-args: -DBUILD_TESTING=OFF ${{ matrix.options == '-DBUILD_SHARED_LIBS=ON' && matrix.options || '' }}
 
     - name: checkout-w3emc
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with: 
         path: w3emc
 
     - name: build-w3emc
       run: |
         set -x
-        cd w3emc
-        mkdir build 
-        cd build
-        cmake -DCMAKE_PREFIX_PATH="~/bacio;~/bufr" -DCMAKE_INSTALL_PREFIX="~/w3emc/install" ${{ matrix.options }} ..
-        make -j2 VERBOSE=1
+        cmake -S w3emc -B w3emc/build -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/nceplibs/NCEPLIBS-bacio;$GITHUB_WORKSPACE/nceplibs/NCEPLIBS-bufr" -DCMAKE_INSTALL_PREFIX="~/w3emc/install" ${{ matrix.options }}
+        cmake --build w3emc/build --parallel 2 --verbose
 
     - name: test-w3emc
       run: |
-        cd w3emc/build
-        ctest --output-on-failure --rerun-failed --verbose
-        make install
+        ctest --test-dir w3emc/build --output-on-failure --rerun-failed --verbose
+        cmake --install w3emc/build
diff --git a/.github/workflows/Linux_versions.yml b/.github/workflows/Linux_versions.yml
@@ -24,67 +24,22 @@ jobs:
         bacio-version: [2.4.1, 2.5.0, 2.6.0]
 
     steps:
-
-    - name: checkout-bacio
-      uses: actions/checkout@v3
-      with:
-        repository: NOAA-EMC/NCEPLIBS-bacio
-        path: bacio
-        ref: v${{ matrix.bacio-version }}
-
-    - name: cache-bacio
-      id: cache-bacio
-      uses: actions/cache@v3
-      with:
-        path: ~/bacio
-        key: bacio-${{ runner.os }}-v${{ matrix.bacio-version }}
-
-    - name: build-bacio
-      if: steps.cache-bacio.outputs.cache-hit != 'true'
-      run: |
-        cd bacio
-        mkdir build && cd build
-        cmake .. -DCMAKE_INSTALL_PREFIX=~/bacio
-        make -j2
-        make install
 
-    - name: checkout-bufr
-      uses: actions/checkout@v3
+    - name: "Build dependencies"
+      uses: NOAA-EMC/ci-build-nceplibs@develop
       with:
-        repository: NOAA-EMC/NCEPLIBS-bufr
-        path: bufr
-        ref: v12.0.1
-
-    - name: cache-bufr
-      id: cache-bufr
-      uses: actions/cache@v3
-      with:
-        path: ~/bufr
-        key: bufr-Linux_options_${{ runner.os }}-v12.0.1
+        bacio-version: v${{ matrix.bacio-version }}
+        bufr-version: v12.0.1
 
-    - name: build-bufr
-      if: steps.cache-bufr.outputs.cache-hit != 'true'
-      run: |
-        cd bufr
-        mkdir build && cd build
-        cmake -DCMAKE_INSTALL_PREFIX=~/bufr ..
-        make -j2
-        make install
-        
     - name: checkout-w3emc
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with: 
         path: w3emc
 
     - name: build-w3emc
       run: |
-        cd w3emc
-        mkdir build 
-        cd build
-        cmake -DCMAKE_PREFIX_PATH="~/bacio;~/bufr" ..
-        make -j2
+        cmake -S w3emc -B w3emc/build -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/nceplibs/NCEPLIBS-bacio;$GITHUB_WORKSPACE/nceplibs/NCEPLIBS-bufr"
+        cmake --build w3emc/build --parallel 2
 
     - name: test-w3emc
-      run: |
-        cd w3emc/build
-        ctest --output-on-failure --rerun-failed --verbose
+      run: ctest --test-dir w3emc/build --output-on-failure --rerun-failed --verbose
diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml
@@ -23,67 +23,23 @@ jobs:
         bacio-version: [2.4.1, 2.5.0, 2.6.0]
 
     steps:
-    - name: checkout-bacio
-      uses: actions/checkout@v3
-      with:
-        repository: NOAA-EMC/NCEPLIBS-bacio
-        path: bacio
-        ref: v${{ matrix.bacio-version }}
-
-    - name: cache-bacio
-      id: cache-bacio
-      uses: actions/cache@v3
-      with:
-        path: ~/bacio
-        key: bacio-${{ runner.os }}-v${{ matrix.bacio-version }}-${{ matrix.compiler }}
-
-    - name: build-bacio
-      if: steps.cache-bacio.outputs.cache-hit != 'true'
-      run: |
-        cd bacio
-        mkdir build && cd build
-        cmake -DCMAKE_INSTALL_PREFIX=~/bacio ..
-        make -j2
-        make install
-
-    - name: checkout-bufr
-      uses: actions/checkout@v3
-      with:
-        repository: NOAA-EMC/NCEPLIBS-bufr
-        path: bufr
-        ref: bufr_v11.7.1
 
-    - name: cache-bufr
-      id: cache-bufr
-      uses: actions/cache@v3
+    - name: "Build dependencies"
+      uses: NOAA-EMC/ci-build-nceplibs@develop
       with:
-        path: ~/bufr
-        key: bufr-Linux_options_${{ runner.os }}-v11.7.1
-
-    - name: build-bufr
-      if: steps.cache-bufr.outputs.cache-hit != 'true'
-      run: |
-        cd bufr
-        mkdir build && cd build
-        cmake -DCMAKE_INSTALL_PREFIX=~/bufr ..
-        make -j2
-        make install
+        bacio-version: v${{ matrix.bacio-version }}
+        bufr-version: bufr_v11.7.1
 
     - name: checkout-w3emc
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with: 
         path: w3emc
 
     - name: build-w3emc
       run: |
-        cd w3emc
-        mkdir build 
-        cd build
-        cmake -DCMAKE_PREFIX_PATH="~/bacio;~/bufr" ..
-        make -j2
+        cmake -S w3emc -B w3emc/build -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/nceplibs/NCEPLIBS-bacio;$GITHUB_WORKSPACE/nceplibs/NCEPLIBS-bufr"
+        cmake --build w3emc/build --parallel 2 --verbose
 
     - name: test-w3emc
-      run: |
-        cd w3emc/build
-        ctest --output-on-failure --rerun-failed --verbose
+      run: ctest --test-dir w3emc/build --output-on-failure --rerun-failed --verbose
 
diff --git a/.github/workflows/Spack.yml b/.github/workflows/Spack.yml
@@ -16,56 +16,35 @@ on:
     - develop
 
 jobs:
-  # This job builds with Spack using every combination of variants and runs the CTest suite each time
+  # This job builds with Spack, exercising all variants, and runs the CTest suite each time
   Spack:
     strategy:
       matrix:
         os: ["ubuntu-latest"]
-        pic_shared: ["+pic +shared", "+pic ~shared", "~pic ~shared"]
-        precision: ["precision=d", "precision=4", "precision=8"]
-        bufr: ["+bufr", "~bufr"]
+        variants: ["+pic +shared +bufr precision=d", "~pic ~shared ~bufr precision=4", "+pic ~shared ~bufr precision=8"]
 
     runs-on: ${{ matrix.os }}
 
     steps:
-
-    - name: checkout-w3emc
-      uses: actions/checkout@v4
-      with: 
-        path: w3emc
-
-    - name: spack-build-and-test
-      run: |
-        git clone -c feature.manyFiles=true https://github.com/jcsda/spack
-        . spack/share/spack/setup-env.sh
-        spack env create w3emc-env
-        spack env activate w3emc-env
-        cp $GITHUB_WORKSPACE/w3emc/spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/w3emc/package.py
-        mv $GITHUB_WORKSPACE/w3emc $SPACK_ENV/w3emc
-        spack develop --no-clone w3emc@develop
-        spack add w3emc@develop%gcc@11 ${{ matrix.pic_shared }} ${{ matrix.precision }} ${{ matrix.bufr }}
-        spack external find cmake gmake
-        spack concretize
-        # Run installation and run CTest suite
-        spack install --verbose --fail-fast --test root
-        # Run 'spack load' to check for obvious errors in setup_run_environment
-        spack load w3emc
 
+    - name: "Build Spack package"
+      uses: NOAA-EMC/ci-test-spack-package@develop
+      with:
+        package-name: w3emc
+        package-variants: ${{ matrix.variants }}
+        custom-recipe: spack/package.py
+        spack-compiler: gcc
+        repo-cache-key-suffix: ${{ matrix.os }}-${{ matrix.variants }}-1
+
   # This job validates the Spack recipe by making sure each cmake build option is represented
   recipe-check:
     runs-on: ubuntu-latest
 
     steps:
 
-    - name: checkout-w3emc
-      uses: actions/checkout@v4
-      with: 
-        path: w3emc
-
     - name: recipe-check
-      run: |
-        echo "If this jobs fails, look at the most recently output CMake option below and make sure that option appears in spack/package.py"
-        for opt in $(grep -ioP '^option\(\K(?!(ENABLE_DOCS))[^ ]+' $GITHUB_WORKSPACE/w3emc/CMakeLists.txt) ; do
-          echo "Checking for presence of '$opt' CMake option in package.py"
-          grep -cP "define.+\b${opt}\b" $GITHUB_WORKSPACE/w3emc/spack/package.py
-        done
+      uses: NOAA-EMC/ci-check-spack-recipe@develop
+      with:
+        recipe-file: package/spack/package.py
+        cmakelists-txt: package/CMakeLists.txt
+        ignore-list: ENABLE_DOCS