Use GlobusComputeExecutor (#136)

* Set globus compute endpoint environment in worker_init

* Update code and tests to use GlobusComputeExecutor

* Fix linting

* MPAS app lint check is done in its own workflow, remove from main workflow

* Update workflows to install "test" variant in CI workflows to make sure pytest is available

.github/workflows/docker-slurm.yaml

@@ -73,7 +73,7 @@ jobs:
         name: Install chiltepin package
         run: |
           docker exec frontend bash -l -c "cd work ; module use /opt/spack-stack/envs/unified-env/install/modulefiles/Core ; module load stack-gcc; module load stack-openmpi; module load stack-python ; python -m venv .chiltepin"
-          docker exec frontend bash -l -c "cd work ; source .chiltepin/bin/activate ; pip --use-deprecated=legacy-resolver install -e ."
+          docker exec frontend bash -l -c "cd work ; source .chiltepin/bin/activate ; pip --use-deprecated=legacy-resolver install -e .[test]"
       -
         name: Run test suite
         env:

.github/workflows/mpas-app.yaml

@@ -1,6 +1,8 @@
 name: MPAS App
 
 on:
+  push:
+    branches: [ develop, main ]
   pull_request:
     branches: [ develop, main ]
     types: [ labeled ]
@@ -72,7 +74,7 @@ jobs:
         name: Install chiltepin package
         run: |
           docker exec frontend bash -l -c "cd work ; module use /opt/spack-stack/envs/unified-env/install/modulefiles/Core ; module load stack-gcc; module load stack-openmpi; module load stack-python ; python -m venv .chiltepin"
-          docker exec frontend bash -l -c "cd work ; source .chiltepin/bin/activate ; pip --use-deprecated=legacy-resolver install -e ."
+          docker exec frontend bash -l -c "cd work ; source .chiltepin/bin/activate ; pip --use-deprecated=legacy-resolver install -e .[test]"
       -
         name: Run mpas workflow
         run: |

README.md

@@ -1,4 +1,5 @@
-[![ExascaleSandboxTests](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yml/badge.svg)](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yml)
+[![ExascaleSandboxTests](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yaml/badge.svg)](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/docker-slurm.yaml)
+[![MPAS App](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/mpas-app.yaml/badge.svg)](https://github.com/NOAA-GSL/ExascaleWorkflowSandbox/actions/workflows/mpas-app.yaml)
 
 ```
 This repository is a scientific product and is not official communication
@@ -34,7 +35,7 @@ Python >= 3.9 is available.
 ```
 python -m venv .chiltepin
 source .chiltepin/bin/activate
-pip --use-deprecated=legacy-resolver install -e .  # Do not forget the dot at the end
+pip --use-deprecated=legacy-resolver install -e .[test]
 ```
 
 Alternatively, a conda environment (anaconda3, miniconda3, miniforge, etc.)
@@ -44,18 +45,17 @@ of certain known (and accepted) dependency conflicts that must be ignored.
 ```
 conda create -n "chiltepin" python=3.10
 source activate chiltepin
-pip --use-deprecated=legacy-resolver install -e .  # Do not forget the dot at the end
+pip --use-deprecated=legacy-resolver install -e .[test]
 ```
 
+NOTE: The `[test]` ensures that dependencies required for running the tests are installed.
+
 NOTE: There may be some warnings about incompatible package versions similar
 to the following:
 
 ```
 ERROR: pip's legacy dependency resolver does not consider dependency conflicts when selecting packages. This behaviour is the source of the following dependency conflicts.
-globus-compute-sdk 2.22.0 requires dill==0.3.6; python_version >= "3.11", but you'll have dill 0.3.8 which is incompatible.
 globus-identity-mapping 0.3.0 requires typing-extensions<4.10,>=4.9, but you'll have typing-extensions 4.12.2 which is incompatible.
-globus-compute-endpoint 2.22.0 requires jsonschema<4.20,>=4.19.0, but you'll have jsonschema 4.22.0 which is incompatible.
-globus-compute-endpoint 2.22.0 requires parsl==2024.3.18, but you'll have parsl 2024.6.3 which is incompatible.
 ```
 
 Those dependency conflicts can be safely ignored.
@@ -113,10 +113,12 @@ container environment), and run the tests
 
 ```
 cd chiltepin
-pip --use-deprecated=legacy-resolver install -e .
+pip --use-deprecated=legacy-resolver install -e .[test]
 pytest --assert=plain --config=tests/config.yaml --platform=docker
 ```
 
+NOTE: the `[test]` ensures that dependencies required for running the tests are installed.
+
 NOTE: Depending on how many cores your machine has and how many you've allocated to Docker,
 you may need to modify the `cores per node` setting in the configuration yaml file to match
 your machine's specifications to get all tests to pass.

apps/mpas/bin/experiment.py

@@ -46,7 +46,10 @@ def main(user_config_file: Path) -> None:
     # Load Parsl resource configs
     resource_config_file = mpas_app / "config" / "resources.yaml"
     yaml_config = chiltepin.configure.parse_file(resource_config_file)
-    resources = chiltepin.configure.load(yaml_config[machine])
+    resources = chiltepin.configure.load(
+        yaml_config[machine]["resources"],
+        resources=["service", "compute", "mpi"],
+    )
     with parsl.load(resources):
 
         # Instantiate LimitedArea object

pyproject.toml

@@ -1,23 +1,42 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "chiltepin"
 version = "0.0.1"
 dependencies = [
-    "pytest",
-    "pytest-flake8",
-    "pytest-black",
-    "pytest-isort",
     "globus-compute-sdk>=2.30.1",
     "globus-compute-endpoint>=2.30.1",
     "parsl[monitoring] @ git+https://github.com/Parsl/parsl.git@globus_compute_executor.py",
     "uwtools @ git+https://github.com/ufs-community/[email protected]#subdirectory=src",
 ]
+requires-python = ">=3.9.0"
+authors = [
+  {name = "Christopher Harrop", email = "[email protected]"}
+]
+maintainers = [
+  {name = "Christopher Harrop", email = "[email protected]"}
+]
+description = "Federated NWP Workflow Tools"
+readme = "README.md"
+license = {file = "LICENSE"}
+keywords = ["federated", "workflow"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Programming Language :: Python",
+]
 
-#[project.scripts]
-#chiltepin = "chiltepin.cli:main"
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-flake8",
+    "pytest-black",
+    "pytest-isort",
+]
 
-[build-system]
-requires = ["setuptools>=61.0"]
-build-backend = "setuptools.build_meta"
+[project.scripts]
+chiltepin = "chiltepin.cli:main"
 
 [tool.pytest.ini_options]
 addopts = [

src/chiltepin/configure.py

@@ -1,6 +1,7 @@
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import yaml
+from globus_compute_sdk import Executor
 from parsl.config import Config
 from parsl.executors import GlobusComputeExecutor, HighThroughputExecutor, MPIExecutor
 from parsl.launchers import SimpleLauncher
@@ -180,7 +181,7 @@ def make_globus_compute_executor(
     """
     e = GlobusComputeExecutor(
         label=name,
-        endpoint_id=config["endpoint id"],
+        executor=Executor(endpoint_id=config["endpoint id"]),
         user_endpoint_config={
             "engine": config.get("engine", "GlobusComputeEngine"),
             "max_mpi_apps": config.get("max mpi apps", 1),
@@ -198,7 +199,7 @@ def make_globus_compute_executor(
     return e
 
 
-def load(config: Dict[str, Any]) -> Config:
+def load(config: Dict[str, Any], resources: List[str] | None = None) -> Config:
     """Construct a list of Executors from the input configuration dictionary
 
     The list returned by this function can be used to construct a Parsl Config
@@ -212,24 +213,30 @@ def load(config: Dict[str, Any]) -> Config:
         YAML configuration block that contains the configuration for a list of
         resources
 
+    resources: List[str] | None
+        A list of the labels of the resources to load. The default is None.
+        If None, all resources are loaded.  Otherwise the resources whose
+        labels are in the list will be loaded.
+
     Returns
     -------
 
     Config
     """
     executors = []
-    for name, spec in config["resources"].items():
-        match spec["engine"]:
-            case "HTEX":
-                # Make a HighThroughputExecutor
-                executors.append(make_htex_executor(name, spec))
-            case "MPI":
-                # Make an MPIExecutor
-                executors.append(make_mpi_executor(name, spec))
-            case "GlobusComputeEngine":
-                # Make a GlobusComputeExecutor for non-MPI jobs
-                executors.append(make_globus_compute_executor(name, spec))
-            case "GlobusMPIEngine":
-                # Make a GlobusComputeExecutor for MPI jobs
-                executors.append(make_globus_compute_executor(name, spec))
+    for name, spec in config.items():
+        if resources is None or name in resources:
+            match spec["engine"]:
+                case "HTEX":
+                    # Make a HighThroughputExecutor
+                    executors.append(make_htex_executor(name, spec))
+                case "MPI":
+                    # Make an MPIExecutor
+                    executors.append(make_mpi_executor(name, spec))
+                case "GlobusComputeEngine":
+                    # Make a GlobusComputeExecutor for non-MPI jobs
+                    executors.append(make_globus_compute_executor(name, spec))
+                case "GlobusMPIEngine":
+                    # Make a GlobusComputeExecutor for MPI jobs
+                    executors.append(make_globus_compute_executor(name, spec))
     return Config(executors)

tests/config.yaml

@@ -53,6 +53,31 @@ hercules:
       partition: "hercules"
       account: "gsd-hpcs"
       <<: *hercules-env
+    gc-mpi:
+      engine: GlobusMPIEngine
+      endpoint id: "{{ mpi_endpoint_id }}"
+      cores per node: 80
+      nodes per block: 3
+      max mpi apps: 2
+      partition: "hercules"
+      account: "gsd-hpcs"
+      <<: *hercules-env
+    gc-service:
+      engine: GlobusComputeEngine
+      endpoint id: "{{ service_endpoint_id }}"
+      cores per node: 1
+      nodes per block: 1
+      partition: "service"
+      account: "gsd-hpcs"
+      <<: *hercules-env
+    gc-compute:
+      engine: GlobusComputeEngine
+      endpoint id: "{{ compute_endpoint_id }}"
+      cores per node: 80
+      nodes per block: 1
+      partition: "hercules"
+      account: "gsd-hpcs"
+      <<: *hercules-env
 
 hera:
   resources:
@@ -78,6 +103,31 @@ hera:
       partition: "hera"
       account: "gsd-hpcs"
       <<: *hera-env
+    gc-mpi:
+      engine: GlobusMPIEngine
+      endpoint id: "{{ mpi_endpoint_id }}"
+      cores per node: 40
+      nodes per block: 3
+      max mpi apps: 2
+      partition: "hera"
+      account: "gsd-hpcs"
+      <<: *hera-env
+    gc-service:
+      engine: GlobusComputeEngine
+      endpoint id: "{{ service_endpoint_id }}"
+      cores per node: 1
+      nodes per block: 1
+      partition: "service"
+      account: "gsd-hpcs"
+      <<: *hera-env
+    gc-compute:
+      engine: GlobusComputeEngine
+      endpoint id: "{{ compute_endpoint_id }}"
+      cores per node: 40
+      nodes per block: 1
+      partition: "hera"
+      account: "gsd-hpcs"
+      <<: *hera-env
 
 docker:
   resources:
@@ -103,3 +153,28 @@ docker:
       partition: "slurmpar"
       account: ""
       <<: *docker-env
+    gc-mpi:
+      engine: GlobusMPIEngine
+      endpoint id: "{{ mpi_endpoint_id }}"
+      cores per node: 8
+      nodes per block: 3
+      max mpi apps: 2
+      partition: "slurmpar"
+      account: ""
+      <<: *docker-env
+    gc-service:
+      engine: GlobusComputeEngine
+      endpoint id: "{{ service_endpoint_id }}"
+      cores per node: 1
+      nodes per block: 1
+      partition: "slurmpar"
+      account: ""
+      <<: *docker-env
+    gc-compute:
+      engine: GlobusComputeEngine
+      endpoint id: "{{ compute_endpoint_id }}"
+      cores per node: 8
+      nodes per block: 1
+      partition: "slurmpar"
+      account: ""
+      <<: *docker-env