From 4f12095be52df8ded0c795fef09a74dab081bb2d Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Fri, 4 Oct 2024 01:34:28 -0600
Subject: [PATCH] update derecho setups for new GPU XML variables

---
 machines/Depends.nvhpc               |  6 ++---
 machines/cmake_macros/nvhpc.cmake    | 34 +++++++++++++---------------
 machines/derecho/config_machines.xml |  7 ++----
 machines/derecho/nvhpc_derecho.cmake |  4 +++-
 4 files changed, 24 insertions(+), 27 deletions(-)

diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
index 84b6596f..f01bdda7 100644
--- a/machines/Depends.nvhpc
+++ b/machines/Depends.nvhpc
@@ -154,9 +154,9 @@ ifeq ($(DEBUG),FALSE)
 #  $(SHR_RANDNUM_C_OBJS): %.o: %.c
 #	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
   $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(OPENACC_GPU_FLAGS) $<
   $(RRTMGP_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
   $(CLUBB_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
 endif
diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake
index d541457e..39c91fce 100644
--- a/machines/cmake_macros/nvhpc.cmake
+++ b/machines/cmake_macros/nvhpc.cmake
@@ -51,21 +51,19 @@ set(MPIFC "mpif90")
 set(SCC "nvc")
 set(SCXX "nvc++")
 set(SFC "nvfortran")
-if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openacc)
-   string(APPEND GPUFLAGS  " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ")
-endif()
-if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openmp)
-   string(APPEND GPUFLAGS  " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ")
-endif()
-if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL combined)
-   string(APPEND GPUFLAGS  " -acc -gpu=cc70,lineinfo,nofma -mp=gpu -Minfo=accel ")
-endif()
-if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openacc)
-   string(APPEND GPUFLAGS  " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ")
-endif()
-if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openmp)
-   string(APPEND GPUFLAGS  " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ")
-endif()
-if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL combined)
-   string(APPEND GPUFLAGS  " -acc -gpu=cc80,lineinfo,nofma -mp=gpu -Minfo=accel")
-endif()
+if (GPU_TYPE STREQUAL v100)
+  if (OPENACC_GPU_OFFLOAD)
+    string(APPEND OPENACC_GPU_FLAGS " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+  endif()
+  if (OPENMP_GPU_OFFLOAD)
+    string(APPEND OPENMP_GPU_FLAGS " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+  endif()
+endif()
+if (GPU_TYPE STREQUAL a100)
+  if (OPENACC_GPU_OFFLOAD)
+    string(APPEND OPENACC_GPU_FLAGS " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+  endif()
+  if (OPENMP_GPU_OFFLOAD)
+    string(APPEND OPENMP_GPU_FLAGS " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+  endif()
+endif()
\ No newline at end of file
diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml
index 12446f37..7bda6387 100644
--- a/machines/derecho/config_machines.xml
+++ b/machines/derecho/config_machines.xml
@@ -16,9 +16,6 @@
     <MAX_GPUS_PER_NODE>4</MAX_GPUS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
     <MAX_CPUTASKS_PER_GPU_NODE>64</MAX_CPUTASKS_PER_GPU_NODE>
-    <GPU_TYPE>none,a100</GPU_TYPE>
-    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_OFFLOAD>
-    <MPI_GPU_WRAPPER_SCRIPT>set_gpu_rank</MPI_GPU_WRAPPER_SCRIPT>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpibind</executable>
@@ -75,7 +72,7 @@
 	<command name="load">cray-mpich/8.1.27</command>
       </modules>
 
-      <modules mpilib="mpich" compiler="nvhpc" gpu_offload="!none">
+      <modules mpilib="mpich" compiler="nvhpc" gpu_type="!none">
         <command name="load">cuda/12.2.1</command>
       </modules>
 
@@ -121,7 +118,7 @@
       <env name="ESMF_RUNTIME_PROFILE">ON</env>
       <env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
     </environment_variables>
-    <environment_variables compiler="nvhpc" gpu_offload="!none">
+    <environment_variables compiler="nvhpc" gpu_type="!none">
       <env name="NCAR_LIBS_CUDA">-lcuda -lcudart</env>
     </environment_variables>
   </machine>
diff --git a/machines/derecho/nvhpc_derecho.cmake b/machines/derecho/nvhpc_derecho.cmake
index f0b3cd87..2c497d0f 100644
--- a/machines/derecho/nvhpc_derecho.cmake
+++ b/machines/derecho/nvhpc_derecho.cmake
@@ -4,4 +4,6 @@ endif()
 if (NOT DEBUG)
   string(APPEND FFLAGS " -tp=zen3")
 endif()
-message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")
+message("GPU_TYPE is ${GPU_TYPE}")
+message("OPENACC_GPU_OFFLOAD is ${OPENACC_GPU_OFFLOAD}")
+message("OPENMP_GPU_OFFLOAD is ${OPENMP_GPU_OFFLOAD}")
\ No newline at end of file