[develop] Port SRW-AQM to Orion and Hercules (ufs-community#1067)

* Port SRW-AQM to Orion and Hercules
chan-hoo · Apr 8, 2024 · bf7b490 · bf7b490
1 parent d1401ec
commit bf7b490
Show file tree

Hide file tree

Showing 23 changed files with 34 additions and 30 deletions.
diff --git a/modulefiles/build_derecho_intel.lua b/modulefiles/build_derecho_intel.lua
@@ -14,5 +14,7 @@ load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3"))
 
 load("srw_common")
 
+load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
+
 setenv("CMAKE_Platform","derecho.intel")
 
diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua
@@ -17,6 +17,7 @@ load("srw_common")
 
 load("nccmp/1.9.0.1")
 load("nco/5.0.6")
+load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
 
 setenv("CFLAGS","-diag-disable=10441")
 setenv("FFLAGS","-diag-disable=10441")

diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua
@@ -18,6 +18,7 @@ load("srw_common")
 load("nccmp/1.9.0.1")
 load("nco/5.0.6")
 load("wget")
+load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
 
 setenv("CMAKE_C_COMPILER","mpiicc")
 setenv("CMAKE_CXX_COMPILER","mpiicpc")

diff --git a/modulefiles/tasks/derecho/nexus_post_split.local.lua b/modulefiles/tasks/derecho/nexus_post_split.local.lua
@@ -1,3 +1,4 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
+load("nco/5.0.6")
+
 load("ncarenv")
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/derecho/python_srw.lua b/modulefiles/tasks/derecho/python_srw.lua
diff --git a/modulefiles/tasks/hercules/aqm_ics.local.lua b/modulefiles/tasks/hercules/aqm_ics.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/hercules/aqm_lbcs.local.lua b/modulefiles/tasks/hercules/aqm_lbcs.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
-load("miniconda_regional_workflow_cmaq")
+load("python_srw_aqm")
diff --git a/modulefiles/tasks/hercules/fire_emission.local.lua b/modulefiles/tasks/hercules/fire_emission.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/hercules/nexus_emission.local.lua b/modulefiles/tasks/hercules/nexus_emission.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/hercules/nexus_post_split.local.lua b/modulefiles/tasks/hercules/nexus_post_split.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/hercules/python_srw.lua b/modulefiles/tasks/hercules/python_srw.lua
diff --git a/modulefiles/tasks/orion/aqm_ics.local.lua b/modulefiles/tasks/orion/aqm_ics.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/orion/aqm_lbcs.local.lua b/modulefiles/tasks/orion/aqm_lbcs.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
-load("miniconda_regional_workflow_cmaq")
+load("python_srw_aqm")
diff --git a/modulefiles/tasks/orion/fire_emission.local.lua b/modulefiles/tasks/orion/fire_emission.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/orion/nexus_emission.local.lua b/modulefiles/tasks/orion/nexus_emission.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/orion/nexus_post_split.local.lua b/modulefiles/tasks/orion/nexus_post_split.local.lua
@@ -1,2 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 load("python_srw_aqm")
diff --git a/modulefiles/tasks/orion/python_srw.lua b/modulefiles/tasks/orion/python_srw.lua
diff --git a/parm/wflow/aqm_prep.yaml b/parm/wflow/aqm_prep.yaml
@@ -106,6 +106,7 @@ task_aqm_ics_ext:
     <<: *default_vars
     PREV_CYCLE_DIR: '&WARMSTART_CYCLE_DIR;'
   join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+  memory: 64G
   dependency:
     and:
       taskdep:
@@ -131,6 +132,7 @@ task_aqm_ics:
     <<: *default_vars
     PREV_CYCLE_DIR: '<cyclestr offset="-{{ workflow.INCR_CYCL_FREQ }}:00:00">&COMIN_DIR;</cyclestr>'
   join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+  memory: 64G
   dependency:
     and:
       taskdep:

diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml
@@ -21,7 +21,7 @@ rocoto:
     task_aqm_ics_ext:
     metatask_run_ensemble:
       task_run_fcst_mem#mem#:
-        walltime: 01:00:00
+        walltime: 01:20:00
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS
   FV3GFS_FILE_FMT_ICS: netcdf

diff --git a/ush/config.aqm.yaml b/ush/config.aqm.yaml
@@ -1,8 +1,8 @@
 metadata:
-  description: config for Online-CMAQ, AQM_NA_13km, warm-start
+  description: config for SRW-AQM, AQM_NA_13km, warm-start
 user:
   RUN_ENVIR: community
-  MACHINE: hera
+  MACHINE: [hera/orion/hercules/derecho]
   ACCOUNT: [account name]
 workflow:
   USE_CRON_TO_RELAUNCH: true
@@ -22,7 +22,9 @@ workflow:
   FIELD_TABLE_TMPL_FN: field_table_aqm.FV3_GFS_v16
   DO_REAL_TIME: false
   COLDSTART: false      # set to true for cold start
-  WARMSTART_CYCLE_DIR: '/scratch2/NAGAPE/epic/SRW-AQM_DATA/aqm_data/restart/2023111000'
+  WARMSTART_CYCLE_DIR: '/scratch2/NAGAPE/epic/SRW-AQM_DATA/aqm_data/restart/2023111000' # for hera
+#  WARMSTART_CYCLE_DIR: '/work/noaa/epic/SRW-AQM_DATA/aqm_data/restart/2023111000' # for orion/hercules
+#  WARMSTART_CYCLE_DIR: '' # for derecho
 nco:
   envir_default: test_aqm_warmstart
   NET_default: aqm
@@ -33,7 +35,7 @@ rocoto:
 #    task_aqm_ics_ext:    # uncomment this in case of COLDSTART: true
     metatask_run_ensemble:
       task_run_fcst_mem#mem#:
-        walltime: 01:00:00
+        walltime: 01:20:00
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS
   FV3GFS_FILE_FMT_ICS: netcdf

diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml
@@ -16,7 +16,7 @@ platform:
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: mpiexec -n $nprocs 
   RUN_CMD_NEXUS: mpiexec -n $nprocs 
-  RUN_CMD_AQMLBC: mpiexec -n ${NUMTS}  
+  RUN_CMD_AQMLBC: mpiexec -n ${numts}
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data
   TEST_AQM_INPUT_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/aqm_data

diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml
@@ -19,7 +19,7 @@ platform:
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: srun --export=ALL -n $nprocs
   RUN_CMD_NEXUS: srun --export=ALL
-  RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS}
+  RUN_CMD_AQMLBC: srun --export=ALL -n ${numts}
   SCHED_NATIVE_CMD: --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data
@@ -33,6 +33,8 @@ platform:
   FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog
   FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo
   FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth
+  FIXaqm: /work/noaa/epic/SRW-AQM_DATA/fix_aqm
+  FIXemis: /work/noaa/epic/SRW-AQM_DATA/fix_emis
   EXTRN_MDL_DATA_STORES: aws
 data:
   ics_lbcs:
@@ -44,3 +46,8 @@ data:
     HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh}
     RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}
     GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh}
+
+cpl_aqm_parm:
+  COMINfire_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/RAVE_fire
+  COMINgefs_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GEFS_DATA
+  NEXUS_GFS_SFC_DIR: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GFS_SFC_DATA
diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml
@@ -19,7 +19,7 @@ platform:
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: srun --export=ALL
   RUN_CMD_NEXUS: srun --export=ALL
-  RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS}
+  RUN_CMD_AQMLBC: srun --export=ALL -n ${numts}
   SCHED_NATIVE_CMD: --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data
@@ -32,6 +32,8 @@ platform:
   FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog
   FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo
   FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth
+  FIXaqm: /work/noaa/epic/SRW-AQM_DATA/fix_aqm
+  FIXemis: /work/noaa/epic/SRW-AQM_DATA/fix_emis
   EXTRN_MDL_DATA_STORES: aws nomads
 data:
   ics_lbcs:
@@ -43,3 +45,8 @@ data:
     HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh}
     RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}
     GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh}
+
+cpl_aqm_parm:
+  COMINfire_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/RAVE_fire
+  COMINgefs_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GEFS_DATA
+  NEXUS_GFS_SFC_DIR: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GFS_SFC_DATA