[develop] Fix failure on warm start option of SRW-AQM (ufs-community#…

…1065)

* Fix failure on the warm start option of SRW-AQM.
* Change the sample script config.aqm.yaml for running a warm start.
* Change cpreq to cp because it does not work correctly on other machines except for WCOSS2.
* Add missing exclusion to .gitignore.

.gitignore

@@ -8,6 +8,7 @@ lib/
 parm/aqm_utils_parm/
 parm/nexus_config/
 parm/ufs_utils_parm/
+parm/upp_parm/
 share/
 sorc/*/
 tests/WE2E/WE2E_tests_*.yaml

parm/wflow/coldstart.yaml

@@ -186,29 +186,21 @@ metatask_run_ensemble:
             attrs:
               task: point_source
         or_aqm_ics:
-          and_no_aqm_ics:
-            not:
-              taskvalid:
-                attrs:
-                  task: aqm_ics_ext
-            not:
-              taskvalid:
-                attrs:
-                  task: aqm_ics
-          and_aqm_atstart:
-            taskvalid:
-              attrs:
-                task: aqm_ics_ext
-            taskdep:
-              attrs:
-                task: aqm_ics_ext
-          and_aqm_cycled:
+          not:
             taskvalid:
               attrs:
                 task: aqm_ics
-            taskdep:
+          taskdep:
+            attrs:
+              task: aqm_ics
+        or_aqm_ics_ext:
+          not:
+            taskvalid:
               attrs:
-                task: aqm_ics
+                task: aqm_ics_ext
+          taskdep:
+            attrs:
+              task: aqm_ics_ext
         or_aqm_lbcs:
           not:
             taskvalid:

scripts/exregional_make_ics.sh

@@ -650,10 +650,10 @@ if [ "${CPL_AQM}" = "TRUE" ]; then
   else
     data_trans_path="${DATA_SHARE}"
   fi
-  cpreq -p out.atm.tile${TILE_RGNL}.nc "${data_trans_path}/${NET}.${cycle}${dot_ensmem}.gfs_data.tile${TILE_RGNL}.halo${NH0}.nc"
-  cpreq -p out.sfc.tile${TILE_RGNL}.nc "${COMOUT}/${NET}.${cycle}${dot_ensmem}.sfc_data.tile${TILE_RGNL}.halo${NH0}.nc"
-  cpreq -p gfs_ctrl.nc "${COMOUT}/${NET}.${cycle}${dot_ensmem}.gfs_ctrl.nc"
-  cpreq -p gfs.bndy.nc "${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile${TILE_RGNL}.f000.nc"
+  cp -p out.atm.tile${TILE_RGNL}.nc "${data_trans_path}/${NET}.${cycle}${dot_ensmem}.gfs_data.tile${TILE_RGNL}.halo${NH0}.nc"
+  cp -p out.sfc.tile${TILE_RGNL}.nc "${COMOUT}/${NET}.${cycle}${dot_ensmem}.sfc_data.tile${TILE_RGNL}.halo${NH0}.nc"
+  cp -p gfs_ctrl.nc "${COMOUT}/${NET}.${cycle}${dot_ensmem}.gfs_ctrl.nc"
+  cp -p gfs.bndy.nc "${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile${TILE_RGNL}.f000.nc"
 else
   mv_vrfy out.atm.tile${TILE_RGNL}.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_data.tile${TILE_RGNL}.halo${NH0}.nc
   mv_vrfy out.sfc.tile${TILE_RGNL}.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.sfc_data.tile${TILE_RGNL}.halo${NH0}.nc

scripts/exregional_make_lbcs.sh

@@ -560,7 +560,7 @@ located in the following directory:
   fcst_hhh=$(( ${lbc_spec_fhrs} - ${EXTRN_MDL_LBCS_OFFSET_HRS} ))
   fcst_hhh_FV3LAM=$( printf "%03d" "$fcst_hhh" )
   if [ "${CPL_AQM}" = "TRUE" ]; then
-    cpreq -p gfs.bndy.nc ${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fcst_hhh_FV3LAM}.nc
+    cp -p gfs.bndy.nc ${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fcst_hhh_FV3LAM}.nc
   else
     mv_vrfy gfs.bndy.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fcst_hhh_FV3LAM}.nc
   fi

scripts/exsrw_aqm_ics.sh

@@ -55,31 +55,54 @@ tial or boundary condition files for the FV3 will be generated.
 #
 rst_dir="${PREV_CYCLE_DIR}/RESTART"
 rst_file="fv_tracer.res.tile1.nc"
-fv_tracer_file="${rst_dir}/${PDY}.${cyc}0000.${rst_file}"
-print_info_msg "Looking for tracer restart file: \"${fv_tracer_file}\""
-if [ ! -r ${fv_tracer_file} ]; then
-  if [ -r ${rst_dir}/coupler.res ]; then
-    rst_info=( $( tail -n 1 ${rst_dir}/coupler.res ) )
-    # Remove leading zeros from ${rst_info[1]}
-    month="${rst_info[1]#"${rst_info[1]%%[!0]*}"}"
-    # Remove leading zeros from ${rst_info[2]}
-    day="${rst_info[2]#"${rst_info[2]%%[!0]*}"}"
-    # Format the date without leading zeros
-    rst_date=$(printf "%04d%02d%02d%02d" ${rst_info[0]} $((10#$month)) $((10#$day)) ${rst_info[3]})
-    print_info_msg "
-  Tracer file not found. Checking available restart date:
-    requested date: \"${PDY}${cyc}\"
-    available date: \"${rst_date}\""
-    if [ "${rst_date}" = "${PDY}${cyc}" ] ; then
-      fv_tracer_file="${rst_dir}/${rst_file}"
-      if [ -r ${fv_tracer_file} ]; then
-        print_info_msg "Tracer file found: \"${fv_tracer_file}\""
-      else
-        message_txt="FATAL ERROR No suitable tracer restart file ${rst_dir}/${rst_file} found."
-        err_exit "${message_txt}"
-        print_err_msg_exit "${message_txt}"
-      fi
+rst_file_with_date="${PDY}.${cyc}0000.${rst_file}"
+if [ -e "${rst_dir}/${rst_file_with_date}" ]; then
+  fv_tracer_file="${rst_dir}/${rst_file_with_date}"
+elif [ -e "${rst_dir}/${rst_file}" ]; then
+  fv_tracer_file="${rst_dir}/${rst_file}"
+else
+  message_txt="Tracer restart file: \"${fv_tracer_file}\" is NOT found"
+  err_exit "${message_txt}"
+  print_err_msg_exit "${message_txt}"
+fi
+print_info_msg "Tracer restart file: \"${fv_tracer_file}\""
+
+cplr_file="coupler.res"
+cplr_file_with_date="${PDY}.${cyc}0000.${cplr_file}"
+if [ -e "${rst_dir}/${cplr_file_with_date}" ]; then
+  coupler_file="${rst_dir}/${cplr_file_with_date}"
+elif [ -e "${rst_dir}/${cplr_file}" ]; then
+  coupler_file="${rst_dir}/${cplr_file}"
+else
+  message_txt="Coupler file: \"${coupler_file}\" is NOT found"
+  err_exit "${message_txt}"
+  print_err_msg_exit "${message_txt}"
+fi
+print_info_msg "Coupler file: \"${coupler_file}\""
+
+if [ -r ${coupler_file} ]; then
+  rst_info=( $( tail -n 1 ${coupler_file} ) )
+  # Remove leading zeros from ${rst_info[1]}
+  month="${rst_info[1]#"${rst_info[1]%%[!0]*}"}"
+  # Remove leading zeros from ${rst_info[2]}
+  day="${rst_info[2]#"${rst_info[2]%%[!0]*}"}"
+  # Format the date without leading zeros
+  rst_date=$(printf "%04d%02d%02d%02d" ${rst_info[0]} $((10#$month)) $((10#$day)) ${rst_info[3]})
+  if [ "${rst_date}" = "${PDY}${cyc}" ]; then
+    if [ -r ${fv_tracer_file} ]; then
+      print_info_msg "Tracer restart file is for ${PDY}${cyc}"
+    else
+      message_txt="Tracer restart file \"${fv_tracer_file}\" is NOT readable."
+      err_exit "${message_txt}"
+      print_err_msg_exit "${message_txt}"
     fi
+  else
+    message_txt="Tracer restart file is NOT for ${PDY}${cyc}. 
+Checking available restart date:
+  requested date: \"${PDY}${cyc}\"
+  available date: \"${rst_date}\""
+    err_exit "${message_txt}"
+    print_err_msg_exit "${message_txt}"
   fi
 fi
 #
@@ -103,7 +126,7 @@ print_info_msg "
     tracer file: \"${fv_tracer_file}\"
     FV3 IC file: \"${gfs_ic_fp}\""
 
-cpreq ${gfs_ic_fp} ${wrk_ic_fp}
+cp -p ${gfs_ic_fp} ${wrk_ic_fp}
 ${USHsrw}/aqm_utils_python/add_aqm_ics.py --fv_tracer_file "${fv_tracer_file}" --wrk_ic_file "${wrk_ic_fp}"
 export err=$?
 if [ $err -ne 0 ]; then
@@ -122,10 +145,7 @@ fi
 
 mv tmp1.nc ${gfs_ic_fn}
 
-cpreq -p ${gfs_ic_fn} ${COMOUT}
-cpreq -p "${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.sfc_data.tile${TILE_RGNL}.halo${NH0}.nc" ${COMOUT}
-cpreq -p "${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.gfs_ctrl.nc" ${COMOUT}
-cpreq -p "${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile${TILE_RGNL}.f000.nc" ${COMOUT}
+cp -p ${gfs_ic_fn} ${COMOUT}
 
 unset fv_tracer_file
 unset wrk_ic_file

scripts/exsrw_aqm_lbcs.sh

@@ -97,7 +97,7 @@ aqm_lbcs_fn_prefix="${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f"
 for hr in 0 ${LBC_SPEC_FCST_HRS[@]}; do
   fhr=$( printf "%03d" "${hr}" )
   aqm_lbcs_fn="${aqm_lbcs_fn_prefix}${fhr}.nc"
-  cpreq "${DATA_SHARE}/${aqm_lbcs_fn}" ${DATA}
+  cp -p "${DATA_SHARE}/${aqm_lbcs_fn}" ${DATA}
 done
 
 if [ "${DO_AQM_CHEM_LBCS}" = "TRUE" ]; then
@@ -106,7 +106,7 @@ if [ "${DO_AQM_CHEM_LBCS}" = "TRUE" ]; then
   chem_lbcs_fp="${FIXaqm}/chemlbc/${chem_lbcs_fn}"
   if [ -f ${chem_lbcs_fp} ]; then
     #Copy the boundary condition file to the current location
-    cpreq ${chem_lbcs_fp} .
+    cp -p ${chem_lbcs_fp} .
   else
     message_txt="The chemical LBC files do not exist:
     CHEM_BOUNDARY_CONDITION_FILE = \"${chem_lbcs_fp}\""
@@ -125,7 +125,7 @@ if [ "${DO_AQM_CHEM_LBCS}" = "TRUE" ]; then
         err_exit "${message_txt}"
         print_err_msg_exit "${message_txt}"
       fi
-      cpreq ${aqm_lbcs_fn} "${aqm_lbcs_fn}_chemlbc"
+      cp -p ${aqm_lbcs_fn} "${aqm_lbcs_fn}_chemlbc"
     fi
   done
 
@@ -229,7 +229,7 @@ fi
 for hr in 0 ${LBC_SPEC_FCST_HRS[@]}; do
   fhr=$( printf "%03d" "${hr}" )
   aqm_lbcs_fn="${aqm_lbcs_fn_prefix}${fhr}.nc"
-  cpreq -p "${DATA}/${aqm_lbcs_fn}" ${COMOUT}
+  cp -p "${DATA}/${aqm_lbcs_fn}" ${COMOUT}
 done
 #
 print_info_msg "

scripts/exsrw_fire_emission.sh

@@ -70,7 +70,7 @@ aqm_fire_file_fn="${AQM_FIRE_FILE_PREFIX}_${YYYYMMDD}_t${HH}z${AQM_FIRE_FILE_SUF
 
 # Check if the fire file exists in the designated directory
 if [ -e "${COMINfire}/${aqm_fire_file_fn}" ]; then
-  cpreq "${COMINfire}/${aqm_fire_file_fn}" ${COMOUT}
+  cp -p "${COMINfire}/${aqm_fire_file_fn}" ${COMOUT}
 else
   # Copy raw data 
   for ihr in {0..23}; do
@@ -83,16 +83,16 @@ else
     yyyymmdd_dn_md1="${missing_download_time:0:8}"
     FILE_13km_md1="RAVE-HrlyEmiss-13km_v*_blend_s${missing_download_time}00000_e${missing_download_time}59590_c*.nc"
     if [ -s `ls ${COMINfire}/${yyyymmdd_dn}/rave/${FILE_13km}` ] && [ $(stat -c %s `ls ${COMINfire}/${yyyymmdd_dn}/rave/${FILE_13km}`) -gt 4000000 ]; then
-      cpreq -p ${COMINfire}/${yyyymmdd_dn}/rave/${FILE_13km} ${FILE_curr}
+      cp -p ${COMINfire}/${yyyymmdd_dn}/rave/${FILE_13km} ${FILE_curr}
     elif [ -s `ls ${COMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1}` ] && [ $(stat -c %s `ls ${COMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1}`) -gt 4000000 ]; then
       echo "WARNING: ${FILE_13km} does not exist or broken. Replacing with the file of previous date ..."
-      cpreq -p ${COMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1} ${FILE_curr}
+      cp -p ${COMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1} ${FILE_curr}
     else
       message_txt="WARNING Fire Emission RAW data does not exist or broken:
   FILE_13km_md1 = \"${FILE_13km_md1}\"
   DCOMINfire = \"${DCOMINfire}\""
 
-      cpreq -p ${FIXaqm}/fire/Hourly_Emissions_13km_dummy.nc ${FILE_curr}
+      cp -p ${FIXaqm}/fire/Hourly_Emissions_13km_dummy.nc ${FILE_curr}
       print_info_msg "WARNING: ${message_txt}. Replacing with the dummy file :: AQM RUN SOFT FAILED."
     fi
   done  
@@ -134,8 +134,8 @@ else
     print_err_msg_exit "${message_txt}"
   fi
 
-  cpreq Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24.nc Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24_1.nc
-  cpreq Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24.nc Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24_2.nc
+  cp -p Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24.nc Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24_1.nc
+  cp -p Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24.nc Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24_2.nc
 
   ncrcat -O -D 2 Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24.nc Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24_1.nc Hourly_Emissions_regrid_NA_13km_${YYYYMMDD}_t${HH}z_h24_2.nc ${aqm_fire_file_fn}
   export err=$?
@@ -155,7 +155,7 @@ else
   mv temp6.nc ${aqm_fire_file_fn}
 
   # Copy the final fire emission file to data share directory
-  cpreq "${DATA}/${aqm_fire_file_fn}" ${COMOUT}
+  cp -p "${DATA}/${aqm_fire_file_fn}" ${COMOUT}
 fi
 #
 #-----------------------------------------------------------------------

scripts/exsrw_nexus_emission.sh

@@ -103,12 +103,12 @@ fi
 #
 #-----------------------------------------------------------------------
 #
-cpreq ${FIXaqm}/nexus/${NEXUS_GRID_FN} ${DATA}/grid_spec.nc
+cp -p ${FIXaqm}/nexus/${NEXUS_GRID_FN} ${DATA}/grid_spec.nc
 
 if [ "${USE_GFS_SFC}" = "TRUE" ]; then
-  cpreq ${PARMsrw}/nexus_config/cmaq_gfs_megan/*.rc ${DATA}
+  cp -p ${PARMsrw}/nexus_config/cmaq_gfs_megan/*.rc ${DATA}
 else
-  cpreq ${PARMsrw}/nexus_config/cmaq/*.rc ${DATA}
+  cp -p ${PARMsrw}/nexus_config/cmaq/*.rc ${DATA}
 fi
 #
 #-----------------------------------------------------------------------

scripts/exsrw_nexus_post_split.sh

@@ -74,12 +74,12 @@ end_date=`$NDATE +${FCST_LEN_HRS} ${YYYYMMDD}${HH}`
 #
 #-----------------------------------------------------------------------
 #
-cpreq ${PARMsrw}/nexus_config/cmaq/HEMCO_sa_Time.rc ${DATA}/HEMCO_sa_Time.rc
-cpreq ${FIXaqm}/nexus/${NEXUS_GRID_FN} ${DATA}/grid_spec.nc
+cp -p ${PARMsrw}/nexus_config/cmaq/HEMCO_sa_Time.rc ${DATA}/HEMCO_sa_Time.rc
+cp -p ${FIXaqm}/nexus/${NEXUS_GRID_FN} ${DATA}/grid_spec.nc
 
 if [ "${NUM_SPLIT_NEXUS}" = "01" ]; then
   nspt="00"
-  cpreq ${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.${nspt}.nc ${DATA}/NEXUS_Expt_combined.nc
+  cp -p ${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.${nspt}.nc ${DATA}/NEXUS_Expt_combined.nc
 else
   ${USHsrw}/nexus_utils/python/concatenate_nexus_post_split.py "${DATA_SHARE}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt_split.*.nc" "${DATA}/NEXUS_Expt_combined.nc"
   export err=$?
@@ -110,7 +110,7 @@ fi
 #
 #-----------------------------------------------------------------------
 #
-mv ${DATA}/NEXUS_Expt.nc ${COMOUT}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt.nc
+cp -p ${DATA}/NEXUS_Expt.nc ${COMOUT}/${NET}.${cycle}${dot_ensmem}.NEXUS_Expt.nc
 #
 # Print message indicating successful completion of script.
 #

ush/config.aqm.community.yaml → ush/config.aqm.yaml

@@ -1,13 +1,13 @@
 metadata:
-  description: config for Online-CMAQ, AQM_NA_13km, community mode
+  description: config for Online-CMAQ, AQM_NA_13km, warm-start
 user:
   RUN_ENVIR: community
   MACHINE: hera
   ACCOUNT: [account name]
 workflow:
   USE_CRON_TO_RELAUNCH: true
   CRON_RELAUNCH_INTVL_MNTS: 3
-  EXPT_SUBDIR: aqm_community_aqmna13
+  EXPT_SUBDIR: aqm_AQMNA13km_warmstart
   PREDEF_GRID_NAME: AQM_NA_13km
   CCPP_PHYS_SUITE: FV3_GFS_v16
   DATE_FIRST_CYCL: '2023111000'
@@ -21,12 +21,16 @@ workflow:
   DIAG_TABLE_TMPL_FN: diag_table_aqm.FV3_GFS_v16
   FIELD_TABLE_TMPL_FN: field_table_aqm.FV3_GFS_v16
   DO_REAL_TIME: false
+  COLDSTART: false      # set to true for cold start
+  WARMSTART_CYCLE_DIR: '/scratch2/NAGAPE/epic/SRW-AQM_DATA/aqm_data/restart/2023111000'
 nco:
+  envir_default: test_aqm_warmstart
   NET_default: aqm
+  RUN_default: aqm
 rocoto:
   tasks:
     taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}'
-    task_aqm_ics_ext:
+#    task_aqm_ics_ext:    # uncomment this in case of COLDSTART: true
     metatask_run_ensemble:
       task_run_fcst_mem#mem#:
         walltime: 01:00:00