[develop] Expand forecast fields for metric test (ufs-community#1048)

This PR expands the number of forecast fields for the Skill Score metric test. The forecast length in the metric WE2E test was extended to 12 hours so that the RMSE metric can be calculated for these additional forecast fields:

 * Specific humidity for the full column
 * Temperature for the full column
 * Wind for the full column
 * Dew point, pressure, temperature, and wind at the surface level for forecast hour 12.

Adding these additional forecast fields will make the skill score metric test more thorough and thus making it a more inclusive test to compare against.

Also, a change was made to the .cicd/scripts/srw_metric_example.sh script to reflect the new conda environment.

---------

Co-authored-by: Parallel Works app-run user <Edward.Snyder@mgmt-edwardsnyder-aws-00115.pw-noaa-us-east-1.pw.local>
Co-authored-by: Parallel Works app-run user <Edward.Snyder@mgmt-edwardsnyder-aws-00122.pw-noaa-us-east-1.pw.local>

.cicd/scripts/srw_metric_example.sh

@@ -3,22 +3,45 @@
 # The goal of this script is to provide an example of performing Indy-Severe-Weather test run and compare results to reference with
 # Skill score index that is calculated by MET Stat-Analysis Tools
 #
-# Required:
+# Required (these options are set in the Jenkins env):
 #    WORKSPACE=</full/path/to/ufs-srweather-app>
 #    SRW_PLATFORM=<supported_platform_host>
 #    SRW_COMPILER=<intel|gnu>
+#    SRW_PROJECT=<platform_account>
 #
 # Optional:
-[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account"
+#[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account"
 [[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true
 set -e -u -x
 
+BUILD_OPT=false
+RUN_WE2E_OPT=false
+RUN_STAT_ANLY_OPT=false
+
+if [[ $# -eq 0 ]]; then
+    BUILD_OPT=true
+    RUN_WE2E_OPT=true
+    RUN_STAT_ANLY_OPT=true
+elif [[ $# -ge 4 ]]; then 
+    echo "Too many arguments, expecting three or less"
+    exit 1
+else 
+    for opt in "$@"; do    
+       case $opt in
+            build) BUILD_OPT=true ;; 
+            run_we2e) RUN_WE2E_OPT=true ;;
+            run_stat_anly) RUN_STAT_ANLY_OPT=true ;;
+            *) echo "Not valid option. Exiting!" ; exit 1 ;;
+        esac
+    done
+fi
+
 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 
 # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set
 # relative to script directory.
 declare workspace
-if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then
+if [[ -d "${WORKSPACE}/${SRW_PLATFORM}" ]]; then
     workspace="${WORKSPACE}/${SRW_PLATFORM}"
 else
     workspace="$(cd -- "${script_dir}/../.." && pwd)"
@@ -39,53 +62,85 @@ we2e_test_name="grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0"
 
 pwd
 
-# Activate the workflow environment ...
+# Setup the build environment
+declare srw_compiler
+srw_compiler=${SRW_COMPILER} 
 source etc/lmod-setup.sh ${platform,,}
 module use modulefiles
-module load build_${platform,,}_${SRW_COMPILER}
+module load build_${platform,,}_${srw_compiler}
+
+# Build srw
+if [[ ${BUILD_OPT} == true ]]; then
+    cd ${workspace}/tests
+    ./build.sh ${platform,,} ${srw_compiler}
+fi
+cd ${workspace}
+
+# Activate workflow environment
 module load wflow_${platform,,}
 
-[[ ${FORGIVE_CONDA} == true ]] && set +e +u    # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests
-conda activate workflow_tools
+[[ ${FORGIVE_CONDA} == true ]] && set +e +u    # Some platforms have incomplete python3 or conda support, but would not necessarily block workflow tests
+conda activate srw_app
 set -e -u
 
-# build srw
-cd ${workspace}/tests
-./build.sh ${platform,,} ${SRW_COMPILER}
-cd ${workspace}
-
-# run test
-[[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir}
-cd ${workspace}/tests/WE2E
-./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${SRW_PROJECT} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q
+# Run test
+declare srw_project
+srw_project=${SRW_PROJECT}
+if [[ ${RUN_WE2E_OPT} == true ]]; then
+    [[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir}
+    cd ${workspace}/tests/WE2E
+    ./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${srw_project} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q
+fi
 cd ${workspace}
 
-# run skill-score check
-[[ ! -f Indy-Severe-Weather.tgz ]] && wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz
-[[ ! -d Indy-Severe-Weather ]] && tar xvfz Indy-Severe-Weather.tgz
-[[ -f skill-score.out ]] && rm skill-score.out
-# Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. 
-# It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases.
-# In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, 
-# temperature, and pressure at lowest level in the atmosphere over 6 hour lead time.
-cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/
-# load met and metplus
-module use modulefiles/tasks/${platform,,}
-module load run_vx.local 
-stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.out
+# Run skill-score check
+if [[ ${RUN_STAT_ANLY_OPT} == true ]]; then
+    # Clear out data
+    rm -rf ${workspace}/Indy-Severe-Weather/
+    # Check if metprd data exists locally otherwise get it from S3
+    TEST_EXTRN_MDL_SOURCE_BASEDIR=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${SRW_PLATFORM}.yaml | awk '{print $NF}')
+    if [[ ! -d $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat ]] ; then
+        mkdir -p Indy-Severe-Weather/metprd/point_stat
+        cp -rp $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat Indy-Severe-Weather/metprd
+    elif [[ -f Indy-Severe-Weather.tgz ]]; then
+        tar xvfz Indy-Severe-Weather.tgz 
+    else
+        wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz
+        tar xvfz Indy-Severe-Weather.tgz
+    fi
+    [[ -f skill-score.txt ]] && rm skill-score.txt
+    # Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. 
+    # It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases.
+    # In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, 
+    # temperature, and pressure at lowest level in the atmosphere over 6 hour lead time.
+    cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/
+    # Remove conda for Orion due to conda env conflicts
+    if [[ ${platform} =~ "orion" ]]; then
+        sed -i 's|load("conda")|--load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua
+    fi
+    # Load met and metplus
+    module use modulefiles/tasks/${platform,,}
+    module load run_vx.local 
+    # Reset Orion run_vx.local file
+    if [[ ${platform} =~ "orion" ]]; then
+       sed -i 's|--load("conda")|load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua
+    fi
+    # Run stat_analysis
+    stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.txt
 
-# check skill-score.out
-cat skill-score.out
+    # check skill-score.txt
+    cat skill-score.txt
 
-# get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0
-# A value greater than 1.0 indicates that the forecast model outperforms the reference, 
-# while a value less than 1.0 indicates that the reference outperforms the forecast.
-tmp_string=$( tail -2 skill-score.out | head -1 )
-SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}')
-echo "Skill Score: ${SS_INDEX}"
-if [[ ${SS_INDEX} < "0.700" ]]; then
-    echo "Your Skill Score is way smaller than 1.00, better check before merging"
-    exit 1
-else
-    echo "Congrats! You pass check!"
+    # get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0
+    # A value greater than 1.0 indicates that the forecast model outperforms the reference, 
+    # while a value less than 1.0 indicates that the reference outperforms the forecast.
+    tmp_string=$( tail -2 skill-score.txt | head -1 )
+    SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}')
+    echo "Skill Score: ${SS_INDEX}"
+    if [[ ${SS_INDEX} < "0.700" ]]; then
+        echo "Your Skill Score is way smaller than 1.00, better check before merging"
+        exit 1
+    else
+        echo "Congrats! You pass check!"
+   fi
 fi

modulefiles/tasks/gaea/run_vx.local.lua

@@ -22,4 +22,6 @@ if (mode() == "unload") then
   unload(pathJoin("met", met_ver))
   unload(pathJoin("metplus",metplus_ver))
 end
-load("python_srw")
+load("conda")
+setenv("SRW_ENV", "srw_app")
+setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6")

modulefiles/tasks/orion/run_vx.local.lua

@@ -1,7 +1,7 @@
 --[[
 Compiler-specific modules are used for met and metplus libraries
 --]]
-load("build_orion_intel")
+--load("build_orion_intel")
 
 local met_ver = (os.getenv("met_ver") or "11.1.0")
 local metplus_ver = (os.getenv("metplus_ver") or "5.1.0")
@@ -27,4 +27,5 @@ if (mode() == "unload") then
 end
 --load("ufs-pyenv")
 load("stack-python/3.10.8")
-load("python_srw")
+load("conda")
+setenv("SRW_ENV", "srw_app")

parm/metplus/STATAnalysisConfig_skill_score

@@ -11,10 +11,19 @@
 //
 model = ["FV3_WoFS_v0_SUBCONUS_3km_test_mem000", "FV3_GFS_v16_SUBCONUS_3km"];
 
-fcst_lead = [ "6",
-              "6",
-              "6",
-              "6"
+fcst_lead = [ "6", "12",
+              "6", "12",
+              "6", "12",
+              "6", "12",
+                   "12",
+                   "12",
+                   "12",
+                   "12",
+                   "12",
+                   "12",
+                   "12",
+                   "12",
+                   "12"
             ];
 obs_lead  = [];
 
@@ -42,17 +51,35 @@ obs_init_inc    = [];
 obs_init_exc    = [];
 obs_init_hour   = [];
 
-fcst_var = [ "WIND",
-             "DPT",
-             "TMP",
-             "PRMSL"
+fcst_var = [ "PRMSL", "PRMSL",
+             "WIND",  "WIND",
+             "DPT",   "DPT",
+             "TMP",   "TMP",
+                      "WIND",
+                      "WIND",
+                      "WIND",
+                      "TMP",
+                      "TMP",
+                      "TMP",
+                      "SPFH",
+                      "SPFH",
+                      "SPFH"
            ];
 obs_var  = [];
 
-fcst_lev = [ "Z10",
-             "Z2",
-             "Z2",
-             "Z0"
+fcst_lev = [ "Z0",  "Z0",
+             "Z10", "Z10",
+             "Z2",  "Z2",
+             "Z2",  "Z2",
+                    "P250",
+                    "P400",
+                    "P850",
+                    "P250",
+                    "P400",
+                    "P850",
+                    "P300",
+                    "P500",
+                    "P850"
            ];
 obs_lev  = [];
 
@@ -74,10 +101,19 @@ line_type = [ "SL1L2" ];
 
 column    = [ "RMSE" ];
 
-weight    = [ 10.0,
-              10.0,
-              10.0,
-              10.0
+weight    = [ 10.0, 8.0,
+              10.0, 8.0,
+              10.0, 8.0,
+              10.0, 8.0,
+                    4.0,
+                    4.0,
+                    4.0,
+                    4.0,
+                    4.0,
+                    4.0,
+                    4.0,
+                    4.0,
+                    4.0
             ];
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -116,6 +152,6 @@ hss_ec_value   = NA;
 rank_corr_flag = FALSE;
 vif_flag       = FALSE;
 tmp_dir        = "/tmp";
-version        = "V10.1.1";
+version        = "V11.1.0";
 
 ////////////////////////////////////////////////////////////////////////////////

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml

@@ -12,7 +12,7 @@ workflow:
   PREDEF_GRID_NAME: SUBCONUS_Ind_3km
   DATE_FIRST_CYCL: '2019061500'
   DATE_LAST_CYCL: '2019061500'
-  FCST_LEN_HRS: 6
+  FCST_LEN_HRS: 12
   PREEXISTING_DIR_METHOD: rename
 rocoto:
   tasks: