From be6204135d8f6dbb247d72197a59c5115fd027f6 Mon Sep 17 00:00:00 2001 From: Seth Underwood Date: Thu, 7 Nov 2024 18:09:28 -0500 Subject: [PATCH] New iceberg_comb.sh --- postprocessing/iceberg_comb/iceberg_comb.sh | 285 +++++++++++++------- 1 file changed, 194 insertions(+), 91 deletions(-) diff --git a/postprocessing/iceberg_comb/iceberg_comb.sh b/postprocessing/iceberg_comb/iceberg_comb.sh index d123e223..8c14b995 100755 --- a/postprocessing/iceberg_comb/iceberg_comb.sh +++ b/postprocessing/iceberg_comb/iceberg_comb.sh @@ -1,5 +1,7 @@ #!/bin/sh +script_name=$(basename "$0") + EXIT_SUCCESS=0 EXIT_FAILURE=1 EXIT_NOTIBGS=255 @@ -28,11 +30,35 @@ help() { exit $EXIT_SUCCESS } +find_command() { + # Check if the command is usable at the given path. If not, check + # if a similar command exists in the user's PATH. + # + # Arguments: + # $1 : Standard command name + # $2 : Full path to command + local rtn="" + command -v $2 > /dev/null 2>&1 + if [ $? -eq 0 ] + then + rtn=$2 + else + # check for the command in PATH + command -v $1 > /dev/null 2>&1 + if [ $? -eq 0 ] + then + # found something in PATH + rtn=$(command -v $1) + fi + fi + echo "$rtn" +} + get_NumFilesInSet() { # Check for the global attribute "NumFilesInSet". If that attribute is # found, the function will return the number of files in this set. If not # found, the funcation will return -1. Get the output from ncdump - local output=$(ncdump -h $1) + local output=$($ncdump -h $1) local rtn=-1 # Return value if NumFilesInSet not in file numFilesInSet=$(echo $output | sed -n 's/.*:NumFilesInSet *= *\([0-9][0-9]*\) *;.*/\1/p') if [ ! x$numFilesInSet = "x" ] @@ -45,16 +71,16 @@ get_NumFilesInSet() { checkValidFileVersion() { # Check for the global attributes "file_format_{major,minor}_version", and # return a decimal separated version number. - local output=$(ncdump -h $1) + local output=$($ncdump -h $1) local ncdump_status=$? local rtn=$EXIT_NOTIBGS # Value if unable to get the version number local major_version=$(echo $output | sed -n 's/.*:file_format_major_version *= *\([0-9][0-9]*\) *;.*/\1/p') local minor_version=$(echo $output | sed -n 's/.*:file_format_minor_version *= *\([0-9][0-9]*\) *;.*/\1/p') if [ $ncdump_status -eq 0 ] then - if [ ( $major_version -gt $ICEBERGS_MAJOR_VERSION ) -o \ - ( $major_version -eq $ICEBERGS_MAJOR_VERSION -a \ - $minor_version -ge $ICEBERGS_MINOR_VERSION ) ] + if [ $major_version -gt $ICEBERGS_MAJOR_VERSION -o \ + $major_version -eq $ICEBERGS_MAJOR_VERSION -a \ + $minor_version -ge $ICEBERGS_MINOR_VERSION ] then rtn=$EXIT_SUCCESS else @@ -63,25 +89,29 @@ checkValidFileVersion() { else rtn=$EXIT_NOTIBGS fi - echo $rtn + return $rtn } checkIfIcebergs() { - # Get the output from ncdump - local output=$(ncdump -h $1) - local ncdump_status=$? - # A file is an icebergs restart file if the file contains the global - # attributes file_format_major_version, file_format_minor_version, and - # and NumFilesInSet - local validFileVersion=$(checkValidFileVersion $1) - local numFilesInSet=$(get_NumFilesInSet $1) - if [ $ncdump_status -ne 0 -a \ - $validFileVersion -eq 0 -a \ - $numFilesInSet -gt 0 ] + # Default return value + local rtn=$EXIT_NOTIBGS + # Use ncdump to determine if the file is a netcdf file + $ncdump -k $1 > /dev/null 2>&1 + if [ $? -eq $EXIT_SUCCESS ] then - rtn=$EXIT_SUCCESS - else - rtn=$EXIT_NOTIBGS + # A file is an icebergs restart file if the file contains the global + # attributes file_format_major_version, file_format_minor_version, and + # and NumFilesInSet + checkValidFileVersion $1 + local validFileVersion=$? + local numFilesInSet=$(get_NumFilesInSet $1) + if [ $validFileVersion -eq 0 -a \ + $numFilesInSet -gt 0 ] + then + rtn=$EXIT_SUCCESS + else + rtn=$EXIT_NOTIBGS + fi fi return $rtn } @@ -118,91 +148,164 @@ while getopts :chvV OPT; do done shift $((OPTIND-1)) -if [ $check -gt 0 -a $# -ge 1 ] +# Check that required commands are still available, if not available +# at the given path, fallback to the something in the path if +# available. +ncdump=$(find_command "ncdump" "${NCDUMP:-/Users/seth.underwood/opt/homebrew/bin/ncdump}") +if [ -z $ncdump ] then - checkIfIcebergs $1 - exit $? -elif [ $# -lt 2 ]; then - # Check that at least 2 arguments have been passed in - echoerr "ERROR: Not enough arguments given" - help_usage + echoerr "$script_name: Unable to find command 'ncdump'" exit $EXIT_FAILURE fi +if [ $verbose -gt 0 ] +then + echoerr "debug1: Using ncdump command '$ncdump'" +fi -# Verify the ncdump executable is in PATH and is executable -ncdumpPath=$( which ncdump 2>&1 ) -if [ $? -ne 0 ]; then - echo "ERROR: Required command 'ncdump' is not in PATH." +ncrcat=$(find_command "ncrcat" "${NCRCAT:-/Users/seth.underwood/opt/homebrew/bin/ncrcat}") +if [ -z $ncrcat ] +then + echoerr "$script_name: Unable to find command 'ncrcat'" + exit $EXIT_FAILURE +fi +if [ $verbose -gt 0 ] +then + echoerr "debug1: Using ncrcat command '$ncrcat'" +fi + +ncatted=$(find_command "ncatted" ${NCATTED:-/Users/seth.underwood/opt/homebrew/bin/ncatted}) +if [ -z $ncatted ] +then + echoerr "$script_name: Unable to find command 'ncatted'" exit $EXIT_FAILURE -else - if [ ! -x $ncdumpPath ]; then - echoerr "ERROR: Required command 'ncdump' is not executable ($ncdumpPath)." - exit 1 - fi +fi +if [ $verbose -gt 0 ] +then + echoerr "debug1: Using ncrcat command '$ncrcat'" fi -# Verify the ncrcat executable is in PATH and is executable (`ncrcat --version` exit status of 0) -which ncrcat > /dev/null 2>&1 -if [ $? -ne 0 ]; then - echoerr "ERROR: NCO command 'ncrcat' is not in PATH." - exit 1 -else - # Check that ncrcat is executable - ncrcat --version > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echoerr "ERROR: NCO command 'ncrcat' is not executable." - exit 1 +# The check option overrides the normal procedure. The user +# should only check one file at a time -- this simplifies the +# reporting +if [ $check -gt 0 ] +then + if [ $# -gt 1 ] + then + echoerr "$script_name: warning: Only checking the first file '$1'" + fi + if [ ! -e $1 ] + then + echoerr "$script_name: $1: No such file or directory" + exit $EXIT_FAILURE fi + checkIfIcebergs $1 + exit $? fi -# All of the input files ( [ ...] need to exist. -# must NOT exist, and should fail if the file exists (unless an -o option given?). +# The user should have supplied at least three items, at least +# two files to combine, and the output file. +if [ $# -lt 3 ] +then + echoerr "$script_name: Not enough arguments given" + exit $EXIT_FAILURE +fi -icebergFiles="" # list of files with icebergs -while [ $# -gt 0 ]; do - curFile=$1; shift - if [ $# = 0 ]; then - # This is the last file listed, and is the output file - # Exit if this file exists - if [[ -e $curFile ]]; then - echoerr "ERROR: out file '$curFile' exists. Refusing to overwrite file." - exit 1 - else - if [ $verbose -gt 1 ]; then - echoerr "Out file: $curFile" - fi - outFile=$curFile - fi +# All but the last argument should be valid icebergs restart files. +# The last argument file should not exist, and should be written +# to a directory the user can write to. +all_files="$@" +out_file=${all_files##* } +in_files=${all_files% *} +if [ $verbose -gt 1 ] +then + echoerr "debug2: Received $(echo "$in_files" | wc -w) input files" +fi + +file_check=$EXIT_SUCCESS +for f in $in_files +do + # If any of the files do not exist, or + if [ ! -e $f ] + then + echoerr "$script_name: $f: No such file or directory" + file_check=$EXIT_FAILURE else - # The in files must exist - if [[ ! -e $curFile ]]; then - echoerr "ERROR: in file '$curFile' does not exist." - exit 1 - else - ncdumpOut=$( ncdump -h $curFile 2>&1 ) - status=$? - if [ $status -ne 0 ]; then - echoerr "WARNING: skipping in file '$curFile' as it is NOT a NetCDF formatted file." - else - # Check each iceberg_res_file to see if it has icebergs. - if [ $( echo "$ncdumpOut" | grep 'UNLIMITED' | awk '{gsub(/\(/," ");print $6}' ) -gt 0 ]; then - if [ $verbose -gt 1 ]; then - echoerr "Using input file $curFile" - fi - icebergFiles="$icebergFiles $curFile" - fi - fi + checkIfIcebergs $f + if [ $? -ne $EXIT_SUCCESS ] + then + echoerr "$script_name: $f: Not an icebergs restart file" + file_check=$EXIT_FAILURE fi fi done - -# Collect the group of files that do have icebergs, and pass that to ncrcat -if [ "X$icebergFiles" = "X" ]; then - echoerr "No files to record concatenate." -else - ncrcat_cmd="ncrcat $icebergFiles $outFile" - if [ $verbose -gt 0 ]; then - echo $ncrcat_cmd +if [ $file_check -ne $EXIT_SUCCESS ] +then + echoerr "$script_name: Unable to combine iceberg files" + exit $EXIT_FAILURE +fi +# Check if the user supplied all the expected number of input files +# The number of files should match the global attribute NumFilesInSet +# We need to also check that all the files have the same value for +# NumFilesInSet. +# Since we are here, we already know that the files are valid +# iceberg restart files +file_check=$EXIT_SUCCESS +num_setFiles_0=$(get_NumFilesInSet $1) +if [ $verbose -gt 1 ] +then + echoerr "debug2: File '$1' expects $num_setFiles_0 files in set" +fi +for f in ${in_files#* } +do + num_setFiles_n=$(get_NumFilesInSet $f) + if [ $num_setFiles_0 -ne $num_setFiles_n ] + then + file_check=$EXIT_FAILURE fi - eval $ncrcat_cmd +done +if [ $file_check -ne $EXIT_SUCCESS ] +then + echoerr "$script_name: Incorrect or inconsistent number of files in set" + exit $EXIT_FAILURE +fi +# Verify that all the set files are given +if [ $num_setFiles_0 -ne $(echo "$in_files" | wc -w) ] +then + echoerr "$script_name: Expected $num_setFiles_0 files, but got $(echo "$in_files" | wc -w)" + exit $EXIT_FAILURE +fi + +# Check if the output file exists. +if [ -e $out_file ] +then + echoerr "$script_name: $f: Output file exists" + exit $EXIT_FAILURE +fi + + +# Run ncrcat on the files +ncrcat_cmd="$ncrcat $@" +if [ $verbose -gt 0 ] +then + echoerr "debug1: Running '$ncrcat_cmd'" fi +eval $ncrcat_cmd +if [ $? -ne 0 ] +then + echoerr "$script_name: Error during executing '$ncrcat_cmd'" + exit $EXIT_FAILURE +fi + +ncatted_cmd="$ncatted -a NumFilesInSet,global,d,, $out_file" +if [ $verbose -gt 0 ] +then + echoerr "debug1: Running '$ncatted_cmd'" +fi +eval $ncatted_cmd +if [ $? -ne 0 ] +then + echoerr "$script_name: Error during executing '$ncatted_cmd'" + exit $EXIT_FAILURE +fi + +exit $EXIT_SUCCESS