Skip to content

Commit

Permalink
New iceberg_comb.sh
Browse files Browse the repository at this point in the history
  • Loading branch information
@underwoo
underwoo committed Nov 7, 2024
1 parent 41c3a6f commit be62041
Showing 1 changed file with 194 additions and 91 deletions.
285 changes: 194 additions & 91 deletions postprocessing/iceberg_comb/iceberg_comb.sh
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/bin/sh

script_name=$(basename "$0")

EXIT_SUCCESS=0
EXIT_FAILURE=1
EXIT_NOTIBGS=255
Expand Down Expand Up @@ -28,11 +30,35 @@ help() {
exit $EXIT_SUCCESS
}

find_command() {
# Check if the command is usable at the given path. If not, check
# if a similar command exists in the user's PATH.
#
# Arguments:
# $1 : Standard command name
# $2 : Full path to command
local rtn=""
command -v $2 > /dev/null 2>&1
if [ $? -eq 0 ]
then
rtn=$2
else
# check for the command in PATH
command -v $1 > /dev/null 2>&1
if [ $? -eq 0 ]
then
# found something in PATH
rtn=$(command -v $1)
fi
fi
echo "$rtn"
}

get_NumFilesInSet() {
# Check for the global attribute "NumFilesInSet". If that attribute is
# found, the function will return the number of files in this set. If not
# found, the funcation will return -1. Get the output from ncdump
local output=$(ncdump -h $1)
local output=$($ncdump -h $1)
local rtn=-1 # Return value if NumFilesInSet not in file
numFilesInSet=$(echo $output | sed -n 's/.*:NumFilesInSet *= *\([0-9][0-9]*\) *;.*/\1/p')
if [ ! x$numFilesInSet = "x" ]
Expand All @@ -45,16 +71,16 @@ get_NumFilesInSet() {
checkValidFileVersion() {
# Check for the global attributes "file_format_{major,minor}_version", and
# return a decimal separated version number.
local output=$(ncdump -h $1)
local output=$($ncdump -h $1)
local ncdump_status=$?
local rtn=$EXIT_NOTIBGS # Value if unable to get the version number
local major_version=$(echo $output | sed -n 's/.*:file_format_major_version *= *\([0-9][0-9]*\) *;.*/\1/p')
local minor_version=$(echo $output | sed -n 's/.*:file_format_minor_version *= *\([0-9][0-9]*\) *;.*/\1/p')
if [ $ncdump_status -eq 0 ]
then
if [ ( $major_version -gt $ICEBERGS_MAJOR_VERSION ) -o \
( $major_version -eq $ICEBERGS_MAJOR_VERSION -a \
$minor_version -ge $ICEBERGS_MINOR_VERSION ) ]
if [ $major_version -gt $ICEBERGS_MAJOR_VERSION -o \
$major_version -eq $ICEBERGS_MAJOR_VERSION -a \
$minor_version -ge $ICEBERGS_MINOR_VERSION ]
then
rtn=$EXIT_SUCCESS
else
Expand All @@ -63,25 +89,29 @@ checkValidFileVersion() {
else
rtn=$EXIT_NOTIBGS
fi
echo $rtn
return $rtn
}

checkIfIcebergs() {
# Get the output from ncdump
local output=$(ncdump -h $1)
local ncdump_status=$?
# A file is an icebergs restart file if the file contains the global
# attributes file_format_major_version, file_format_minor_version, and
# and NumFilesInSet
local validFileVersion=$(checkValidFileVersion $1)
local numFilesInSet=$(get_NumFilesInSet $1)
if [ $ncdump_status -ne 0 -a \
$validFileVersion -eq 0 -a \
$numFilesInSet -gt 0 ]
# Default return value
local rtn=$EXIT_NOTIBGS
# Use ncdump to determine if the file is a netcdf file
$ncdump -k $1 > /dev/null 2>&1
if [ $? -eq $EXIT_SUCCESS ]
then
rtn=$EXIT_SUCCESS
else
rtn=$EXIT_NOTIBGS
# A file is an icebergs restart file if the file contains the global
# attributes file_format_major_version, file_format_minor_version, and
# and NumFilesInSet
checkValidFileVersion $1
local validFileVersion=$?
local numFilesInSet=$(get_NumFilesInSet $1)
if [ $validFileVersion -eq 0 -a \
$numFilesInSet -gt 0 ]
then
rtn=$EXIT_SUCCESS
else
rtn=$EXIT_NOTIBGS
fi
fi
return $rtn
}
Expand Down Expand Up @@ -118,91 +148,164 @@ while getopts :chvV OPT; do
done
shift $((OPTIND-1))

if [ $check -gt 0 -a $# -ge 1 ]
# Check that required commands are still available, if not available
# at the given path, fallback to the something in the path if
# available.
ncdump=$(find_command "ncdump" "${NCDUMP:-/Users/seth.underwood/opt/homebrew/bin/ncdump}")
if [ -z $ncdump ]
then
checkIfIcebergs $1
exit $?
elif [ $# -lt 2 ]; then
# Check that at least 2 arguments have been passed in
echoerr "ERROR: Not enough arguments given"
help_usage
echoerr "$script_name: Unable to find command 'ncdump'"
exit $EXIT_FAILURE
fi
if [ $verbose -gt 0 ]
then
echoerr "debug1: Using ncdump command '$ncdump'"
fi

# Verify the ncdump executable is in PATH and is executable
ncdumpPath=$( which ncdump 2>&1 )
if [ $? -ne 0 ]; then
echo "ERROR: Required command 'ncdump' is not in PATH."
ncrcat=$(find_command "ncrcat" "${NCRCAT:-/Users/seth.underwood/opt/homebrew/bin/ncrcat}")
if [ -z $ncrcat ]
then
echoerr "$script_name: Unable to find command 'ncrcat'"
exit $EXIT_FAILURE
fi
if [ $verbose -gt 0 ]
then
echoerr "debug1: Using ncrcat command '$ncrcat'"
fi

ncatted=$(find_command "ncatted" ${NCATTED:-/Users/seth.underwood/opt/homebrew/bin/ncatted})
if [ -z $ncatted ]
then
echoerr "$script_name: Unable to find command 'ncatted'"
exit $EXIT_FAILURE
else
if [ ! -x $ncdumpPath ]; then
echoerr "ERROR: Required command 'ncdump' is not executable ($ncdumpPath)."
exit 1
fi
fi
if [ $verbose -gt 0 ]
then
echoerr "debug1: Using ncrcat command '$ncrcat'"
fi

# Verify the ncrcat executable is in PATH and is executable (`ncrcat --version` exit status of 0)
which ncrcat > /dev/null 2>&1
if [ $? -ne 0 ]; then
echoerr "ERROR: NCO command 'ncrcat' is not in PATH."
exit 1
else
# Check that ncrcat is executable
ncrcat --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
echoerr "ERROR: NCO command 'ncrcat' is not executable."
exit 1
# The check option overrides the normal procedure. The user
# should only check one file at a time -- this simplifies the
# reporting
if [ $check -gt 0 ]
then
if [ $# -gt 1 ]
then
echoerr "$script_name: warning: Only checking the first file '$1'"
fi
if [ ! -e $1 ]
then
echoerr "$script_name: $1: No such file or directory"
exit $EXIT_FAILURE
fi
checkIfIcebergs $1
exit $?
fi

# All of the input files (<iceberg_res_file> [<iceberg_res_file> ...] need to exist.
# <iceberg_out_file> must NOT exist, and should fail if the file exists (unless an -o option given?).
# The user should have supplied at least three items, at least
# two files to combine, and the output file.
if [ $# -lt 3 ]
then
echoerr "$script_name: Not enough arguments given"
exit $EXIT_FAILURE
fi

icebergFiles="" # list of files with icebergs
while [ $# -gt 0 ]; do
curFile=$1; shift
if [ $# = 0 ]; then
# This is the last file listed, and is the output file
# Exit if this file exists
if [[ -e $curFile ]]; then
echoerr "ERROR: out file '$curFile' exists. Refusing to overwrite file."
exit 1
else
if [ $verbose -gt 1 ]; then
echoerr "Out file: $curFile"
fi
outFile=$curFile
fi
# All but the last argument should be valid icebergs restart files.
# The last argument file should not exist, and should be written
# to a directory the user can write to.
all_files="$@"
out_file=${all_files##* }
in_files=${all_files% *}
if [ $verbose -gt 1 ]
then
echoerr "debug2: Received $(echo "$in_files" | wc -w) input files"
fi

file_check=$EXIT_SUCCESS
for f in $in_files
do
# If any of the files do not exist, or
if [ ! -e $f ]
then
echoerr "$script_name: $f: No such file or directory"
file_check=$EXIT_FAILURE
else
# The in files must exist
if [[ ! -e $curFile ]]; then
echoerr "ERROR: in file '$curFile' does not exist."
exit 1
else
ncdumpOut=$( ncdump -h $curFile 2>&1 )
status=$?
if [ $status -ne 0 ]; then
echoerr "WARNING: skipping in file '$curFile' as it is NOT a NetCDF formatted file."
else
# Check each iceberg_res_file to see if it has icebergs.
if [ $( echo "$ncdumpOut" | grep 'UNLIMITED' | awk '{gsub(/\(/," ");print $6}' ) -gt 0 ]; then
if [ $verbose -gt 1 ]; then
echoerr "Using input file $curFile"
fi
icebergFiles="$icebergFiles $curFile"
fi
fi
checkIfIcebergs $f
if [ $? -ne $EXIT_SUCCESS ]
then
echoerr "$script_name: $f: Not an icebergs restart file"
file_check=$EXIT_FAILURE
fi
fi
done

# Collect the group of files that do have icebergs, and pass that to ncrcat
if [ "X$icebergFiles" = "X" ]; then
echoerr "No files to record concatenate."
else
ncrcat_cmd="ncrcat $icebergFiles $outFile"
if [ $verbose -gt 0 ]; then
echo $ncrcat_cmd
if [ $file_check -ne $EXIT_SUCCESS ]
then
echoerr "$script_name: Unable to combine iceberg files"
exit $EXIT_FAILURE
fi
# Check if the user supplied all the expected number of input files
# The number of files should match the global attribute NumFilesInSet
# We need to also check that all the files have the same value for
# NumFilesInSet.
# Since we are here, we already know that the files are valid
# iceberg restart files
file_check=$EXIT_SUCCESS
num_setFiles_0=$(get_NumFilesInSet $1)
if [ $verbose -gt 1 ]
then
echoerr "debug2: File '$1' expects $num_setFiles_0 files in set"
fi
for f in ${in_files#* }
do
num_setFiles_n=$(get_NumFilesInSet $f)
if [ $num_setFiles_0 -ne $num_setFiles_n ]
then
file_check=$EXIT_FAILURE
fi
eval $ncrcat_cmd
done
if [ $file_check -ne $EXIT_SUCCESS ]
then
echoerr "$script_name: Incorrect or inconsistent number of files in set"
exit $EXIT_FAILURE
fi
# Verify that all the set files are given
if [ $num_setFiles_0 -ne $(echo "$in_files" | wc -w) ]
then
echoerr "$script_name: Expected $num_setFiles_0 files, but got $(echo "$in_files" | wc -w)"
exit $EXIT_FAILURE
fi

# Check if the output file exists.
if [ -e $out_file ]
then
echoerr "$script_name: $f: Output file exists"
exit $EXIT_FAILURE
fi


# Run ncrcat on the files
ncrcat_cmd="$ncrcat $@"
if [ $verbose -gt 0 ]
then
echoerr "debug1: Running '$ncrcat_cmd'"
fi
eval $ncrcat_cmd
if [ $? -ne 0 ]
then
echoerr "$script_name: Error during executing '$ncrcat_cmd'"
exit $EXIT_FAILURE
fi

ncatted_cmd="$ncatted -a NumFilesInSet,global,d,, $out_file"
if [ $verbose -gt 0 ]
then
echoerr "debug1: Running '$ncatted_cmd'"
fi
eval $ncatted_cmd
if [ $? -ne 0 ]
then
echoerr "$script_name: Error during executing '$ncatted_cmd'"
exit $EXIT_FAILURE
fi

exit $EXIT_SUCCESS

0 comments on commit be62041

Please sign in to comment.