Skip to content

Commit

Permalink
Merge pull request #356 from LBL-EESA/deeplab_threads_support
Browse files Browse the repository at this point in the history 
Supporting thread num specification from the deeplab app
  • Loading branch information
@elbashandy
elbashandy authored May 8, 2020
2 parents 80f2e6d + d229bda commit bf7996f
Showing 1 changed file with 61 additions and 87 deletions.
148 changes: 61 additions & 87 deletions apps/teca_deeplabv3p_ar_detect.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,91 +14,60 @@ except ImportError:
def main():
# parse the command line
parser = argparse.ArgumentParser()
parser.add_argument(
'--input_file', type=str, required=False,
help='file path to the simulation to search for atmospheric rivers'
)
parser.add_argument('--input_file', type=str, required=False,
help='file path to the simulation to search for atmospheric rivers')

parser.add_argument(
'--input_regex', type=str, required=False,
help='regex matching simulation files to search for atmospheric rivers'
)
parser.add_argument('--input_regex', type=str, required=False,
help='regex matching simulation files to search for atmospheric rivers')

parser.add_argument(
'--output_file', type=str, required=False,
parser.add_argument('--output_file', type=str, required=False,
default=r'deeplabv3p_ar_detect_%t%.nc',
help=r'file pattern for output netcdf files (%%t%% is the time index)'
)
help=r'file pattern for output netcdf files (%%t%% is the time index)')

parser.add_argument(
'--ivt', type=str, default='IVT',
help='name of variable with integrated vapor transport (IVT)'
)
parser.add_argument('--ivt', type=str, default='IVT',
help='name of variable with integrated vapor transport (IVT)')

parser.add_argument(
'--pytorch_deeplab_model', type=str, required=False,
help='the pretrained deeplabv3plus model file'
)
parser.add_argument('--num_threads', type=int, default=-1,
help='number of threads used in torch for parallelizing CPU operations')

parser.add_argument(
'--pytorch_resnet_model', type=str, required=False,
help='the pretrained resnet model file'
)
parser.add_argument('--pytorch_deeplab_model', type=str, required=False,
help='the pretrained deeplabv3plus model file')

parser.add_argument(
'--t_axis_variable', type=str, required=False,
help='time dimension name'
)
parser.add_argument('--pytorch_resnet_model', type=str, required=False,
help='the pretrained resnet model file')

parser.add_argument(
'--t_calendar', type=str, required=False,
help='time calendar'
)
parser.add_argument('--t_axis_variable', type=str, required=False,
help='time dimension name')

parser.add_argument(
'--t_units', type=str, required=False,
help='time unit'
)
parser.add_argument('--t_calendar', type=str, required=False,
help='time calendar')

parser.add_argument(
'--filename_time_template', type=str, required=False,
help='filename time template'
)
parser.add_argument('--t_units', type=str, required=False,
help='time unit')

parser.add_argument(
'--compression_level', type=int, required=False,
help='the compression level used for each variable'
)
parser.add_argument('--filename_time_template', type=str, required=False,
help='filename time template')

parser.add_argument(
'--date_format', type=str, required=False,
help='the format for the date to write in the filename'
)
parser.add_argument('--compression_level', type=int, required=False,
help='the compression level used for each variable')

parser.add_argument(
'--first_step', type=int, required=False,
help='first time step to process'
)
parser.add_argument('--date_format', type=str, required=False,
help='the format for the date to write in the filename')

parser.add_argument(
'--last_step', type=int, required=False,
help='last time step to process'
)
parser.add_argument('--first_step', type=int, required=False,
help='first time step to process')

parser.add_argument(
'--steps_per_file', type=int, required=False,
help='number of time steps per output file'
)
parser.add_argument('--last_step', type=int, required=False,
help='last time step to process')

parser.add_argument(
'--start_date', type=str, required=False,
help='first time to proces in YYYY-MM-DD hh:mm:ss format'
)
parser.add_argument('--steps_per_file', type=int, required=False,
help='number of time steps per output file')

parser.add_argument(
'--end_date', type=str, required=False,
help='end time to proces in YYYY-MM-DD hh:mm:ss format'
)
parser.add_argument('--start_date', type=str, required=False,
help='first time to proces in YYYY-MM-DD hh:mm:ss format')

parser.add_argument('--end_date', type=str, required=False,
help='end time to proces in YYYY-MM-DD hh:mm:ss format')

args = parser.parse_args()

Expand All @@ -107,7 +76,7 @@ def main():
raise parser.error(
"missing file name or regex for simulation reader. "
"See --help for a list of command line options."
)
)

cf_reader = teca_cf_reader.New()

Expand All @@ -128,75 +97,80 @@ def main():
if args.input_file:
cf_reader.append_file_name(
args.input_file
)
)

if args.input_regex:
cf_reader.set_files_regex(
args.input_regex
)
)

if args.output_file:
cf_writer.set_file_name(
args.output_file
)
)

if args.ivt:
deeplabv3p_ar_detect.set_variable_name(
args.ivt
)
)

if args.num_threads:
deeplabv3p_ar_detect.set_num_threads(
args.num_threads
)

if args.pytorch_deeplab_model and args.pytorch_resnet_model:
deeplabv3p_ar_detect.build_model(
args.pytorch_deeplab_model,
args.pytorch_resnet_model
)
)
else:
deeplabv3p_ar_detect.build_model()

if args.t_axis_variable is not None:
cf_reader.set_t_axis_variable(
args.t_axis_variable
)
)

if args.t_calendar:
cf_reader.set_t_calendar(
args.t_calendar
)
)

if args.t_units:
cf_reader.set_t_units(
args.t_units
)
)

if args.filename_time_template:
cf_reader.set_filename_time_template(
args.filename_time_template
)
)

if args.compression_level:
cf_writer.set_compression_level(
args.compression_level
)
)

if args.date_format:
cf_writer.set_date_format(
args.date_format
)
)

if args.first_step:
cf_writer.set_first_step(
args.first_step
)
)

if args.last_step:
cf_writer.set_last_step(
args.last_step
)
)

if args.steps_per_file:
cf_writer.set_steps_per_file(
args.steps_per_file
)
)

# some minimal check for missing options
if (cf_reader.get_number_of_file_names() == 0 and
Expand All @@ -205,14 +179,14 @@ def main():
raise ValueError(
"missing file name or regex for simulation reader. "
"See --help for a list of command line options."
)
)

if not cf_writer.get_file_name():
if rank == 0:
raise ValueError(
"missing file name pattern for netcdf writer. "
"See --help for a list of command line options."
)
)

if args.start_date or args.end_date:

Expand All @@ -227,14 +201,14 @@ def main():
if args.start_date:
first_step = teca_coordinate.time_step_of(
time, True, calendar, units, args.start_date
)
)
cf_writer.set_first_step(first_step)

# and end date
if args.end_date:
last_step = teca_coordinate.time_step_of(
time, False, calendar, units, args.end_date
)
)
cf_writer.set_last_step(last_step)

# run the pipeline
Expand Down

0 comments on commit bf7996f

Please sign in to comment.