From b0224c3ddea987ac4922d474704ca2866432c42b Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Tue, 29 Aug 2023 10:59:01 +0100 Subject: [PATCH 01/15] fix path --- examples/desy1/desy1_input_pz.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/desy1/desy1_input_pz.yml b/examples/desy1/desy1_input_pz.yml index 1f34dfa50..f2c2a5b65 100644 --- a/examples/desy1/desy1_input_pz.yml +++ b/examples/desy1/desy1_input_pz.yml @@ -44,7 +44,7 @@ output_dir: data/desy1/outputs # configuration settings -config: desy1/config_input_pz.yml +config: examples/desy1/config_input_pz.yml # On NERSC, set this before running: # export DATA=${LSST}/groups/WL/users/zuntz/data/metacal-testbed From fc07ba1e3a61e8fc54d036003a582181e0c641f8 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Tue, 29 Aug 2023 11:01:49 +0100 Subject: [PATCH 02/15] add tool to update pipelines for next ceci version --- bin/update-all-for-ceci2.py | 32 ++++++++++++++ bin/update-pipeline-for-ceci-2.py | 71 +++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 bin/update-all-for-ceci2.py create mode 100755 bin/update-pipeline-for-ceci-2.py diff --git a/bin/update-all-for-ceci2.py b/bin/update-all-for-ceci2.py new file mode 100644 index 000000000..4bbc87576 --- /dev/null +++ b/bin/update-all-for-ceci2.py @@ -0,0 +1,32 @@ +import yaml +import os +import collections +groups = collections.defaultdict(list) + +for dirname in os.listdir("examples/"): + + for filename in os.listdir("examples/" + dirname): + if not (filename.endswith(".yaml") or filename.endswith(".yml")): + continue + + filename = "examples/" + dirname + "/" + filename + with open(filename) as f: + yaml_str = f.read() + info = yaml.safe_load(yaml_str) + + if "config" in info: + config = info["config"] + groups[config].append(filename) + + +for config_filename, group in groups.items(): + try: + with open(config_filename) as f: + yaml_str = f.read() + except FileNotFoundError: + print('# missing', config_filename) + continue + if not "alias" in yaml_str: + continue + + print("bin/update-pipeline-for-ceci-2.py", " ".join(group)) diff --git a/bin/update-pipeline-for-ceci-2.py b/bin/update-pipeline-for-ceci-2.py new file mode 100755 index 000000000..79b7c0ace --- /dev/null +++ b/bin/update-pipeline-for-ceci-2.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +import ruamel.yaml +import sys + + +def update_pipeline_file(pipeline_info, config_info): + for stage_info in pipeline_info["stages"]: + name = stage_info["name"] + + stage_config = config_info.get(name, None) + + if stage_config is None: + continue + + aliases = stage_config.get("aliases", None) + + if aliases is not None: + aliases = {k:v.strip() for k, v in aliases.items()} + stage_info["aliases"] = aliases + +def update_config_file(config_info): + for stage_info in config_info.values(): + stage_info.pop("aliases", None) + + +def main(pipeline_files): + # configure yaml - these are the approximate + # settings we have mostly used in TXPipe + yaml = ruamel.yaml.YAML() + yaml.indent(sequence=4, offset=4, mapping=4) + yaml.allow_duplicate_keys = True + + # Read all the pipeline files + pipeline_infos = [] + for pipeline_file in pipeline_files: + + with open(pipeline_file) as f: + yaml_str = f.read() + + pipeline_info = yaml.load(yaml_str) + config_file = pipeline_info["config"] + pipeline_infos.append(pipeline_info) + + # Check that all the pipeline files use the same config file + for pipeline_info in pipeline_infos: + if not pipeline_info["config"] == config_file: + raise ValueError("All pipeline files supplied to this script should use the same config file. Run the script multiple times on different files otherwise.") + + # Read the config file + with open(config_file) as f: + yaml_str = f.read() + config_info = yaml.load(yaml_str) + + # Update all the pipeline files. + for pipeline_info in pipeline_infos: + update_pipeline_file(pipeline_info, config_info) + + # Only now can we delete the alias information + update_config_file(config_info) + + # Update all the files in-place + for pipeline_file, pipeline_info in zip(pipeline_files, pipeline_infos): + with open(pipeline_file, "w") as f: + yaml.dump(pipeline_info, f) + + with open(config_file, "w") as f: + yaml.dump(config_info, f) + +if __name__ == "__main__": + input_files = sys.argv[1:] + main(input_files) From 3a853daa57e4210d5f79780fcfae7f39b0bcabd6 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Wed, 27 Sep 2023 09:25:08 +0100 Subject: [PATCH 03/15] remove scripts, moved to ceci --- bin/update-all-for-ceci2.py | 32 -------------- bin/update-pipeline-for-ceci-2.py | 71 ------------------------------- 2 files changed, 103 deletions(-) delete mode 100644 bin/update-all-for-ceci2.py delete mode 100755 bin/update-pipeline-for-ceci-2.py diff --git a/bin/update-all-for-ceci2.py b/bin/update-all-for-ceci2.py deleted file mode 100644 index 4bbc87576..000000000 --- a/bin/update-all-for-ceci2.py +++ /dev/null @@ -1,32 +0,0 @@ -import yaml -import os -import collections -groups = collections.defaultdict(list) - -for dirname in os.listdir("examples/"): - - for filename in os.listdir("examples/" + dirname): - if not (filename.endswith(".yaml") or filename.endswith(".yml")): - continue - - filename = "examples/" + dirname + "/" + filename - with open(filename) as f: - yaml_str = f.read() - info = yaml.safe_load(yaml_str) - - if "config" in info: - config = info["config"] - groups[config].append(filename) - - -for config_filename, group in groups.items(): - try: - with open(config_filename) as f: - yaml_str = f.read() - except FileNotFoundError: - print('# missing', config_filename) - continue - if not "alias" in yaml_str: - continue - - print("bin/update-pipeline-for-ceci-2.py", " ".join(group)) diff --git a/bin/update-pipeline-for-ceci-2.py b/bin/update-pipeline-for-ceci-2.py deleted file mode 100755 index 79b7c0ace..000000000 --- a/bin/update-pipeline-for-ceci-2.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -import ruamel.yaml -import sys - - -def update_pipeline_file(pipeline_info, config_info): - for stage_info in pipeline_info["stages"]: - name = stage_info["name"] - - stage_config = config_info.get(name, None) - - if stage_config is None: - continue - - aliases = stage_config.get("aliases", None) - - if aliases is not None: - aliases = {k:v.strip() for k, v in aliases.items()} - stage_info["aliases"] = aliases - -def update_config_file(config_info): - for stage_info in config_info.values(): - stage_info.pop("aliases", None) - - -def main(pipeline_files): - # configure yaml - these are the approximate - # settings we have mostly used in TXPipe - yaml = ruamel.yaml.YAML() - yaml.indent(sequence=4, offset=4, mapping=4) - yaml.allow_duplicate_keys = True - - # Read all the pipeline files - pipeline_infos = [] - for pipeline_file in pipeline_files: - - with open(pipeline_file) as f: - yaml_str = f.read() - - pipeline_info = yaml.load(yaml_str) - config_file = pipeline_info["config"] - pipeline_infos.append(pipeline_info) - - # Check that all the pipeline files use the same config file - for pipeline_info in pipeline_infos: - if not pipeline_info["config"] == config_file: - raise ValueError("All pipeline files supplied to this script should use the same config file. Run the script multiple times on different files otherwise.") - - # Read the config file - with open(config_file) as f: - yaml_str = f.read() - config_info = yaml.load(yaml_str) - - # Update all the pipeline files. - for pipeline_info in pipeline_infos: - update_pipeline_file(pipeline_info, config_info) - - # Only now can we delete the alias information - update_config_file(config_info) - - # Update all the files in-place - for pipeline_file, pipeline_info in zip(pipeline_files, pipeline_infos): - with open(pipeline_file, "w") as f: - yaml.dump(pipeline_info, f) - - with open(config_file, "w") as f: - yaml.dump(config_info, f) - -if __name__ == "__main__": - input_files = sys.argv[1:] - main(input_files) From 4fde136b8f8a6ea65b6b9e58fc2653baa990c850 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Thu, 8 Aug 2024 14:44:53 +0100 Subject: [PATCH 04/15] new rail versions --- bin/install.sh | 38 +++++++++++++++---------------- environment-nopip.yml | 33 +++++++++++++++++++++++++++ environment-piponly.yml | 27 ++++++++++++++++++++++ environment.yml | 50 ----------------------------------------- 4 files changed, 78 insertions(+), 70 deletions(-) create mode 100644 environment-nopip.yml create mode 100644 environment-piponly.yml delete mode 100644 environment.yml diff --git a/bin/install.sh b/bin/install.sh index 8e3347c67..4a11dd4bf 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -12,20 +12,16 @@ then fi # Figure out operating system details -OS=$(uname -s) -if [ "$OS" = "Darwin" ] -then - OS=MacOSX - CHIPSET=$(uname -m) -else - CHIPSET=x86_64 -fi +INSTALLER_VERSION=24.3.0-0 export SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True + + # URL to download -URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-3/Mambaforge-23.1.0-3-${OS}-${CHIPSET}.sh" +URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" + # Download and run the conda installer Miniforge conda installer echo "Downloading conda installer from $URL" wget -O Mambaforge3.sh $URL @@ -33,17 +29,19 @@ chmod +x Mambaforge3.sh ./Mambaforge3.sh -b -p ./conda source ./conda/bin/activate -# conda-installable stuff -mamba env update --file environment.yml - - -if [[ "$CHIPSET" = "arm64" || "$CHIPSET" = "aarch64" ]] -then - echo "Pymaster cannot be correctly conda- or pip-installed on Apple Silicon yet, so we are skipping it." - echo "The twopoint fourier and some covariance stage(s) will not work" -else - mamba install -c conda-forge namaster -fi +# It seems to be a bug on conda that if our required pip-installable +# packages depend on the conda-forge packages they don't detect them +# properly if they are all in the same environment file. So we need to +# split them up +mamba env update --file environment-nopip.yml +mamba env update --file environment-piponly.yml + +# We do this to get around a bug in the healpy installation +# where it installs its own copy of libomp instead of using +# the shared one. +cat > ./conda/etc/conda/activate.d/libomp_healpy_workaround.sh <=3=mpi_mpich_* + - healpy=1.16.* + - healsparse=1.10.* + - jax=0.4.27 + - jaxlib=0.4.23 + - jupyter=1.0.* + - matplotlib=3.8.* + - mpi4py>=3 + - mpich>=4.* + - namaster=2.0.* + - numpy=1.26.* + - pandas=2.2.* + - psutil=5.9.* + - pyccl=3.0.* + - python=3.10.* + - scikit-learn=1.5.* + - scipy=1.13.* + - tables-io-full=0.9.* + - threadpoolctl=3.5.* + - tjpcov=0.4.* + - treecorr=5.0.* + - pip=24.0.* + - pygraphviz=1.13.* diff --git a/environment-piponly.yml b/environment-piponly.yml new file mode 100644 index 000000000..765f617a5 --- /dev/null +++ b/environment-piponly.yml @@ -0,0 +1,27 @@ +dependencies: + - pip: + - ceci==2.0.1 + - git+https://github.com/jlvdb/hyperbolic@b88b107a291fa16c2006cf971ce610248d58e94c + - dask-mpi>=2022.4.0 + - git+https://github.com/LSSTDESC/CLMM.git@1.12.5 + - parallel-statistics==0.13 + - sacc[all]==0.14 + - jax==0.4.27 + - jaxlib==0.4.23 + - glass==2023.7 + - git+https://github.com/beckermr/hybrideb@a3198ea5bc8175542b058e5c1289d910f05fd99d + # we must specifically request versions of rail algorithms + # to ensure that we get the ceci v2 versions + - pz-rail[algos] @ git+https://github.com/LSSTDESC/RAIL@v1.0.1 + - pz-rail-astro-tools==1.0.4 + - pz-rail-base==1.0.4 + - pz-rail-bpz==1.0.1 + - pz-rail-cmnn==1.0.0 # no release for ceci2 yet + - pz-rail-dsps==0.0.3 # no pypi release for more than 0.3 + - pz-rail-flexzboost==1.0.1 + - pz-rail-fsps==1.0.1 + - pz-rail-gpz-v1==1.0.2 + - pz-rail-lephare==0.2 # no release for ceci2 yet - cannot use + - pz-rail-pzflow==1.0.1 + - pz-rail-sklearn==1.0.1 + - pz-rail-som==1.0.0 # no release for ceci2 yet - cannot use diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 40f1dbbfe..000000000 --- a/environment.yml +++ /dev/null @@ -1,50 +0,0 @@ -channels: - - conda-forge -dependencies: - - astropy=5.2.* - - camb=1.4.* - - cosmosis=2.4.1 - - dask=2023.4.1 - - dm-tree=0.1.7 - - firecrown=1.4.0 - - fitsio=1.1.8 - - h5py=3.8.*=mpi_mpich_* - - healpy=1.16.* - - healsparse=1.6.* - - jax=0.4.8 - - jupyter=1.0.* - - matplotlib=3.7.* - - mpi4py=3.1.* - - mpich=4.1.* - - numpy=1.24.* - - pandas=1.5.* - - psutil=5.9.* - - pyccl=2.6.* - - python=3.10.* - - scikit-learn=1.2.* - - scipy=1.10.* - - tables-io-full=0.8.* - - threadpoolctl=3.1.* - - treecorr=4.3.* - - pip - - pip: - - ceci==1.13 - - git+https://github.com/jlvdb/hyperbolic@b88b107a291fa16c2006cf971ce610248d58e94c - - git+https://github.com/joezuntz/dask-mpi@31e35759cfb03780a12ad6ffe2b82279b0fab442 - - git+https://github.com/LSSTDESC/CLMM.git@1.8.0 - - healpix==2023.1.13 - - parallel-statistics==0.13 - - sacc==0.8 - - tjpcov==0.3.0 - - pz-rail[dev]==0.99.1 - - pz-rail-bpz==0.1.0 - - pz-rail-astro-tools==0.0.2 - - pz-rail-base==0.0.4 - - pz-rail-dsps==0.0.2 - - pz-rail-flexzboost==0.1.1 - - pz-rail-fsps==0.0.2 - - pz-rail-gpz-v1==0.1.2 - - pz-rail-pipelines==0.1.0 - - pz-rail-pzflow==0.0.1 - - pz-rail-sklearn==0.0.1 - - pz-rail-som==0.0.2 From 6e395aa180114db19babf63266ed1d1abbc663a7 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Thu, 8 Aug 2024 14:56:42 +0100 Subject: [PATCH 05/15] Merge rail-tomography --- examples/2.2i/config.yml | 63 ++---- examples/2.2i/pipeline.yml | 164 ++++++++------- examples/2.2i/pipeline_1tract.yml | 74 +++---- examples/clmm/config.yml | 13 +- examples/clmm/cosmodc2.yml | 20 +- examples/clmm/pipeline.yml | 25 ++- .../CLClusterBinning-20deg2-CL.yml | 9 +- .../CLClusterEnsemble-20deg2-CL.yml | 14 +- .../CLClusterShearCat-20deg2-CL.yml | 12 +- .../Cluster_pipelines/config-1deg2-CL.yml | 45 ++-- .../Cluster_pipelines/config-20deg2-CL.yml | 51 ++--- .../pipeline-1deg2-CL-in2p3.yml | 21 +- .../pipeline-1deg2-CL-nersc.yml | 23 +- .../pipeline-20deg2-CL-in2p3.yml | 40 ++-- .../pipeline-20deg2-CL-nersc.yml | 44 ++-- examples/cosmodc2/config-1deg2-CL.yml | 45 ++-- examples/cosmodc2/config-20deg2-CL.yml | 51 ++--- examples/cosmodc2/config.yml | 93 ++++----- examples/cosmodc2/pipeline-1deg2-CL-in2p3.yml | 21 +- .../cosmodc2/pipeline-20deg2-CL-in2p3.yml | 21 +- examples/cosmodc2/pipeline.yml | 152 ++++++++------ examples/cosmodc2/pipeline_redmagic.yml | 139 ++++++------ examples/cosmodc2/pipeline_redmagic_full.yml | 151 +++++++------- examples/dp0.2/config.yml | 49 +---- examples/dp0.2/pipeline.yml | 129 +++++++----- examples/hscy3/config.yml | 197 ++++++------------ examples/hscy3/hsc_pipeline.yaml | 99 +++++---- examples/lensfit/config.yml | 83 ++------ examples/lensfit/pipeline.yml | 139 +++++++----- examples/metacal/config.yml | 129 ++++-------- examples/metacal/pipeline.yml | 173 +++++++++------ examples/metadetect/config.yml | 141 ++++--------- examples/metadetect/pipeline.yml | 191 ++++++++++------- examples/metadetect_source_only/config.yml | 82 ++------ examples/metadetect_source_only/pipeline.yml | 93 +++++---- examples/redmagic/config.yml | 73 +++---- examples/redmagic/pipeline.yml | 108 +++++----- examples/redmagic/pipeline_complete.yml | 91 ++++---- examples/skysim/config.yml | 76 +++---- examples/skysim/pipeline.yml | 160 +++++++------- examples/ssi/config_mag.yml | 34 +-- examples/ssi/pipeline_mag.yml | 48 +++-- txpipe/source_selector.py | 5 +- 43 files changed, 1614 insertions(+), 1777 deletions(-) diff --git a/examples/2.2i/config.yml b/examples/2.2i/config.yml index 4a6579c11..2953327d2 100644 --- a/examples/2.2i/config.yml +++ b/examples/2.2i/config.yml @@ -6,7 +6,7 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.2i_dr6c_with_metacal TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXIngestStars: cat_name: dc2_object_run2.2i_dr6c @@ -15,12 +15,12 @@ TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal FlexZPipe: - has_redshift: False + has_redshift: false nz: 300 - metacal_fluxes: False + metacal_fluxes: false chunk_rows: 100 @@ -32,24 +32,10 @@ PZPDFMLZ: TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: source_nz - TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: lens_nz - - TXTruePhotozStackSource: name: TXTruePhotozStackSource - aliases: - tomography_catalog: shear_tomography_catalog - catalog: shear_catalog - weights_catalog: shear_catalog - photoz_stack: shear_photoz_stack weight_col: metacal/weight redshift_group: metacal zmax: 2.0 @@ -57,11 +43,6 @@ TXTruePhotozStackSource: TXTruePhotozStackLens: name: TXTruePhotozStackLens - aliases: - tomography_catalog: lens_tomography_catalog - catalog: photometry_catalog - weights_catalog: dummy - photoz_stack: lens_photoz_stack redshift_group: photometry zmax: 2.0 nz: 201 @@ -73,7 +54,7 @@ TXSourceTrueNumberDensity: chunk_rows: 100000 TXSourceSelector: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -98,36 +79,36 @@ TXRandomCat: TXTwoPoint: binslop: 0.1 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: False # False now seems to be right for metacal + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: false # False now seems to be right for metacal min_sep: 2.5 max_sep: 250 nbins: 20 verbose: 0 - subtract_mean_shear: True + subtract_mean_shear: true TXSourceMaps: pixelization: healpix nside: 1024 - sparse: True + sparse: true TXLensMaps: pixelization: healpix nside: 1024 - sparse: True + sparse: true TXAuxiliarySourceMaps: - chunk_rows: 100000 - sparse: True - psf_prefix: psf_ + chunk_rows: 100000 + sparse: true + psf_prefix: psf_ TXAuxiliaryLensMaps: - chunk_rows: 100000 - sparse: True - bright_obj_threshold: 22.0 + chunk_rows: 100000 + sparse: true + bright_obj_threshold: 22.0 TXSimpleMask: depth_cut: 23.0 @@ -140,8 +121,8 @@ TXLensDiagnosticPlots: {} TXTwoPointFourier: chunk_rows: 100000 - flip_g2: True - flip_g1: True + flip_g2: true + flip_g1: true bandwidth: 200 apodization_size: 0.0 cache_dir: ./cache @@ -156,7 +137,7 @@ TXLensingNoiseMaps: TXMeanLensSelector: # Mag cuts chunk_rows: 100000 - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -168,7 +149,7 @@ TXMeanLensSelector: TXTruthLensSelector: # Mag cuts chunk_rows: 100000 - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 diff --git a/examples/2.2i/pipeline.yml b/examples/2.2i/pipeline.yml index e6f6c947a..54092ccda 100644 --- a/examples/2.2i/pipeline.yml +++ b/examples/2.2i/pipeline.yml @@ -17,87 +17,103 @@ modules: txpipe # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/FlexZPipe + - submodules/WLMassMap/python/desc/ + - submodules/FlexZPipe stages: - - name: TXJackknifeCenters - - name: TXSourceNoiseMaps - nprocess: 32 - - name: TXLensNoiseMaps - nprocess: 32 - - name: TXSourceSelector - nprocess: 32 - - name: TXShearCalibration - nodes: 1 - nprocess: 8 - - name: TXTruthLensCatalogSplitter - nodes: 1 - nprocess: 8 - - name: TXTruthLensSelector - nprocess: 32 - - name: TXTruePhotozStackSource - classname: TXTruePhotozStack - nprocess: 32 - nodes: 2 - - name: TXTruePhotozStackLens - classname: TXTruePhotozStack - nprocess: 32 - nodes: 2 - - name: TXTwoPointTheoryReal - nprocess: 1 - threads_per_process: 8 - - name: TXPhotozPlotSource - classname: TXPhotozPlot - nprocess: 8 - - name: TXPhotozPlotLens - classname: TXPhotozPlot - nprocess: 8 - - name: TXSourceMaps - nprocess: 16 - nodes: 2 - - name: TXLensMaps - nprocess: 16 - nodes: 2 - - name: TXAuxiliarySourceMaps - nprocess: 16 - nodes: 2 - - name: TXAuxiliaryLensMaps - nprocess: 16 - nodes: 2 - - name: TXSimpleMask - - name: TXLSSWeightsUnit - - name: TXDensityMaps - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXRandomCat - nprocess: 32 - nodes: 2 - - name: TXTwoPoint - nprocess: 2 - nodes: 2 - threads_per_process: 32 - - name: TXNullBlinding - threads_per_process: 32 + - name: TXJackknifeCenters + - name: TXSourceNoiseMaps + nprocess: 32 + - name: TXLensNoiseMaps + nprocess: 32 + - name: TXSourceSelector + nprocess: 32 + - name: TXShearCalibration + nodes: 1 + nprocess: 8 + - name: TXTruthLensCatalogSplitter + nodes: 1 + nprocess: 8 + - name: TXTruthLensSelector + nprocess: 32 + - name: TXTruePhotozStackSource + classname: TXTruePhotozStack + nprocess: 32 + nodes: 2 + aliases: + tomography_catalog: shear_tomography_catalog + catalog: shear_catalog + weights_catalog: shear_catalog + photoz_stack: shear_photoz_stack + - name: TXTruePhotozStackLens + classname: TXTruePhotozStack + nprocess: 32 + nodes: 2 + aliases: + tomography_catalog: lens_tomography_catalog + catalog: photometry_catalog + weights_catalog: dummy + photoz_stack: lens_photoz_stack + - name: TXTwoPointTheoryReal + nprocess: 1 + threads_per_process: 8 + - name: TXPhotozPlotSource + classname: TXPhotozPlot + nprocess: 8 + aliases: + photoz_stack: shear_photoz_stack + nz_plot: lens_nz + - name: TXPhotozPlotLens + classname: TXPhotozPlot + nprocess: 8 + aliases: + photoz_stack: lens_photoz_stack + nz_plot: source_nz + - name: TXSourceMaps + nprocess: 16 + nodes: 2 + - name: TXLensMaps + nprocess: 16 + nodes: 2 + - name: TXAuxiliarySourceMaps + nprocess: 16 + nodes: 2 + - name: TXAuxiliaryLensMaps + nprocess: 16 + nodes: 2 + - name: TXSimpleMask + - name: TXLSSWeightsUnit + - name: TXDensityMaps + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXRandomCat + nprocess: 32 + nodes: 2 + - name: TXTwoPoint + nprocess: 2 + nodes: 2 + threads_per_process: 32 + - name: TXNullBlinding + threads_per_process: 32 # - name: TXTwoPointFourier # nprocess: 2 # nodes: 2 # threads_per_process: 32 - - name: TXTwoPointPlots - - name: TXSourceDiagnosticPlots - nprocess: 16 - - name: TXLensDiagnosticPlots - nprocess: 16 - - name: TXStarCatalogSplitter - nprocess: 1 + - name: TXTwoPointPlots + - name: TXSourceDiagnosticPlots + nprocess: 16 + - name: TXLensDiagnosticPlots + nprocess: 16 + - name: TXStarCatalogSplitter + nprocess: 1 # - name: TXGammaTStars # threads_per_process: 32 - - name: TXBrighterFatterPlot - - name: TXRoweStatistics - - name: TXPSFDiagnostics - - name: TXConvergenceMaps - threads_per_process: 32 - - name: TXConvergenceMapPlots + - name: TXBrighterFatterPlot + - name: TXRoweStatistics + - name: TXPSFDiagnostics + - name: TXConvergenceMaps + threads_per_process: 32 + - name: TXConvergenceMapPlots # - name: TXMapCorrelations output_dir: data/2.2i_dr6/outputs @@ -119,7 +135,7 @@ inputs: exposures: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/2.2i-inputs/exposures.hdf5 dummy: none -resume: True +resume: true log_dir: data/2.2i_dr6/logs pipeline_log: data/2.2i_dr6/log.txt diff --git a/examples/2.2i/pipeline_1tract.yml b/examples/2.2i/pipeline_1tract.yml index a58fcfe47..7427202dc 100644 --- a/examples/2.2i/pipeline_1tract.yml +++ b/examples/2.2i/pipeline_1tract.yml @@ -17,44 +17,44 @@ modules: txpipe # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/FlexZPipe + - submodules/WLMassMap/python/desc/ + - submodules/FlexZPipe stages: - - name: TXJackknifeCenters - - name: TXSourceNoiseMaps - - name: TXLensNoiseMaps - - name: TXSourceSelector - - name: PZRailEstimateSource - - name: PZRailEstimateLensFromSource - - name: TXShearCalibration - - name: TXTruthLensSelector - - name: TXLensCatalogSplitter - - name: TXPhotozSourceStack - classname: TXPhotozStack - - name: TXPhotozLensStack - classname: TXPhotozStack - - name: TXPhotozPlot - - name: TXSourceMaps - - name: TXLensMaps - - name: TXAuxiliarySourceMaps - - name: TXAuxiliaryLensMaps - - name: TXSimpleMask - - name: TXDensityMaps - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXRandomCat - - name: TXTwoPoint - - name: TXNullBlinding - - name: TXTwoPointTheoryReal - - name: TXStarCatalogSplitter - - name: TXTwoPointPlots - - name: TXSourceDiagnosticPlots - - name: TXLensDiagnosticPlots - - name: TXGammaTStars - - name: TXBrighterFatterPlot - - name: TXRoweStatistics - - name: TXPSFDiagnostics + - name: TXJackknifeCenters + - name: TXSourceNoiseMaps + - name: TXLensNoiseMaps + - name: TXSourceSelector + - name: PZRailEstimateSource + - name: PZRailEstimateLensFromSource + - name: TXShearCalibration + - name: TXTruthLensSelector + - name: TXLensCatalogSplitter + - name: TXPhotozSourceStack + classname: TXPhotozStack + - name: TXPhotozLensStack + classname: TXPhotozStack + - name: TXPhotozPlot + - name: TXSourceMaps + - name: TXLensMaps + - name: TXAuxiliarySourceMaps + - name: TXAuxiliaryLensMaps + - name: TXSimpleMask + - name: TXDensityMaps + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXRandomCat + - name: TXTwoPoint + - name: TXNullBlinding + - name: TXTwoPointTheoryReal + - name: TXStarCatalogSplitter + - name: TXTwoPointPlots + - name: TXSourceDiagnosticPlots + - name: TXLensDiagnosticPlots + - name: TXGammaTStars + - name: TXBrighterFatterPlot + - name: TXRoweStatistics + - name: TXPSFDiagnostics output_dir: data/2.2i-single-tract/outputs config: examples/2.2i/config.yml @@ -67,7 +67,7 @@ inputs: calibration_table: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/2.2i-t3828-inputs/sample_cosmodc2_w10year_errors.dat star_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/2.2i-t3828-inputs/star_catalog.hdf5 -resume: True +resume: true log_dir: data/2.2i-single-tract/logs pipeline_log: data/2.2i-single-tract/log.txt diff --git a/examples/clmm/config.yml b/examples/clmm/config.yml index b6bd94c49..5ccd63288 100755 --- a/examples/clmm/config.yml +++ b/examples/clmm/config.yml @@ -4,9 +4,6 @@ global: PZPrepareEstimatorSource: name: PZPrepareEstimatorSource classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: source_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -19,7 +16,7 @@ PZPrepareEstimatorSource: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01] @@ -28,10 +25,6 @@ PZPrepareEstimatorSource: PZEstimatorSource: name: PZEstimatorSource classname: BPZliteEstimator - aliases: - model: source_photoz_model - input: shear_catalog - output: source_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -59,14 +52,14 @@ PZEstimatorSource: TXSourceSelectorMetadetect: # change to False to use realistic selection based on riz - true_z: True + true_z: true bands: riz # used for selection T_cut: 0.5 s2n_cut: 10.0 max_rows: 1000 delta_gamma: 0.02 source_zbin_edges: [0.1, 3.0] - shear_prefix: "" + shear_prefix: '' # No options here diff --git a/examples/clmm/cosmodc2.yml b/examples/clmm/cosmodc2.yml index 96b8d50b0..93130dac5 100755 --- a/examples/clmm/cosmodc2.yml +++ b/examples/clmm/cosmodc2.yml @@ -10,18 +10,18 @@ site: modules: txpipe rail python_paths: - - submodules/RAIL + - submodules/RAIL stages: - - name: BPZ_lite # Run BPZ to get photo-z PDFs - nodes: 1 - nprocess: 32 - processes_per_node: 32 - threads_per_process: 1 - - name: TXSourceSelectorMetadetect # Select a source sample - - name: CLIngestRedmapper # Ingest redmapper catalog from GCR - - name: CLClusterShearCatalogs # Find shear catalogs around each cluster + - name: BPZ_lite # Run BPZ to get photo-z PDFs + nodes: 1 + nprocess: 32 + processes_per_node: 32 + threads_per_process: 1 + - name: TXSourceSelectorMetadetect # Select a source sample + - name: CLIngestRedmapper # Ingest redmapper catalog from GCR + - name: CLClusterShearCatalogs # Find shear catalogs around each cluster output_dir: data/clmm/outputs @@ -37,7 +37,7 @@ inputs: calibration_table: data/example/inputs/sample_cosmodc2_w10year_errors.dat -resume: True +resume: true log_dir: data/clmm/logs pipeline_log: data/clmm/log.txt diff --git a/examples/clmm/pipeline.yml b/examples/clmm/pipeline.yml index 07502f3c6..fabee077a 100755 --- a/examples/clmm/pipeline.yml +++ b/examples/clmm/pipeline.yml @@ -11,17 +11,24 @@ site: modules: txpipe rail.stages python_paths: - - submodules/RAIL + - submodules/RAIL stages: - - name: PZPrepareEstimatorSource # Prepare the p(z) estimator - classname: BPZliteInformer - - name: PZEstimatorSource # Measure lens galaxy PDFs - classname: BPZliteEstimator - threads_per_process: 1 - - name: TXSourceSelectorMetadetect # Select a source sample - - name: CLClusterShearCatalogs # Find shear catalogs around each cluster + - name: PZPrepareEstimatorSource # Prepare the p(z) estimator + classname: BPZliteInformer + aliases: + input: spectroscopic_catalog + model: source_photoz_model + - name: PZEstimatorSource # Measure lens galaxy PDFs + classname: BPZliteEstimator + threads_per_process: 1 + aliases: + model: source_photoz_model + input: shear_catalog + output: source_photoz_pdfs + - name: TXSourceSelectorMetadetect # Select a source sample + - name: CLClusterShearCatalogs # Find shear catalogs around each cluster output_dir: data/clmm/outputs @@ -34,7 +41,7 @@ inputs: calibration_table: data/example/inputs/sample_cosmodc2_w10year_errors.dat spectroscopic_catalog: data/example/inputs/mock_spectroscopic_catalog.hdf5 -resume: True +resume: true log_dir: data/clmm/logs pipeline_log: data/clmm/log.txt diff --git a/examples/cosmodc2/Cluster_pipelines/CLClusterBinning-20deg2-CL.yml b/examples/cosmodc2/Cluster_pipelines/CLClusterBinning-20deg2-CL.yml index 9edfacc3f..3d1013ea3 100644 --- a/examples/cosmodc2/Cluster_pipelines/CLClusterBinning-20deg2-CL.yml +++ b/examples/cosmodc2/Cluster_pipelines/CLClusterBinning-20deg2-CL.yml @@ -2,7 +2,7 @@ #for CCin2p3 site: name: cc-parallel - mpi_command: "mpirun -n" + mpi_command: mpirun -n #for NERSC #site: @@ -27,8 +27,8 @@ stages: # nprocess: 1 # - name: BPZ_lite # nprocess: 30 - - name: CLClusterBinningRedshiftRichness - nprocess: 1 + - name: CLClusterBinningRedshiftRichness + nprocess: 1 # - name: CLClusterShearCatalogs # nprocess: 1 #>1 does not work with mpi # - name: CLClusterEnsembleProfiles @@ -51,8 +51,7 @@ inputs: cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5 #shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 #source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 - -resume: True +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_20deg2.txt diff --git a/examples/cosmodc2/Cluster_pipelines/CLClusterEnsemble-20deg2-CL.yml b/examples/cosmodc2/Cluster_pipelines/CLClusterEnsemble-20deg2-CL.yml index 76bd75719..f90ac527b 100644 --- a/examples/cosmodc2/Cluster_pipelines/CLClusterEnsemble-20deg2-CL.yml +++ b/examples/cosmodc2/Cluster_pipelines/CLClusterEnsemble-20deg2-CL.yml @@ -2,7 +2,7 @@ #for CCin2p3 site: name: cc-parallel - mpi_command: "mpirun -n" + mpi_command: mpirun -n #for NERSC #site: @@ -31,8 +31,8 @@ stages: # nprocess: 1 # - name: CLClusterShearCatalogs # nprocess: 1 #>1 does not work with mpi - - name: CLClusterEnsembleProfiles - nprocess: 1 + - name: CLClusterEnsembleProfiles + nprocess: 1 # - name: CLClusterDataVector # nprocess: 1 @@ -51,10 +51,10 @@ inputs: #cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5 #shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 #source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 - cluster_catalog_tomography : ./data/cosmodc2/20deg2/cluster_catalog_tomography.hdf5 - cluster_shear_catalogs : ./data/cosmodc2/20deg2/cluster_shear_catalogs.hdf5 - -resume: True + cluster_catalog_tomography: ./data/cosmodc2/20deg2/cluster_catalog_tomography.hdf5 + cluster_shear_catalogs: ./data/cosmodc2/20deg2/cluster_shear_catalogs.hdf5 + +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_20deg2.txt diff --git a/examples/cosmodc2/Cluster_pipelines/CLClusterShearCat-20deg2-CL.yml b/examples/cosmodc2/Cluster_pipelines/CLClusterShearCat-20deg2-CL.yml index 880d33073..53ad65c7a 100644 --- a/examples/cosmodc2/Cluster_pipelines/CLClusterShearCat-20deg2-CL.yml +++ b/examples/cosmodc2/Cluster_pipelines/CLClusterShearCat-20deg2-CL.yml @@ -2,7 +2,7 @@ #for CCin2p3 site: name: cc-parallel - mpi_command: "mpirun -n" + mpi_command: mpirun -n #for NERSC #site: @@ -29,8 +29,8 @@ stages: # nprocess: 30 # - name: CLClusterBinningRedshiftRichness # nprocess: 1 - - name: CLClusterShearCatalogs - nprocess: 1 #>1 does not work with mpi + - name: CLClusterShearCatalogs + nprocess: 1 #>1 does not work with mpi # - name: CLClusterEnsembleProfiles # nprocess: 1 # - name: CLClusterDataVector @@ -42,13 +42,13 @@ config: ./examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 + shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5 shear_tomography_catalog: ./data/example/outputs_metadetect shear_tomography_catalog.hdf5 source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml - -resume: True + +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_20deg2.txt diff --git a/examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml b/examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml index e1a7102b0..c2027680f 100644 --- a/examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml +++ b/examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml @@ -1,5 +1,5 @@ TXSourceSelectorMetadetect: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -7,13 +7,10 @@ TXSourceSelectorMetadetect: delta_gamma: 0.02 source_zbin_edges: [0.1, 3.0] chunk_rows: 100000 - true_z: False + true_z: false shear_prefix: '' Inform_BPZ_lite: - aliases: - input: spectroscopic_catalog - model: photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -27,19 +24,15 @@ Inform_BPZ_lite: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01] hdf5_groupname: photometry - + BPZ_lite: - aliases: - model: photoz_model - input: shear_catalog - output: source_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -64,23 +57,23 @@ BPZ_lite: mag_z: 27.98 CLClusterBinningRedshiftRichness: - zedge : [0.1, 0.4, 0.6, 0.8] - richedge : [5., 10., 20.,25.] + zedge: [0.1, 0.4, 0.6, 0.8] + richedge: [5., 10., 20., 25.] + +CLClusterShearCatalogs: + chunk_rows: 100_000 # rows to read at once from source cat + max_radius: 5 # Mpc + delta_z: 0.2 # redshift buffer + redshift_criterion: mean # might also need PDF + subtract_mean_shear: true -CLClusterShearCatalogs: - chunk_rows : 100_000 # rows to read at once from source cat - max_radius : 5 # Mpc - delta_z : 0.2 # redshift buffer - redshift_criterion : "mean" # might also need PDF - subtract_mean_shear : True - CLClusterEnsembleProfiles: #radial bin definition - r_min : 0.3 #in Mpc - r_max : 3.0 #in Mpc - nbins : 4 # number of bins + r_min: 0.3 #in Mpc + r_max: 3.0 #in Mpc + nbins: 4 # number of bins #type of profile - delta_sigma_profile : True - shear_profile : False - magnification_profile : False + delta_sigma_profile: true + shear_profile: false + magnification_profile: false diff --git a/examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml b/examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml index 8c2beb377..558e5054e 100644 --- a/examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml +++ b/examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml @@ -1,5 +1,5 @@ TXSourceSelectorMetadetect: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -7,13 +7,10 @@ TXSourceSelectorMetadetect: delta_gamma: 0.02 source_zbin_edges: [0.1, 3.0] chunk_rows: 100000 - true_z: False + true_z: false shear_prefix: '' Inform_BPZ_lite: - aliases: - input: spectroscopic_catalog - model: photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -27,19 +24,15 @@ Inform_BPZ_lite: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01] hdf5_groupname: photometry - + BPZ_lite: - aliases: - model: photoz_model - input: shear_catalog - output: source_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -64,29 +57,29 @@ BPZ_lite: mag_z: 27.98 CLClusterBinningRedshiftRichness: - zedge : [0.2, 0.4, 0.6, 0.8] - richedge : [5., 10., 20.,25., 50.] + zedge: [0.2, 0.4, 0.6, 0.8] + richedge: [5., 10., 20., 25., 50.] + +CLClusterShearCatalogs: + chunk_rows: 100_000 # rows to read at once from source cat + max_radius: 5 # Mpc + delta_z: 0.2 # redshift buffer + redshift_criterion: mean # might also need PDF + subtract_mean_shear: true -CLClusterShearCatalogs: - chunk_rows : 100_000 # rows to read at once from source cat - max_radius : 5 # Mpc - delta_z : 0.2 # redshift buffer - redshift_criterion : "mean" # might also need PDF - subtract_mean_shear : True - CLClusterEnsembleProfiles: #radial bin definition - r_min : 0.2 #in Mpc - r_max : 3.0 #in Mpc - nbins : 4 # number of bins + r_min: 0.2 #in Mpc + r_max: 3.0 #in Mpc + nbins: 4 # number of bins #type of profile - delta_sigma_profile : True - shear_profile : False - magnification_profile : False + delta_sigma_profile: true + shear_profile: false + magnification_profile: false + + + - - - diff --git a/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-in2p3.yml b/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-in2p3.yml index 7b1262a1e..140a59de4 100644 --- a/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-in2p3.yml +++ b/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-in2p3.yml @@ -2,7 +2,7 @@ #for CCin2p3 site: name: cc-parallel - mpi_command: "mpirun -n" + mpi_command: mpirun -n #for NERSC #site: @@ -14,7 +14,7 @@ site: launcher: name: mini interval: 3.0 - + modules: > txpipe rail.estimation.algos.bpz_lite @@ -28,12 +28,12 @@ stages: # nprocess: 1 # - name: BPZ_lite # nprocess: 1 - - name: CLClusterBinningRedshiftRichness - nprocess: 1 - - name: CLClusterShearCatalogs - nprocess: 1 - - name: CLClusterEnsembleProfiles - nprocess: 1 + - name: CLClusterBinningRedshiftRichness + nprocess: 1 + - name: CLClusterShearCatalogs + nprocess: 1 + - name: CLClusterEnsembleProfiles + nprocess: 1 # - name: CLClusterDataVector # nprocess: 1 @@ -44,7 +44,7 @@ config: examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 + shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 #photometry_catalog: ./data/example/inputs/photometry_catalog.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml #calibration_table: ./data/example/inputs/sample_cosmodc2_w10year_errors.dat @@ -53,8 +53,7 @@ inputs: shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 #cluster_shear_catalogs: ./data/cosmodc2/outputs-1deg2-CL/cluster_shear_catalogs.hdf5 - -resume: True +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_1deg2.txt diff --git a/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-nersc.yml b/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-nersc.yml index 8c44475d3..f93c9c557 100644 --- a/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-nersc.yml +++ b/examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-nersc.yml @@ -6,15 +6,15 @@ #for NERSC site: - name: cori-batch - image: ghcr.io/lsstdesc/txpipe-dev + name: cori-batch + image: ghcr.io/lsstdesc/txpipe-dev #all the following steps should not depend on where you run launcher: name: mini interval: 3.0 - + modules: > txpipe rail.estimation.algos.bpz_lite @@ -28,12 +28,12 @@ stages: # nprocess: 1 # - name: BPZ_lite # nprocess: 1 - - name: CLClusterBinningRedshiftRichness - nprocess: 1 - - name: CLClusterShearCatalogs - nprocess: 1 - - name: CLClusterEnsembleProfiles - nprocess: 1 + - name: CLClusterBinningRedshiftRichness + nprocess: 1 + - name: CLClusterShearCatalogs + nprocess: 1 + - name: CLClusterEnsembleProfiles + nprocess: 1 # - name: CLClusterDataVector # nprocess: 1 @@ -44,7 +44,7 @@ config: examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 + shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 #photometry_catalog: ./data/example/inputs/photometry_catalog.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml #calibration_table: ./data/example/inputs/sample_cosmodc2_w10year_errors.dat @@ -53,8 +53,7 @@ inputs: shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 #cluster_shear_catalogs: ./data/cosmodc2/outputs-1deg2-CL/cluster_shear_catalogs.hdf5 - -resume: True +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_1deg2.txt diff --git a/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml b/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml index 47f1ff0d3..357c05dd0 100644 --- a/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml +++ b/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml @@ -2,7 +2,7 @@ #for CCin2p3 site: name: cc-parallel - mpi_command: "mpirun -n" + mpi_command: mpirun -n #for NERSC #site: @@ -21,18 +21,25 @@ modules: > python_paths: [] stages: - - name: TXSourceSelectorMetadetect - nprocess: 30 - - name: Inform_BPZ_lite - nprocess: 1 - - name: BPZ_lite - nprocess: 30 - - name: CLClusterBinningRedshiftRichness - nprocess: 1 - - name: CLClusterShearCatalogs - nprocess: 1 #>1 does not work with mpi - - name: CLClusterEnsembleProfiles - nprocess: 1 + - name: TXSourceSelectorMetadetect + nprocess: 30 + - name: Inform_BPZ_lite + nprocess: 1 + aliases: + input: spectroscopic_catalog + model: photoz_model + - name: BPZ_lite + nprocess: 30 + aliases: + model: photoz_model + input: shear_catalog + output: source_photoz_pdfs + - name: CLClusterBinningRedshiftRichness + nprocess: 1 + - name: CLClusterShearCatalogs + nprocess: 1 #>1 does not work with mpi + - name: CLClusterEnsembleProfiles + nprocess: 1 # - name: CLClusterDataVector # nprocess: 1 @@ -43,16 +50,15 @@ config: ./examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 + shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 #photometry_catalog: ./data/cosmodc2/20deg2/photometry_catalog.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml - calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat + calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat spectroscopic_catalog: ./data/cosmodc2/20deg2/spectroscopic_catalog.hdf5 cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5 #shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 #source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 - -resume: True +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_20deg2.txt diff --git a/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml b/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml index 36d27e978..29ddf674a 100644 --- a/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml +++ b/examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml @@ -6,8 +6,8 @@ #for NERSC site: - name: cori-batch - image: ghcr.io/lsstdesc/txpipe-dev + name: cori-batch + image: ghcr.io/lsstdesc/txpipe-dev #all the following steps should not depend on where you run @@ -21,18 +21,25 @@ modules: > python_paths: [] stages: - - name: TXSourceSelectorMetadetect - nprocess: 30 - - name: Inform_BPZ_lite - nprocess: 1 - - name: BPZ_lite - nprocess: 30 - - name: CLClusterBinningRedshiftRichness - nprocess: 1 - - name: CLClusterShearCatalogs - nprocess: 1 #>1 does not work with mpi - - name: CLClusterEnsembleProfiles - nprocess: 1 + - name: TXSourceSelectorMetadetect + nprocess: 30 + - name: Inform_BPZ_lite + nprocess: 1 + aliases: + input: spectroscopic_catalog + model: photoz_model + - name: BPZ_lite + nprocess: 30 + aliases: + model: photoz_model + input: shear_catalog + output: source_photoz_pdfs + - name: CLClusterBinningRedshiftRichness + nprocess: 1 + - name: CLClusterShearCatalogs + nprocess: 1 #>1 does not work with mpi + - name: CLClusterEnsembleProfiles + nprocess: 1 # - name: CLClusterDataVector # nprocess: 1 @@ -43,15 +50,14 @@ config: ./examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 + shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 #photometry_catalog: ./data/cosmodc2/20deg2/photometry_catalog.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml - calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat + calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat spectroscopic_catalog: ./data/cosmodc2/20deg2/spectroscopic_catalog.hdf5 cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5 #shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 #source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 - -resume: True +resume: true log_dir: ./data/cosmodc2/logs -pipeline_log: ./data/cosmodc2/log_20deg2.txt \ No newline at end of file +pipeline_log: ./data/cosmodc2/log_20deg2.txt diff --git a/examples/cosmodc2/config-1deg2-CL.yml b/examples/cosmodc2/config-1deg2-CL.yml index 10e126daf..446b5de17 100644 --- a/examples/cosmodc2/config-1deg2-CL.yml +++ b/examples/cosmodc2/config-1deg2-CL.yml @@ -1,5 +1,5 @@ TXSourceSelectorMetadetect: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -7,13 +7,10 @@ TXSourceSelectorMetadetect: delta_gamma: 0.02 source_zbin_edges: [0.1, 3.0] chunk_rows: 100000 - true_z: False + true_z: false shear_prefix: '' BPZliteInformer: - aliases: - input: spectroscopic_catalog - model: photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -27,19 +24,15 @@ BPZliteInformer: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01] hdf5_groupname: photometry - + BPZ_lite: - aliases: - model: photoz_model - input: shear_catalog - output: source_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -64,23 +57,23 @@ BPZ_lite: mag_z: 27.98 CLClusterBinningRedshiftRichness: - zedge : [0.1, 0.4, 0.6, 0.8] - richedge : [5., 10., 20.,25.] + zedge: [0.1, 0.4, 0.6, 0.8] + richedge: [5., 10., 20., 25.] + +CLClusterShearCatalogs: + chunk_rows: 100_000 # rows to read at once from source cat + max_radius: 5 # Mpc + delta_z: 0.2 # redshift buffer + redshift_criterion: mean # might also need PDF + subtract_mean_shear: true -CLClusterShearCatalogs: - chunk_rows : 100_000 # rows to read at once from source cat - max_radius : 5 # Mpc - delta_z : 0.2 # redshift buffer - redshift_criterion : "mean" # might also need PDF - subtract_mean_shear : True - CLClusterEnsembleProfiles: #radial bin definition - r_min : 0.3 #in Mpc - r_max : 3.0 #in Mpc - nbins : 4 # number of bins + r_min: 0.3 #in Mpc + r_max: 3.0 #in Mpc + nbins: 4 # number of bins #type of profile - delta_sigma_profile : True - shear_profile : False - magnification_profile : False + delta_sigma_profile: true + shear_profile: false + magnification_profile: false diff --git a/examples/cosmodc2/config-20deg2-CL.yml b/examples/cosmodc2/config-20deg2-CL.yml index c5aa3667a..60bb74e2f 100644 --- a/examples/cosmodc2/config-20deg2-CL.yml +++ b/examples/cosmodc2/config-20deg2-CL.yml @@ -1,5 +1,5 @@ TXSourceSelectorMetadetect: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -7,13 +7,10 @@ TXSourceSelectorMetadetect: delta_gamma: 0.02 source_zbin_edges: [0.1, 3.0] chunk_rows: 100000 - true_z: False + true_z: false shear_prefix: '' BPZliteInformer: - aliases: - input: spectroscopic_catalog - model: photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -27,19 +24,15 @@ BPZliteInformer: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01] hdf5_groupname: photometry - + BPZ_lite: - aliases: - model: photoz_model - input: shear_catalog - output: source_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -64,29 +57,29 @@ BPZ_lite: mag_z: 27.98 CLClusterBinningRedshiftRichness: - zedge : [0.2, 0.4, 0.6, 0.8] - richedge : [5., 10., 20.,25., 50.] + zedge: [0.2, 0.4, 0.6, 0.8] + richedge: [5., 10., 20., 25., 50.] + +CLClusterShearCatalogs: + chunk_rows: 100_000 # rows to read at once from source cat + max_radius: 5 # Mpc + delta_z: 0.2 # redshift buffer + redshift_criterion: mean # might also need PDF + subtract_mean_shear: true -CLClusterShearCatalogs: - chunk_rows : 100_000 # rows to read at once from source cat - max_radius : 5 # Mpc - delta_z : 0.2 # redshift buffer - redshift_criterion : "mean" # might also need PDF - subtract_mean_shear : True - CLClusterEnsembleProfiles: #radial bin definition - r_min : 0.2 #in Mpc - r_max : 3.0 #in Mpc - nbins : 4 # number of bins + r_min: 0.2 #in Mpc + r_max: 3.0 #in Mpc + nbins: 4 # number of bins #type of profile - delta_sigma_profile : True - shear_profile : False - magnification_profile : False + delta_sigma_profile: true + shear_profile: false + magnification_profile: false + + + - - - diff --git a/examples/cosmodc2/config.yml b/examples/cosmodc2/config.yml index fa84ecf2e..6b7faf309 100644 --- a/examples/cosmodc2/config.yml +++ b/examples/cosmodc2/config.yml @@ -6,18 +6,18 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 - extra_cols: redshift_true size_true shear_1 shear_2 Mag_true_r_sdss_z0 - flip_g2: True # to match metacal + extra_cols: redshift_true size_true shear_1 shear_2 Mag_true_r_sdss_z0 + flip_g2: true # to match metacal snr_limit: 4.0 Mag_r_limit: -19 - unit_response: True - apply_mag_cut: False + unit_response: true + apply_mag_cut: false TXIngestRedmagic: lens_zbin_edges: [0.15, 0.3, 0.45, 0.6, 0.75, 0.9] @@ -27,11 +27,6 @@ TXSourceTrueNumberDensity: nz: 601 zmax: 3.0 chunk_rows: 100000 - aliases: - tomography_catalog: shear_tomography_catalog - catalog: shear_catalog - weights_catalog: shear_catalog - photoz_stack: shear_photoz_stack weight_col: metacal/weight redshift_group: metacal @@ -40,50 +35,45 @@ TXLensTrueNumberDensity: nz: 601 zmax: 3.0 chunk_rows: 100000 - aliases: - tomography_catalog: lens_tomography_catalog - catalog: photometry_catalog - weights_catalog: none - photoz_stack: lens_photoz_stack redshift_group: photometry TXLensMaps: - chunk_rows: 100000 + chunk_rows: 100000 pixelization: healpix nside: 2048 - sparse: True + sparse: true TXSourceMaps: nside: 2048 - sparse: True - chunk_rows: 100000 + sparse: true + chunk_rows: 100000 pixelization: healpix - true_shear: False + true_shear: false TXExternalLensMaps: nside: 2048 - sparse: True - chunk_rows: 100000 + sparse: true + chunk_rows: 100000 pixelization: healpix TXExternalLensNoiseMaps: nside: 2048 - chunk_rows: 100000 + chunk_rows: 100000 pixelization: healpix TXAuxiliarySourceMaps: - chunk_rows: 100000 - sparse: True - psf_prefix: psf_ + chunk_rows: 100000 + sparse: true + psf_prefix: psf_ TXAuxiliaryLensMaps: - chunk_rows: 100000 - sparse: True - bright_obj_threshold: 22.0 + chunk_rows: 100000 + sparse: true + bright_obj_threshold: 22.0 TXSimpleMask: depth_cut: 23.0 - bright_object_max: 10.0 + bright_object_max: 10.0 PZPDFMLZ: @@ -101,29 +91,29 @@ TXTrueNumberDensity: chunk_rows: 100000 TXSourceSelectorMetacal: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 max_rows: 1000 delta_gamma: 0.02 - source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3. ] # 7 bins + source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3.] # 7 bins # source_zbin_edges: [0.25588604, 0.55455363, 0.91863365, 1.38232001, 2.] # 4 bins chunk_rows: 100000 - true_z: False + true_z: false shear_prefix: mcal_ TXSourceSelectorMetadetect: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 max_rows: 1000 delta_gamma: 0.02 - source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3. ] # 7 bins + source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3.] # 7 bins # source_zbin_edges: [0.25588604, 0.55455363, 0.91863365, 1.38232001, 2.] # 4 bins chunk_rows: 100000 - true_z: False + true_z: false shear_prefix: '' @@ -150,7 +140,7 @@ TXRealGaussianCovariance: min_sep: 2.5 max_sep: 250. nbins: 20 - use_true_shear: False + use_true_shear: false nprocess: 4 threads_per_process: 2 nodes: 4 @@ -158,30 +148,30 @@ TXRealGaussianCovariance: TXTwoPointFourier: chunk_rows: 100000 - flip_g1: True - flip_g2: True + flip_g1: true + flip_g2: true apodization_size: 0.0 cache_dir: ./cache_nmt/cosmodc2/nside2048/ - true_shear: False + true_shear: false n_ell: 30 ell_max: 6144 # nside * 3 , since Namaster computes that anyway. nside: 2048 - analytic_noise: True - + analytic_noise: true + TXTwoPoint: bin_slop: 0.01 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: True # use true when using metacal shears + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 250 nbins: 20 verbose: 0 var_method: jackknife - + TXClusteringNoiseMaps: n_realization: 30 @@ -191,7 +181,7 @@ TXLensingNoiseMaps: TXTruthLensSelector: # Mag cuts chunk_rows: 100000 - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -202,12 +192,5 @@ TXTruthLensSelector: TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: lens_nz - TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: source_nz diff --git a/examples/cosmodc2/pipeline-1deg2-CL-in2p3.yml b/examples/cosmodc2/pipeline-1deg2-CL-in2p3.yml index 09f6f4bf5..86c177ddd 100644 --- a/examples/cosmodc2/pipeline-1deg2-CL-in2p3.yml +++ b/examples/cosmodc2/pipeline-1deg2-CL-in2p3.yml @@ -2,7 +2,7 @@ #for CCin2p3 site: name: cc-parallel - mpi_command: "mpirun -n" + mpi_command: mpirun -n #for NERSC #site: @@ -14,7 +14,7 @@ site: launcher: name: mini interval: 3.0 - + modules: > txpipe rail.estimation.algos.bpz_lite @@ -28,12 +28,12 @@ stages: # nprocess: 1 # - name: BPZ_lite # nprocess: 1 - - name: CLClusterBinningRedshiftRichness - nprocess: 1 - - name: CLClusterShearCatalogs - nprocess: 1 - - name: CLClusterEnsembleProfiles - nprocess: 1 + - name: CLClusterBinningRedshiftRichness + nprocess: 1 + - name: CLClusterShearCatalogs + nprocess: 1 + - name: CLClusterEnsembleProfiles + nprocess: 1 # - name: CLClusterDataVector # nprocess: 1 @@ -44,7 +44,7 @@ config: examples/cosmodc2/config-1deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 + shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 #photometry_catalog: ./data/example/inputs/photometry_catalog.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml #calibration_table: ./data/example/inputs/sample_cosmodc2_w10year_errors.dat @@ -53,8 +53,7 @@ inputs: shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 #cluster_shear_catalogs: ./data/cosmodc2/outputs-1deg2-CL/cluster_shear_catalogs.hdf5 - -resume: True +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_1deg2.txt diff --git a/examples/cosmodc2/pipeline-20deg2-CL-in2p3.yml b/examples/cosmodc2/pipeline-20deg2-CL-in2p3.yml index 7cfc9dfed..e60033e00 100644 --- a/examples/cosmodc2/pipeline-20deg2-CL-in2p3.yml +++ b/examples/cosmodc2/pipeline-20deg2-CL-in2p3.yml @@ -2,7 +2,7 @@ #for CCin2p3 site: name: cc-parallel - mpi_command: "mpirun -n" + mpi_command: mpirun -n #for NERSC #site: @@ -27,12 +27,12 @@ stages: # nprocess: 1 # - name: BPZ_lite # nprocess: 30 - - name: CLClusterBinningRedshiftRichness - nprocess: 1 - - name: CLClusterShearCatalogs - nprocess: 1 #>1 does not work with mpi - - name: CLClusterEnsembleProfiles - nprocess: 1 + - name: CLClusterBinningRedshiftRichness + nprocess: 1 + - name: CLClusterShearCatalogs + nprocess: 1 #>1 does not work with mpi + - name: CLClusterEnsembleProfiles + nprocess: 1 # - name: CLClusterDataVector # nprocess: 1 @@ -43,16 +43,15 @@ config: ./examples/cosmodc2/config-20deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 + shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5 #photometry_catalog: ./data/cosmodc2/20deg2/photometry_catalog.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml - calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat + calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat spectroscopic_catalog: ./data/cosmodc2/20deg2/spectroscopic_catalog.hdf5 cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5 #shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 #source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 - -resume: True +resume: true log_dir: ./data/cosmodc2/logs pipeline_log: ./data/cosmodc2/log_20deg2.txt diff --git a/examples/cosmodc2/pipeline.yml b/examples/cosmodc2/pipeline.yml index c06644993..62a61df20 100644 --- a/examples/cosmodc2/pipeline.yml +++ b/examples/cosmodc2/pipeline.yml @@ -9,71 +9,87 @@ site: modules: txpipe python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/FlexZPipe + - submodules/WLMassMap/python/desc/ + - submodules/FlexZPipe stages: - - name: TXRandomCat - - name: TXSourceSelectorMetacal - nprocess: 32 - - name: TXTruthLensSelector - nprocess: 32 - - name: TXLensTrueNumberDensity - classname: TXTruePhotozStack - - name: TXSourceTrueNumberDensity - classname: TXTruePhotozStack - - name: TXPhotozPlotSource - classname: TXPhotozPlot - - name: TXPhotozPlotLens - classname: TXPhotozPlot - - name: TXShearCalibration - - name: TXTruthLensCatalogSplitter - - name: TXTwoPointFourier - nprocess: 2 - nodes: 2 - threads_per_process: 32 - - name: TXJackknifeCenters - - name: TXSourceMaps - nprocess: 8 - - name: TXLensMaps - nprocess: 8 - - name: TXAuxiliarySourceMaps - nprocess: 8 - - name: TXAuxiliaryLensMaps - nprocess: 8 - - name: TXDensityMaps - - name: TXSourceNoiseMaps - nprocess: 4 - nodes: 1 - threads_per_process: 1 - - name: TXLensNoiseMaps - nprocess: 4 - nodes: 1 - threads_per_process: 1 - - name: TXSimpleMask - - name: TXLSSWeightsUnit - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXNullBlinding - - name: TXTwoPoint - threads_per_process: 32 - nprocess: 2 - nodes: 2 - - name: TXTwoPointPlots - - name: TXLensDiagnosticPlots - nprocess: 16 - nodes: 1 - - name: TXSourceDiagnosticPlots - nprocess: 16 - nodes: 1 - - name: TXFourierGaussianCovariance - threads_per_process: 32 - - name: TXTwoPointTheoryReal - - name: TXRealGaussianCovariance - threads_per_process: 32 - - name: TXConvergenceMaps - threads_per_process: 32 - - name: TXConvergenceMapPlots + - name: TXRandomCat + - name: TXSourceSelectorMetacal + nprocess: 32 + - name: TXTruthLensSelector + nprocess: 32 + - name: TXLensTrueNumberDensity + classname: TXTruePhotozStack + aliases: + tomography_catalog: lens_tomography_catalog + catalog: photometry_catalog + weights_catalog: none + photoz_stack: lens_photoz_stack + - name: TXSourceTrueNumberDensity + classname: TXTruePhotozStack + aliases: + tomography_catalog: shear_tomography_catalog + catalog: shear_catalog + weights_catalog: shear_catalog + photoz_stack: shear_photoz_stack + - name: TXPhotozPlotSource + classname: TXPhotozPlot + aliases: + photoz_stack: shear_photoz_stack + nz_plot: source_nz + - name: TXPhotozPlotLens + classname: TXPhotozPlot + aliases: + photoz_stack: lens_photoz_stack + nz_plot: lens_nz + - name: TXShearCalibration + - name: TXTruthLensCatalogSplitter + - name: TXTwoPointFourier + nprocess: 2 + nodes: 2 + threads_per_process: 32 + - name: TXJackknifeCenters + - name: TXSourceMaps + nprocess: 8 + - name: TXLensMaps + nprocess: 8 + - name: TXAuxiliarySourceMaps + nprocess: 8 + - name: TXAuxiliaryLensMaps + nprocess: 8 + - name: TXDensityMaps + - name: TXSourceNoiseMaps + nprocess: 4 + nodes: 1 + threads_per_process: 1 + - name: TXLensNoiseMaps + nprocess: 4 + nodes: 1 + threads_per_process: 1 + - name: TXSimpleMask + - name: TXLSSWeightsUnit + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXNullBlinding + - name: TXTwoPoint + threads_per_process: 32 + nprocess: 2 + nodes: 2 + - name: TXTwoPointPlots + - name: TXLensDiagnosticPlots + nprocess: 16 + nodes: 1 + - name: TXSourceDiagnosticPlots + nprocess: 16 + nodes: 1 + - name: TXFourierGaussianCovariance + threads_per_process: 32 + - name: TXTwoPointTheoryReal + - name: TXRealGaussianCovariance + threads_per_process: 32 + - name: TXConvergenceMaps + threads_per_process: 32 + - name: TXConvergenceMapPlots output_dir: data/cosmodc2/outputs config: examples/cosmodc2/config.yml @@ -83,14 +99,14 @@ config: examples/cosmodc2/config.yml inputs: # See README for paths to download these files - shear_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/shear_catalog.hdf5 - photometry_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 - photoz_trained_model: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/cosmoDC2_trees_i25.3.npy + shear_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/shear_catalog.hdf5 + photometry_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 + photoz_trained_model: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/cosmoDC2_trees_i25.3.npy fiducial_cosmology: data/fiducial_cosmology.yml - calibration_table: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/sample_cosmodc2_w10year_errors.dat + calibration_table: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/sample_cosmodc2_w10year_errors.dat none: none -resume: True +resume: true log_dir: data/cosmodc2/logs pipeline_log: data/cosmodc2/log.txt diff --git a/examples/cosmodc2/pipeline_redmagic.yml b/examples/cosmodc2/pipeline_redmagic.yml index fbb79bbc9..3bc0e9ece 100644 --- a/examples/cosmodc2/pipeline_redmagic.yml +++ b/examples/cosmodc2/pipeline_redmagic.yml @@ -9,69 +9,74 @@ site: modules: txpipe python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/FlexZPipe + - submodules/WLMassMap/python/desc/ + - submodules/FlexZPipe stages: - - name: TXRandomCat - - name: TXIngestRedmagic - - name: TXSourceSelectorMetacal - nprocess: 64 - nodes: 2 - - name: TXShearCalibration - nprocess: 7 - nodes: 1 - - name: TXExternalLensCatalogSplitter - nprocess: 5 - nodes: 1 - - name: TXSourceTrueNumberDensity - - name: TXPhotozPlot - - name: TXJackknifeCenters - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXSourceMaps - nprocess: 12 - threads_per_process: 1 - nodes: 3 - - name: TXExternalLensMaps - - name: TXAuxiliarySourceMaps - nprocess: 8 - threads_per_process: 1 - nodes: 1 - - name: TXAuxiliaryLensMaps - nprocess: 8 - threads_per_process: 1 - nodes: 1 - - name: TXSimpleMask - - name: TXDensityMaps - - name: TXNullBlinding - - name: TXTwoPoint - threads_per_process: 32 - nprocess: 4 - nodes: 4 - - name: TXTwoPointTheoryReal - - name: TXTwoPointTheoryFourier - - name: TXTwoPointPlots - - name: TXRealGaussianCovariance - threads_per_process: 64 - - name: TXSourceNoiseMaps - nprocess: 4 - nodes: 1 - threads_per_process: 1 - - name: TXExternalLensNoiseMaps - nprocess: 8 - nodes: 2 - threads_per_process: 1 - - name: TXTwoPointFourier - nprocess: 2 - nodes: 2 - threads_per_process: 64 - - name: TXFourierGaussianCovariance - threads_per_process: 64 - - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - threads_per_process: 32 - - name: TXConvergenceMapPlots # Plot the convergence map - - name: TXTwoPointPlotsFourier + - name: TXRandomCat + - name: TXIngestRedmagic + - name: TXSourceSelectorMetacal + nprocess: 64 + nodes: 2 + - name: TXShearCalibration + nprocess: 7 + nodes: 1 + - name: TXExternalLensCatalogSplitter + nprocess: 5 + nodes: 1 + - name: TXSourceTrueNumberDensity + aliases: + tomography_catalog: shear_tomography_catalog + catalog: shear_catalog + weights_catalog: shear_catalog + photoz_stack: shear_photoz_stack + - name: TXPhotozPlot + - name: TXJackknifeCenters + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXSourceMaps + nprocess: 12 + threads_per_process: 1 + nodes: 3 + - name: TXExternalLensMaps + - name: TXAuxiliarySourceMaps + nprocess: 8 + threads_per_process: 1 + nodes: 1 + - name: TXAuxiliaryLensMaps + nprocess: 8 + threads_per_process: 1 + nodes: 1 + - name: TXSimpleMask + - name: TXDensityMaps + - name: TXNullBlinding + - name: TXTwoPoint + threads_per_process: 32 + nprocess: 4 + nodes: 4 + - name: TXTwoPointTheoryReal + - name: TXTwoPointTheoryFourier + - name: TXTwoPointPlots + - name: TXRealGaussianCovariance + threads_per_process: 64 + - name: TXSourceNoiseMaps + nprocess: 4 + nodes: 1 + threads_per_process: 1 + - name: TXExternalLensNoiseMaps + nprocess: 8 + nodes: 2 + threads_per_process: 1 + - name: TXTwoPointFourier + nprocess: 2 + nodes: 2 + threads_per_process: 64 + - name: TXFourierGaussianCovariance + threads_per_process: 64 + - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps + threads_per_process: 32 + - name: TXConvergenceMapPlots # Plot the convergence map + - name: TXTwoPointPlotsFourier output_dir: data/cosmodc2/outputs_redmagic/2021/december19/7sbins/ config: examples/cosmodc2/config.yml @@ -81,16 +86,16 @@ config: examples/cosmodc2/config.yml inputs: # See README for paths to download these files - shear_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/shear_catalog.hdf5 - photometry_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 - lens_photometry_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 - photoz_trained_model: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/cosmoDC2_trees_i25.3.npy + shear_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/shear_catalog.hdf5 + photometry_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 + lens_photometry_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 + photoz_trained_model: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/cosmoDC2_trees_i25.3.npy fiducial_cosmology: data/fiducial_cosmology.yml - calibration_table: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/sample_cosmodc2_w10year_errors.dat + calibration_table: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/sample_cosmodc2_w10year_errors.dat redmagic_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/redmagic/cosmoDC2_v1.1.4_run_redmagic_highdens.fit -resume: True +resume: true log_dir: data/cosmodc2/logs_redmagic pipeline_log: data/cosmodc2/log.txt diff --git a/examples/cosmodc2/pipeline_redmagic_full.yml b/examples/cosmodc2/pipeline_redmagic_full.yml index 5710e265b..41b43d333 100644 --- a/examples/cosmodc2/pipeline_redmagic_full.yml +++ b/examples/cosmodc2/pipeline_redmagic_full.yml @@ -9,78 +9,83 @@ site: modules: txpipe python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/FlexZPipe + - submodules/WLMassMap/python/desc/ + - submodules/FlexZPipe stages: - - name: TXCosmoDC2Mock - nprocess: 1 - threads_per_process: 20 - - name: TXRandomCat - nodes: 4 - nprocess: 64 - - name: TXIngestRedmagic - - name: TXSourceSelectorMetadetect - nprocess: 64 - nodes: 2 - - name: TXShearCalibration - nprocess: 7 - nodes: 1 - - name: TXExternalLensCatalogSplitter - nprocess: 5 - nodes: 1 - - name: TXSourceTrueNumberDensity - - name: TXPhotozPlot - - name: TXJackknifeCenters - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXSourceMaps - nprocess: 12 - threads_per_process: 1 - nodes: 3 - - name: TXExternalLensMaps - - name: TXAuxiliarySourceMaps - nprocess: 8 - threads_per_process: 1 - nodes: 1 - - name: TXAuxiliaryLensMaps - nprocess: 8 - threads_per_process: 1 - nodes: 1 - - name: TXSimpleMask - - name: TXDensityMaps - - name: TXNullBlinding - - name: TXTwoPoint - threads_per_process: 32 - nprocess: 6 - nodes: 6 - - name: TXTwoPointTheoryReal - - name: TXTwoPointTheoryFourier - - name: TXTwoPointPlots - - name: TXRealGaussianCovariance - threads_per_process: 64 - - name: TXSourceNoiseMaps - nprocess: 4 - nodes: 1 - threads_per_process: 1 - - name: TXExternalLensNoiseMaps - nprocess: 8 - nodes: 2 - threads_per_process: 1 - - name: TXTwoPointFourier - nprocess: 3 - nodes: 3 - threads_per_process: 64 + - name: TXCosmoDC2Mock + nprocess: 1 + threads_per_process: 20 + - name: TXRandomCat + nodes: 4 + nprocess: 64 + - name: TXIngestRedmagic + - name: TXSourceSelectorMetadetect + nprocess: 64 + nodes: 2 + - name: TXShearCalibration + nprocess: 7 + nodes: 1 + - name: TXExternalLensCatalogSplitter + nprocess: 5 + nodes: 1 + - name: TXSourceTrueNumberDensity + aliases: + tomography_catalog: shear_tomography_catalog + catalog: shear_catalog + weights_catalog: shear_catalog + photoz_stack: shear_photoz_stack + - name: TXPhotozPlot + - name: TXJackknifeCenters + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXSourceMaps + nprocess: 12 + threads_per_process: 1 + nodes: 3 + - name: TXExternalLensMaps + - name: TXAuxiliarySourceMaps + nprocess: 8 + threads_per_process: 1 + nodes: 1 + - name: TXAuxiliaryLensMaps + nprocess: 8 + threads_per_process: 1 + nodes: 1 + - name: TXSimpleMask + - name: TXDensityMaps + - name: TXNullBlinding + - name: TXTwoPoint + threads_per_process: 32 + nprocess: 6 + nodes: 6 + - name: TXTwoPointTheoryReal + - name: TXTwoPointTheoryFourier + - name: TXTwoPointPlots + - name: TXRealGaussianCovariance + threads_per_process: 64 + - name: TXSourceNoiseMaps + nprocess: 4 + nodes: 1 + threads_per_process: 1 + - name: TXExternalLensNoiseMaps + nprocess: 8 + nodes: 2 + threads_per_process: 1 + - name: TXTwoPointFourier + nprocess: 3 + nodes: 3 + threads_per_process: 64 #- name: TXFourierGaussianCovariance # threads_per_process: 64 - - name: TXFourierTJPCovariance - nodes: 4 - nprocess: 4 - threads_per_process: 32 - - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - threads_per_process: 32 - - name: TXConvergenceMapPlots # Plot the convergence map - - name: TXTwoPointPlotsFourier + - name: TXFourierTJPCovariance + nodes: 4 + nprocess: 4 + threads_per_process: 32 + - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps + threads_per_process: 32 + - name: TXConvergenceMapPlots # Plot the convergence map + - name: TXTwoPointPlotsFourier output_dir: data/cosmodc2/outputs_redmagic/2022/june6/ config: examples/cosmodc2/config.yml @@ -90,15 +95,15 @@ config: examples/cosmodc2/config.yml inputs: # See README for paths to download these files - lens_photometry_catalog: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 - photoz_trained_model: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/cosmoDC2_trees_i25.3.npy + lens_photometry_catalog: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/photometry_catalog.hdf5 + photoz_trained_model: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/cosmoDC2_trees_i25.3.npy fiducial_cosmology: data/fiducial_cosmology.yml - calibration_table: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/sample_cosmodc2_w10year_errors.dat + calibration_table: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/sample_cosmodc2_w10year_errors.dat redmagic_catalog: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/redmagic/cosmoDC2_v1.1.4_run_redmagic_highdens.fit - response_model: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/DESY1-R-model.hdf5 + response_model: /global/cfs/cdirs/lsst/groups/WL/users/zuntz/data/DESY1-R-model.hdf5 -resume: True +resume: true log_dir: data/cosmodc2/logs_redmagic/june6/ pipeline_log: data/cosmodc2/logs_redmagic/june6/log.txt diff --git a/examples/dp0.2/config.yml b/examples/dp0.2/config.yml index d77301de1..3a9ee9ea2 100644 --- a/examples/dp0.2/config.yml +++ b/examples/dp0.2/config.yml @@ -2,12 +2,9 @@ global: chunk_rows: 1000000 pixelization: healpix nside: 256 - sparse: True + sparse: true BPZliteInformer: - aliases: - input: spectroscopic_catalog - model: photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -20,7 +17,7 @@ BPZliteInformer: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -28,10 +25,6 @@ BPZliteInformer: BPZ_lite: - aliases: - model: photoz_model - input: photometry_catalog - output: photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -57,40 +50,29 @@ BPZ_lite: mag_r: 29.06 mag_i: 28.62 mag_z: 27.98 - mag_y: 27.05 + mag_y: 27.05 TXMeanLensSelector: - aliases: - lens_photoz_pdfs: photoz_pdfs lens_zbin_edges: [0.3, 0.4, 0.5, 0.6, 0.8, 1.0] selection_type: maglim maglim_limit: 24.0 -TXLensCatalogSplitter: - aliases: - lens_photoz_pdfs: photoz_pdfs - +TXLensCatalogSplitter: {} TXSourceSelectorHSC: bands: ugrizy - verbose: True + verbose: true T_cut: 1.3 s2n_cut: 10.0 source_zbin_edges: [0.3, 0.5, 0.7, 0.9] - shear_prefix: "" + shear_prefix: '' max_shear_cut: 10.0 TXShearCalibration: {} -TXPhotozSourceStack: - aliases: - source_photoz_pdfs: photoz_pdfs - -TXPhotozLensStack: - aliases: - lens_photoz_pdfs: photoz_pdfs - +TXPhotozSourceStack: {} +TXPhotozLensStack: {} TXSourceMaps: {} TXLensMaps: {} @@ -104,14 +86,11 @@ TXTracerMetadata: {} TXRandomCat: method: spherical_projection - aliases: - lens_photoz_pdfs: photoz_pdfs - TXJackknifeCenters: npatch: 100 -TXTwoPoint: - flip_g2: True +TXTwoPoint: + flip_g2: true min_sep: 2.5 max_sep: 250.0 nbins: 20 @@ -120,10 +99,4 @@ TXTwoPoint: #TXRealGaussianCovariance: -TXTwoPointPlotsTheory: - aliases: - # we want to use the summary statistics for the plotting - # since the shot noise error bars here are much less reliable - # than the MCPCov-estimated ones, because of the poor shear - # calibration - twopoint_data_real: summary_statistics_real +TXTwoPointPlotsTheory: {} diff --git a/examples/dp0.2/pipeline.yml b/examples/dp0.2/pipeline.yml index a1ede6289..4e7266b01 100644 --- a/examples/dp0.2/pipeline.yml +++ b/examples/dp0.2/pipeline.yml @@ -1,59 +1,78 @@ # Stages to run stages: # - name: TXIngestDataPreview02 - - name: BPZliteInformer - - name: BPZ_lite - nodes: 2 - nprocess: 128 - threads_per_process: 2 - - name: TXShearCalibration - nprocess: 32 - threads_per_process: 2 - - name: TXMeanLensSelector - nprocess: 32 - threads_per_process: 2 - - name: TXLensCatalogSplitter - nprocess: 32 - threads_per_process: 2 - - name: TXSourceSelectorHSC - nprocess: 32 - threads_per_process: 2 - - name: TXPhotozSourceStack - nprocess: 32 - threads_per_process: 2 - - name: TXPhotozLensStack - nprocess: 32 - threads_per_process: 2 - - name: TXSourceMaps - nprocess: 8 - threads_per_process: 8 - - name: TXLensMaps - nprocess: 8 - threads_per_process: 8 - - name: TXAuxiliarySourceMaps # make PSF and flag maps - nprocess: 8 - threads_per_process: 8 - - name: TXAuxiliaryLensMaps # make depth and bright object maps - nprocess: 8 - threads_per_process: 8 - - name: TXSimpleMask # combine maps to make a simple mask - - name: TXTracerMetadata # collate metadata - - name: TXRandomCat # generate lens bin random catalogs - nprocess: 32 - threads_per_process: 1 - - name: TXJackknifeCenters # Split the area into jackknife regions - - name: TXTwoPoint # Compute real-space 2-point correlations - nprocess: 16 - threads_per_process: 64 - nodes: 8 - - name: TXTwoPointPlotsTheory - - name: TXTwoPointTheoryReal - - name: TXNullBlinding - - name: TXMapPlots - - name: TXDensityMaps - - name: TXRealGaussianCovariance - threads_per_process: 32 - - name: TXPhotozPlot + - name: BPZliteInformer + aliases: + input: spectroscopic_catalog + model: photoz_model + - name: BPZ_lite + nodes: 2 + nprocess: 128 + threads_per_process: 2 + aliases: + model: photoz_model + input: photometry_catalog + output: photoz_pdfs + - name: TXShearCalibration + nprocess: 32 + threads_per_process: 2 + - name: TXMeanLensSelector + nprocess: 32 + threads_per_process: 2 + aliases: + lens_photoz_pdfs: photoz_pdfs + - name: TXLensCatalogSplitter + nprocess: 32 + threads_per_process: 2 + aliases: + lens_photoz_pdfs: photoz_pdfs + - name: TXSourceSelectorHSC + nprocess: 32 + threads_per_process: 2 + - name: TXPhotozSourceStack + nprocess: 32 + threads_per_process: 2 + aliases: + source_photoz_pdfs: photoz_pdfs + - name: TXPhotozLensStack + nprocess: 32 + threads_per_process: 2 + aliases: + lens_photoz_pdfs: photoz_pdfs + - name: TXSourceMaps + nprocess: 8 + threads_per_process: 8 + - name: TXLensMaps + nprocess: 8 + threads_per_process: 8 + - name: TXAuxiliarySourceMaps # make PSF and flag maps + nprocess: 8 + threads_per_process: 8 + - name: TXAuxiliaryLensMaps # make depth and bright object maps + nprocess: 8 + threads_per_process: 8 + - name: TXSimpleMask # combine maps to make a simple mask + - name: TXTracerMetadata # collate metadata + - name: TXRandomCat # generate lens bin random catalogs + nprocess: 32 + threads_per_process: 1 + aliases: + lens_photoz_pdfs: photoz_pdfs + - name: TXJackknifeCenters # Split the area into jackknife regions + - name: TXTwoPoint # Compute real-space 2-point correlations + nprocess: 16 + threads_per_process: 64 + nodes: 8 + - name: TXTwoPointPlotsTheory + aliases: + twopoint_data_real: summary_statistics_real + - name: TXTwoPointTheoryReal + - name: TXNullBlinding + - name: TXMapPlots + - name: TXDensityMaps + - name: TXRealGaussianCovariance + threads_per_process: 32 + - name: TXPhotozPlot # Where to put outputs output_dir: data/dp0.2/outputs @@ -90,8 +109,8 @@ inputs: # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/dp0.2/logs # where to put an overall parsl pipeline log -pipeline_log: data/dp0.2/log.txt \ No newline at end of file +pipeline_log: data/dp0.2/log.txt diff --git a/examples/hscy3/config.yml b/examples/hscy3/config.yml index 60c269e39..c66abfcce 100644 --- a/examples/hscy3/config.yml +++ b/examples/hscy3/config.yml @@ -8,8 +8,8 @@ global: # These mapping options are also read by a range of stages pixelization: healpix nside: 1024 - sparse: True # Generate sparse maps - faster if using small areas - shear_catalog_type: 'hsc' + sparse: true # Generate sparse maps - faster if using small areas + shear_catalog_type: hsc TXGCRTwoCatalogInput: @@ -20,14 +20,14 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal TXIngestRedmagic: lens_zbin_edges: [0.1, 0.3, 0.5] @@ -43,9 +43,6 @@ TXLensTrueNumberDensity: PZPrepareEstimatorLens: name: PZPrepareEstimatorLens classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: lens_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -59,7 +56,7 @@ PZPrepareEstimatorLens: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -69,9 +66,6 @@ PZPrepareEstimatorLens: PZPrepareEstimatorSource: name: PZPrepareEstimatorSource classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: source_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -85,7 +79,7 @@ PZPrepareEstimatorSource: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -95,10 +89,6 @@ PZPrepareEstimatorSource: PZEstimatorLens: name: PZEstimatorLens classname: BPZliteEstimator - aliases: - model: lens_photoz_model - input: photometry_catalog - output: lens_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -125,7 +115,7 @@ PZEstimatorLens: mag_r: 29.06 mag_i: 28.62 mag_z: 27.98 - mag_y: 27.05 + mag_y: 27.05 @@ -135,19 +125,18 @@ TXSourceTrueNumberDensity: zmax: 3.0 TXSourceSelectorHSC: - input_pz: True - true_z : False - bands : i #used for selection - T_cut : 0.0 - s2n_cut : 10.0 + input_pz: true + true_z: false + bands: i #used for selection + T_cut: 0.0 + s2n_cut: 10.0 max_rows: 10000000000 source_zbin_edges: [0.5, 1.5, 2.5, 3.5, 4.5] shear_prefix: '' - shear_catalog_type: 'hsc' - + shear_catalog_type: hsc TXSourceSelectorMetacal: - input_pz: True - true_z: False + input_pz: true + true_z: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -155,19 +144,19 @@ TXSourceSelectorMetacal: delta_gamma: 0.02 source_zbin_edges: [0.0, 0.2, 0.4, 0.6, 0.8] shear_prefix: mcal_ - use_diagonal_response : True - + use_diagonal_response: true + TXShearCalibration: - use_true_shear: False + use_true_shear: false chunk_rows: 100000 - subtract_mean_shear: True - extra_cols: [""] - shear_catalog_type: "hsc" + subtract_mean_shear: true + extra_cols: [''] + shear_catalog_type: hsc TXTruthLensSelector: # Mag cuts - input_pz: False - true_z: True + input_pz: false + true_z: true lens_zbin_edges: [0.1, 0.3, 0.5] cperp_cut: 0.2 r_cpar_cut: 13.5 @@ -180,7 +169,7 @@ TXTruthLensSelector: TXMeanLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -190,26 +179,26 @@ TXMeanLensSelector: r_i_cut: 2.0 TXRoweStatistics: - min_sep : 3.0 - max_sep : 102.0 - nbins : 20 - bin_slop : 0 - sep_units: 'arcmin' - psf_size_units: 'Tmodel' # HSC uses the Tfrac=(Tmeas-Tmodel)/Tmodel definition - definition : 'hsc-y3' # Set all definitions to HSC - subtract_mean : False # subtract mean e1/e2 + min_sep: 3.0 + max_sep: 102.0 + nbins: 20 + bin_slop: 0 + sep_units: arcmin + psf_size_units: Tmodel # HSC uses the Tfrac=(Tmeas-Tmodel)/Tmodel definition + definition: hsc-y3 # Set all definitions to HSC + subtract_mean: false # subtract mean e1/e2 TXPSFMomentCorr: - min_sep : 1.0 - max_sep : 200.0 - nbins : 20 - bin_slop : 0 - sep_units: 'arcmin' - subtract_mean : False # subtract mean e1/e2 + min_sep: 1.0 + max_sep: 200.0 + nbins: 20 + bin_slop: 0 + sep_units: arcmin + subtract_mean: false # subtract mean e1/e2 TXModeLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -223,15 +212,15 @@ TXRandomCat: TXTwoPoint: bin_slop: 0.02 - do_pos_pos: False - do_shear_shear: True - do_shear_pos: False - flip_g2: True # use true when using metacal shears + do_pos_pos: false + do_shear_shear: true + do_shear_pos: false + flip_g2: true # use true when using metacal shears min_sep: 5.3 max_sep: 248 nbins: 13 verbose: 0 - subtract_mean_shear: True + subtract_mean_shear: true TXGammaTBrightStars: {} @@ -250,7 +239,7 @@ TXBlinding: sigma8: [0.801, 0.01] n_s: [0.971, 0.03] b0: 0.95 ## we use bias of the form b0/g - delete_unblinded: True + delete_unblinded: true TXSourceDiagnosticPlots: psf_prefix: mcal_psf_ @@ -268,7 +257,7 @@ TXSourceDiagnosticPlots: TXLensDiagnosticPlots: {} TXDiagnosticMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas snr_threshold: 10.0 snr_delta: 1.0 # pixelization: gnomonic @@ -281,22 +270,20 @@ TXDiagnosticMaps: psf_prefix: mcal_psf_ TXSourceMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXLensMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas nside: 1024 # For redmagic mapping TXExternalLensMaps: chunk_rows: 100000 - sparse: True + sparse: true pixelization: healpix nside: 1024 -TXSourceMaps: {} -TXLensMaps: {} TXDensityMaps: nside: 1024 @@ -309,7 +296,7 @@ TXAuxiliarySourceMaps: TXAuxiliaryLensMaps: flag_exponent_max: 8 bright_obj_threshold: 22.0 # The magnitude threshold for a object to be counted as bright - depth_band : i + depth_band: i snr_threshold: 10.0 # The S/N value to generate maps for (e.g. 5 for 5-sigma depth) snr_delta: 1.0 # The range threshold +/- delta is used for finding objects at the boundary @@ -325,15 +312,15 @@ TXJackknifeCenters: TXTwoPointFourier: - flip_g2: True + flip_g2: true ell_min: 80 ell_max: 2048 bandwidth: 20 cache_dir: ./cache/workspaces - analytic_noise: True + analytic_noise: true TXSimpleMask: - depth_cut : 23.5 + depth_cut: 23.5 bright_object_max: 10.0 TXMapCorrelations: @@ -348,13 +335,8 @@ PZRailSummarizeLens: zmax: 3.0 nzbins: 50 name: PZRailSummarizeLens - catalog_group: "photometry" - tomography_name: "lens" - aliases: - tomography_catalog: lens_tomography_catalog - photometry_catalog: photometry_catalog - model: lens_direct_calibration_model - photoz_stack: lens_photoz_stack + catalog_group: photometry + tomography_name: lens model: None @@ -363,47 +345,19 @@ PZRailSummarizeSource: zmin: 0.0 zmax: 3.0 nzbins: 50 - mag_prefix: "i" - tomography_name: "source" + mag_prefix: i + tomography_name: source name: PZRailSummarizeSource - aliases: - tomography_catalog: shear_tomography_catalog - photometry_catalog: photometry_catalog - model: source_direct_calibration_model - photoz_stack: shear_photoz_stack - - TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: lens_nz - TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: source_nz - FlowCreator: n_samples: 1000000 seed: 5763248 - aliases: - # This input was generated using get_example_flow in pzflow, - # not something specific. - output: ideal_specz_catalog - model: flow - InvRedshiftIncompleteness: pivot_redshift: 0.8 - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq - GridSelection: - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq redshift_cut: 5.1 ratio_file: data/example/inputs/hsc_ratios_and_specz.hdf5 settings_file: data/example/inputs/HSC_grid_settings.pkl @@ -415,37 +369,22 @@ GridSelection: TXParqetToHDF: hdf_group: photometry - aliases: - input: specz_catalog_pq - output: spectroscopic_catalog - - NZDirInformerSource: name: NZDirInformerSource usecols: [r, i, z] hdf5_groupname: photometry - aliases: - input: spectroscopic_catalog - model: source_direct_calibration_model - NZDirInformerLens: name: NZDirInformerLens - usecols: [u, g, r, i, z, "y"] + usecols: [u, g, r, i, z, y] hdf5_groupname: photometry - aliases: - input: spectroscopic_catalog - model: lens_direct_calibration_model - - TXpureB: - method : 'namaster' # 'namaster' or 'hybrideb' - Nell : 20 # Number of ell bins - lmin : 100 # ell min - lmax : 2000 # ell max - lspacing : 'log' # ell spacing for binning - bin_file : # Load predfined bin edges instead - theta_min : 2.5 # (only for hybrideb) theta min in arcmin - theta_max : 250 # (only for hybrideb) theta max in arcmin - Ntheta : 1000 # (only for hybrideb) Number of theta bins - precomputed_hybrideb_weights : '/global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/geb_nosep_dict.npz' - + method: namaster # 'namaster' or 'hybrideb' + Nell: 20 # Number of ell bins + lmin: 100 # ell min + lmax: 2000 # ell max + lspacing: log # ell spacing for binning + bin_file: # Load predfined bin edges instead + theta_min: 2.5 # (only for hybrideb) theta min in arcmin + theta_max: 250 # (only for hybrideb) theta max in arcmin + Ntheta: 1000 # (only for hybrideb) Number of theta bins + precomputed_hybrideb_weights: /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/geb_nosep_dict.npz diff --git a/examples/hscy3/hsc_pipeline.yaml b/examples/hscy3/hsc_pipeline.yaml index ea6386c7f..4bcf2ca84 100644 --- a/examples/hscy3/hsc_pipeline.yaml +++ b/examples/hscy3/hsc_pipeline.yaml @@ -4,28 +4,44 @@ stages: #- name: TXRoweStatistics #- name: TXTauStatistics #- name: TXPSFDiagnostics - - name: TXSimpleMask - - name: TXAuxiliaryLensMaps + - name: TXSimpleMask + - name: TXAuxiliaryLensMaps #- name: TXSourceNoiseMaps - - name: TXTwoPoint # Compute real-space 2-point correlations - threads_per_process: 128 - - name: TXTracerMetadata - - name: FlowCreator # Simulate a spectroscopic population - - name: GridSelection # Simulate a spectroscopic sample - - name: TXParqetToHDF # Convert the spec sample format - - name: TXSourceSelectorHSC # select and split objects into source bins - - name: TXShearCalibration - - name: TXTruthLensSelector - - name: PZPrepareEstimatorLens - classname: Inform_BPZ_lite - - name: PZEstimatorLens - classname: BPZ_lite - - name: TXMeanLensSelector # select objects for lens bins from the PDFs - - name: Inform_NZDirLens # Prepare the DIR method inputs for the lens sample - classname: Inform_NZDir - - name: Inform_NZDirSource # Prepare the DIR method inputs for the source sample - classname: Inform_NZDir - - name: TXJackknifeCenters # Split the area into jackknife regions + - name: TXTwoPoint # Compute real-space 2-point correlations + threads_per_process: 128 + - name: TXTracerMetadata + - name: FlowCreator # Simulate a spectroscopic population + aliases: + output: ideal_specz_catalog + model: flow + - name: GridSelection # Simulate a spectroscopic sample + aliases: + input: ideal_specz_catalog + output: specz_catalog_pq + - name: TXParqetToHDF # Convert the spec sample format + aliases: + input: specz_catalog_pq + output: spectroscopic_catalog + - name: TXSourceSelectorHSC # select and split objects into source bins + - name: TXShearCalibration + - name: TXTruthLensSelector + - name: PZPrepareEstimatorLens + classname: Inform_BPZ_lite + aliases: + input: spectroscopic_catalog + model: lens_photoz_model + - name: PZEstimatorLens + classname: BPZ_lite + aliases: + model: lens_photoz_model + input: photometry_catalog + output: lens_photoz_pdfs + - name: TXMeanLensSelector # select objects for lens bins from the PDFs + - name: Inform_NZDirLens # Prepare the DIR method inputs for the lens sample + classname: Inform_NZDir + - name: Inform_NZDirSource # Prepare the DIR method inputs for the source sample + classname: Inform_NZDir + - name: TXJackknifeCenters # Split the area into jackknife regions #- name: PZRailSummarizeLens # Run the DIR method on the lens sample to find n(z) # classname: PZRailSummarize #- name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) @@ -69,10 +85,10 @@ modules: > # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/TJPCov - - submodules/FlexZPipe - - submodules/RAIL + - submodules/WLMassMap/python/desc/ + - submodules/TJPCov + - submodules/FlexZPipe + - submodules/RAIL # Where to put outputs output_dir: data/hsc-y3/outputs/shearsys/ @@ -98,24 +114,23 @@ config: examples/hscy3/config.yml inputs: # The following files are REQUIRED files - shear_catalog : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/txpipe_allfield_shear.h5 - calibration_table : data/example/inputs/sample_cosmodc2_w10year_errors.dat - star_catalog : /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/nulltests_txpipe/hscy1/star_catalog_hscy1_allfields.h5 - fiducial_cosmology : data/fiducial_cosmology.yml - random_cats_source : Null - random_cats : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/random_cats.hdf5 - - binned_random_catalog : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/random_cats.hdf5 + shear_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/txpipe_allfield_shear.h5 + calibration_table: data/example/inputs/sample_cosmodc2_w10year_errors.dat + star_catalog: /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/nulltests_txpipe/hscy1/star_catalog_hscy1_allfields.h5 + fiducial_cosmology: data/fiducial_cosmology.yml + random_cats_source: + random_cats: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/random_cats.hdf5 + + binned_random_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/random_cats.hdf5 binned_random_catalog_sub: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/random_cats.hdf5 - + # The following are just extracted from other pipelines which never gets used for cosmic shear-only analyses - calibration_table : data/example/inputs/sample_cosmodc2_w10year_errors.dat - flow : data/example/inputs/example_flow.pkl - photometry_catalog : data/example/inputs/photometry_catalog.hdf5 - lens_tomography_catalog : /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/data/desy3a/outputs/lens_tomography_catalog_unweighted.hdf5 - shear_photoz_stack : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hscy1/shear_pz_stack.hdf5 - lens_photoz_stack : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hscy1/lens_pz_stack.hdf5 - binned_lens_catalog : /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/data/desy3a/outputs/binned_lens_catalog.hdf5 + flow: data/example/inputs/example_flow.pkl + photometry_catalog: data/example/inputs/photometry_catalog.hdf5 + lens_tomography_catalog: /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/data/desy3a/outputs/lens_tomography_catalog_unweighted.hdf5 + shear_photoz_stack: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hscy1/shear_pz_stack.hdf5 + lens_photoz_stack: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hscy1/lens_pz_stack.hdf5 + binned_lens_catalog: /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/data/desy3a/outputs/binned_lens_catalog.hdf5 #patch_centers : /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/data/desy3a/outputs/patch_centers.txt @@ -123,7 +138,7 @@ inputs: # if interrupteda # # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/hsc-y3/logs # where to put an overall parsl pipeline log diff --git a/examples/lensfit/config.yml b/examples/lensfit/config.yml index 395a15c37..892d4ed24 100644 --- a/examples/lensfit/config.yml +++ b/examples/lensfit/config.yml @@ -1,6 +1,6 @@ global: chunk_rows: 100000 - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas pixelization: healpix nside: 512 @@ -12,14 +12,14 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal PZPDFMLZ: @@ -42,46 +42,24 @@ PZRailTrainLens: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: ugrizy zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] PZRailEstimateLens: - convert_unseen: True + convert_unseen: true bands: ugrizy TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: nz_source - TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: nz_lens - TXPhotozLensStack: name: TXPhotozLensStack - aliases: - photoz_pdfs: lens_photoz_pdfs - tomography_catalog: lens_tomography_catalog - weights_catalog: none - photoz_stack: lens_photoz_stack - TXPhotozSourceStack: name: TXPhotozSourceStack weight_col: shear/weight - aliases: - # not a mistake - we use the same photoz_pdfs for both - photoz_pdfs: lens_photoz_pdfs - tomography_catalog: shear_tomography_catalog - weights_catalog: shear_catalog - photoz_stack: shear_photoz_stack - -# Mock version of stacking: TXSourceTrueNumberDensity: nz: 301 zmax: 3.0 @@ -90,7 +68,7 @@ TXLensTrueNumberDensity: zmax: 3.0 TXSourceSelectorLensfit: - input_pz: True + input_pz: true bands: ri #used for selection T_cut: 0.25 s2n_cut: 5.0 @@ -98,14 +76,11 @@ TXSourceSelectorLensfit: delta_gamma: 0.02 source_zbin_edges: [0.1, 0.5, 1.0, 2.0] shear_prefix: '' - input_m_is_weighted: True + input_m_is_weighted: true PZPrepareEstimatorLens: name: PZPrepareEstimatorLens classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: lens_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -119,7 +94,7 @@ PZPrepareEstimatorLens: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -128,7 +103,7 @@ PZPrepareEstimatorLens: TXTruthLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -140,7 +115,7 @@ TXTruthLensSelector: TXMeanLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -155,16 +130,16 @@ TXRandomCat: TXTwoPoint: bin_slop: 0.1 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: True # use true when using metacal shears + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 60.0 nbins: 10 verbose: 0 shear_catalog_type: metacal - subtract_mean_shear: True + subtract_mean_shear: true # Using the arc metric here to test it works metric: Arc @@ -186,7 +161,7 @@ TXBlinding: sigma8: [0.801, 0.01] n_s: [0.971, 0.03] b0: 0.95 ## we use bias of the form b0/g - delete_unblinded: True + delete_unblinded: true TXSourceDiagnosticPlots: psf_prefix: psf_ @@ -214,7 +189,7 @@ TXMapPlots: projection: moll TXDiagnosticMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas snr_threshold: 10.0 snr_delta: 1.0 pixelization: healpix @@ -238,7 +213,7 @@ TXAuxiliarySourceMaps: TXAuxiliaryLensMaps: bright_obj_threshold: 22.0 # The magnitude threshold for a object to be counted as bright - depth_band : i + depth_band: i snr_threshold: 10.0 # The S/N value to generate maps for (e.g. 5 for 5-sigma depth) snr_delta: 1.0 # The range threshold +/- delta is used for finding objects at the boundary @@ -255,27 +230,18 @@ TXJackknifeCenters: TXTwoPointFourier: - flip_g2: True + flip_g2: true bandwidth: 100 TXSimpleMask: - depth_cut : 21.5 + depth_cut: 21.5 bright_object_max: 10.0 FlowCreator: n_samples: 1000000 seed: 5763248 - aliases: - # This input was generated using get_example_flow in pzflow, - # not something specific. - output: ideal_specz_catalog - model: flow - GridSelection: - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq redshift_cut: 5.1 ratio_file: data/example/inputs/hsc_ratios_and_specz.hdf5 settings_file: data/example/inputs/HSC_grid_settings.pkl @@ -285,18 +251,9 @@ GridSelection: TXParqetToHDF: hdf_group: photometry - aliases: - input: specz_catalog_pq - output: spectroscopic_catalog - - PZEstimatorLens: name: PZEstimatorLens classname: BPZliteEstimator - aliases: - model: lens_photoz_model - input: photometry_catalog - output: lens_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 diff --git a/examples/lensfit/pipeline.yml b/examples/lensfit/pipeline.yml index 2ea5abbf4..98516f6a6 100644 --- a/examples/lensfit/pipeline.yml +++ b/examples/lensfit/pipeline.yml @@ -1,57 +1,88 @@ - # Stages to run stages: - - name: FlowCreator # Simulate a spectroscopic population - - name: GridSelection # Simulate a spectroscopic sample - - name: TXParqetToHDF # Convert the spec sample format - - name: TXSourceSelectorLensfit - - name: TXShearCalibration - - name: TXLensCatalogSplitter - - name: TXStarCatalogSplitter - - name: TXTruthLensSelector - - name: PZPrepareEstimatorLens - classname: BPZliteInformer - - name: PZEstimatorLens - classname: BPZliteEstimator - - name: TXPhotozSourceStack - classname: TXPhotozStack - - name: TXPhotozLensStack - classname: TXPhotozStack - - name: TXSourceMaps - - name: TXLensMaps - - name: TXAuxiliarySourceMaps - - name: TXAuxiliaryLensMaps - - name: TXSimpleMask - - name: TXLSSWeightsUnit - - name: TXDensityMaps - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXRandomCat - - name: TXJackknifeCenters - - name: TXTwoPoint - threads_per_process: 2 - - name: TXBlinding - threads_per_process: 2 - - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later - - name: TXTwoPointPlots - - name: TXSourceDiagnosticPlots - - name: TXLensDiagnosticPlots - - name: TXGammaTStars - threads_per_process: 2 - - name: TXGammaTRandoms - threads_per_process: 2 - - name: TXRoweStatistics - threads_per_process: 2 - - name: TXGalaxyStarDensity - - name: TXGalaxyStarShear - - name: TXPSFDiagnostics - - name: TXBrighterFatterPlot - - name: TXPhotozPlotLens - classname: TXPhotozPlot - - name: TXPhotozPlotSource - classname: TXPhotozPlot - - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - - name: TXConvergenceMapPlots # Plot the convergence map + - name: FlowCreator # Simulate a spectroscopic population + aliases: + output: ideal_specz_catalog + model: flow + - name: GridSelection # Simulate a spectroscopic sample + aliases: + input: ideal_specz_catalog + output: specz_catalog_pq + - name: TXParqetToHDF # Convert the spec sample format + aliases: + input: specz_catalog_pq + output: spectroscopic_catalog + - name: TXSourceSelectorLensfit + - name: TXShearCalibration + - name: TXLensCatalogSplitter + - name: TXStarCatalogSplitter + - name: TXTruthLensSelector + - name: PZPrepareEstimatorLens + classname: BPZliteInformer + aliases: + input: spectroscopic_catalog + model: lens_photoz_model + - name: PZEstimatorLens + classname: BPZliteEstimator + aliases: + model: lens_photoz_model + input: photometry_catalog + output: lens_photoz_pdfs + - name: TXPhotozSourceStack + classname: TXPhotozStack + aliases: + photoz_pdfs: lens_photoz_pdfs + tomography_catalog: shear_tomography_catalog + weights_catalog: shear_catalog + photoz_stack: shear_photoz_stack + - name: TXPhotozLensStack + classname: TXPhotozStack + aliases: + photoz_pdfs: lens_photoz_pdfs + tomography_catalog: lens_tomography_catalog + weights_catalog: none + photoz_stack: lens_photoz_stack + - name: TXSourceMaps + - name: TXLensMaps + - name: TXAuxiliarySourceMaps + - name: TXAuxiliaryLensMaps + - name: TXSimpleMask + - name: TXLSSWeightsUnit + - name: TXDensityMaps + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXRandomCat + - name: TXJackknifeCenters + - name: TXTwoPoint + threads_per_process: 2 + - name: TXBlinding + threads_per_process: 2 + - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later + - name: TXTwoPointPlots + - name: TXSourceDiagnosticPlots + - name: TXLensDiagnosticPlots + - name: TXGammaTStars + threads_per_process: 2 + - name: TXGammaTRandoms + threads_per_process: 2 + - name: TXRoweStatistics + threads_per_process: 2 + - name: TXGalaxyStarDensity + - name: TXGalaxyStarShear + - name: TXPSFDiagnostics + - name: TXBrighterFatterPlot + - name: TXPhotozPlotLens + classname: TXPhotozPlot + aliases: + photoz_stack: lens_photoz_stack + nz_plot: nz_lens + - name: TXPhotozPlotSource + classname: TXPhotozPlot + aliases: + photoz_stack: shear_photoz_stack + nz_plot: nz_source + - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps + - name: TXConvergenceMapPlots # Plot the convergence map # disabling these as they takes too long for a quick test # - name: TXRealGaussianCovariance # - name: TXTwoPointFourier @@ -83,7 +114,7 @@ modules: > # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ + - submodules/WLMassMap/python/desc/ # configuration settings config: examples/lensfit/config.yml @@ -107,7 +138,7 @@ inputs: # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/example/logs_lensfit # where to put an overall parsl pipeline log diff --git a/examples/metacal/config.yml b/examples/metacal/config.yml index 49691182e..0100294ee 100644 --- a/examples/metacal/config.yml +++ b/examples/metacal/config.yml @@ -8,7 +8,7 @@ global: # These mapping options are also read by a range of stages pixelization: healpix nside: 64 - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXGCRTwoCatalogInput: metacal_dir: /global/cscratch1/sd/desc/DC2/data/Run2.2i/dpdd/Run2.2i-t3828/metacal_table_summary @@ -18,14 +18,14 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal TXIngestRedmagic: lens_zbin_edges: [0.1, 0.3, 0.5] @@ -41,9 +41,6 @@ TXLensTrueNumberDensity: PZPrepareEstimatorLens: name: PZPrepareEstimatorLens classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: lens_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -57,7 +54,7 @@ PZPrepareEstimatorLens: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -67,9 +64,6 @@ PZPrepareEstimatorLens: PZPrepareEstimatorSource: name: PZPrepareEstimatorSource classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: source_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -83,7 +77,7 @@ PZPrepareEstimatorSource: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -93,10 +87,6 @@ PZPrepareEstimatorSource: PZEstimatorLens: name: PZEstimatorLens classname: BPZliteEstimator - aliases: - model: lens_photoz_model - input: photometry_catalog - output: lens_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -123,7 +113,7 @@ PZEstimatorLens: mag_r: 29.06 mag_i: 28.62 mag_z: 27.98 - mag_y: 27.05 + mag_y: 27.05 @@ -134,21 +124,19 @@ TXSourceTrueNumberDensity: TXSourceSelectorMetacal: - input_pz: False - true_z: True + input_pz: false + true_z: true bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 max_rows: 1000 delta_gamma: 0.02 source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0] - shear_prefix: "mcal_" - true_z: False - + shear_prefix: mcal_ TXTruthLensSelector: # Mag cuts - input_pz: False - true_z: True + input_pz: false + true_z: true lens_zbin_edges: [0.1, 0.3, 0.5] cperp_cut: 0.2 r_cpar_cut: 13.5 @@ -161,7 +149,7 @@ TXTruthLensSelector: TXMeanLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -172,7 +160,7 @@ TXMeanLensSelector: TXModeLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -187,15 +175,15 @@ TXRandomCat: TXTwoPoint: bin_slop: 0.1 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: True # use true when using metacal shears + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 60.0 nbins: 10 verbose: 0 - subtract_mean_shear: True + subtract_mean_shear: true TXGammaTBrightStars: {} @@ -214,7 +202,7 @@ TXBlinding: sigma8: [0.801, 0.01] n_s: [0.971, 0.03] b0: 0.95 ## we use bias of the form b0/g - delete_unblinded: True + delete_unblinded: true TXSourceDiagnosticPlots: psf_prefix: mcal_psf_ @@ -232,7 +220,7 @@ TXSourceDiagnosticPlots: TXLensDiagnosticPlots: {} TXDiagnosticMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas snr_threshold: 10.0 snr_delta: 1.0 # pixelization: gnomonic @@ -245,21 +233,19 @@ TXDiagnosticMaps: psf_prefix: mcal_psf_ TXSourceMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXLensMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas # For redmagic mapping TXExternalLensMaps: chunk_rows: 100000 - sparse: True + sparse: true pixelization: healpix nside: 512 -TXSourceMaps: {} -TXLensMaps: {} TXAuxiliarySourceMaps: flag_exponent_max: 8 @@ -268,7 +254,7 @@ TXAuxiliarySourceMaps: TXAuxiliaryLensMaps: flag_exponent_max: 8 bright_obj_threshold: 22.0 # The magnitude threshold for a object to be counted as bright - depth_band : i + depth_band: i snr_threshold: 10.0 # The S/N value to generate maps for (e.g. 5 for 5-sigma depth) snr_delta: 1.0 # The range threshold +/- delta is used for finding objects at the boundary @@ -284,18 +270,18 @@ TXJackknifeCenters: TXTwoPointFourier: - flip_g2: True + flip_g2: true ell_min: 30 ell_max: 150 bandwidth: 20 cache_dir: ./cache/workspaces - do_shear_shear: True - do_shear_pos: True - do_pos_pos: True - compute_theory: True + do_shear_shear: true + do_shear_pos: true + do_pos_pos: true + compute_theory: true TXSimpleMask: - depth_cut : 23.5 + depth_cut: 23.5 bright_object_max: 10.0 TXMapCorrelations: @@ -310,13 +296,8 @@ PZRailSummarizeLens: zmax: 3.0 nzbins: 50 name: PZRailSummarizeLens - catalog_group: "photometry" - tomography_name: "lens" - aliases: - tomography_catalog: lens_tomography_catalog - photometry_catalog: photometry_catalog - model: lens_direct_calibration_model - photoz_stack: lens_photoz_stack + catalog_group: photometry + tomography_name: lens model: None @@ -325,47 +306,19 @@ PZRailSummarizeSource: zmin: 0.0 zmax: 3.0 nzbins: 50 - mag_prefix: "/shear/mcal_mag_" - tomography_name: "source" + mag_prefix: /shear/mcal_mag_ + tomography_name: source name: PZRailSummarizeSource - aliases: - tomography_catalog: shear_tomography_catalog - photometry_catalog: shear_catalog - model: source_direct_calibration_model - photoz_stack: shear_photoz_stack - - TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: lens_nz - TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: source_nz - FlowCreator: n_samples: 1000000 seed: 5763248 - aliases: - # This input was generated using get_example_flow in pzflow, - # not something specific. - output: ideal_specz_catalog - model: flow - InvRedshiftIncompleteness: pivot_redshift: 0.8 - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq - GridSelection: - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq redshift_cut: 5.1 ratio_file: data/example/inputs/hsc_ratios_and_specz.hdf5 settings_file: data/example/inputs/HSC_grid_settings.pkl @@ -377,23 +330,11 @@ GridSelection: TXParqetToHDF: hdf_group: photometry - aliases: - input: specz_catalog_pq - output: spectroscopic_catalog - - NZDirInformerSource: name: NZDirInformerSource usecols: [r, i, z] hdf5_groupname: photometry - aliases: - input: spectroscopic_catalog - model: source_direct_calibration_model - NZDirInformerLens: name: NZDirInformerLens - usecols: [u, g, r, i, z, "y"] + usecols: [u, g, r, i, z, y] hdf5_groupname: photometry - aliases: - input: spectroscopic_catalog - model: lens_direct_calibration_model diff --git a/examples/metacal/pipeline.yml b/examples/metacal/pipeline.yml index 861b264e3..705fbf3da 100644 --- a/examples/metacal/pipeline.yml +++ b/examples/metacal/pipeline.yml @@ -1,70 +1,107 @@ - # Stages to run stages: - - name: FlowCreator # Simulate a spectroscopic population - - name: GridSelection # Simulate a spectroscopic sample - - name: TXParqetToHDF # Convert the spec sample format - - name: PZPrepareEstimatorLens # Prepare the p(z) estimator - classname: BPZliteInformer - - name: PZEstimatorLens # Measure lens galaxy PDFs - classname: BPZliteEstimator - threads_per_process: 1 - - name: TXMeanLensSelector # select objects for lens bins from the PDFs - - name: NZDirInformerLens # Prepare the DIR method inputs for the lens sample - classname: NZDirInformer - - name: PZRailSummarizeLens # Run the DIR method on the lens sample to find n(z) - classname: PZRailSummarize - - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) - classname: PZRailSummarize - - name: TXSourceSelectorMetacal # select and split objects into source bins - - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample - classname: NZDirInformer - - name: TXShearCalibration # Calibrate and split the source sample tomographically - - name: TXLensCatalogSplitter # Split the lens sample tomographically - - name: TXStarCatalogSplitter # Split the star catalog into separate bins (psf/non-psf) - - name: TXSourceMaps # make source g1 and g2 maps - - name: TXLensMaps # make source lens and n_gal maps - - name: TXAuxiliarySourceMaps # make PSF and flag maps - - name: TXAuxiliaryLensMaps # make depth and bright object maps - - name: TXSimpleMask # combine maps to make a simple mask - - name: TXLSSWeightsUnit # add systematic weights to the lens sample (weight=1 for this example) - - name: TXSourceNoiseMaps # Compute shear noise using rotations - - name: TXLensNoiseMaps # Compute lens noise using half-splits - - name: TXDensityMaps # turn mask and ngal maps into overdensity maps - - name: TXMapPlots # make pictures of all the maps - - name: TXTracerMetadata # collate metadata - - name: TXRandomCat # generate lens bin random catalogs - - name: TXJackknifeCenters # Split the area into jackknife regions - - name: TXTwoPoint # Compute real-space 2-point correlations - threads_per_process: 2 - - name: TXBlinding # Blind the data following Muir et al - threads_per_process: 2 - - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later - - name: TXTwoPointPlotsTheory # Make plots of 2pt correlations - - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots - - name: TXLensDiagnosticPlots # Make a suite of diagnostic plots - - name: TXGammaTFieldCenters # Compute and plot gamma_t around center points - threads_per_process: 2 - - name: TXGammaTStars # Compute and plot gamma_t around bright stars - threads_per_process: 2 - - name: TXGammaTRandoms # Compute and plot gamma_t around randoms - threads_per_process: 2 - - name: TXRoweStatistics # Compute and plot Rowe statistics - threads_per_process: 2 - - name: TXGalaxyStarDensity # Compute and plot the star-galaxy density cross-correlation - - name: TXGalaxyStarShear # Compute and plot the star-galaxy shear cross-correlation - - name: TXPSFDiagnostics # Compute and plots other PSF diagnostics - - name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect - - name: TXPhotozPlotSource # Plot the bin n(z) - classname: TXPhotozPlot - - name: TXPhotozPlotLens # Plot the bin n(z) - classname: TXPhotozPlot - - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - - name: TXConvergenceMapPlots # Plot the convergence map - - name: TXMapCorrelations # plot the correlations between systematics and data - - name: TXApertureMass # Compute aperture-mass statistics - threads_per_process: 2 - - name: TXTwoPointFourier # Compute power spectra C_ell + - name: FlowCreator # Simulate a spectroscopic population + aliases: + output: ideal_specz_catalog + model: flow + - name: GridSelection # Simulate a spectroscopic sample + aliases: + input: ideal_specz_catalog + output: specz_catalog_pq + - name: TXParqetToHDF # Convert the spec sample format + aliases: + input: specz_catalog_pq + output: spectroscopic_catalog + - name: PZPrepareEstimatorLens # Prepare the p(z) estimator + classname: BPZliteInformer + aliases: + input: spectroscopic_catalog + model: lens_photoz_model + - name: PZEstimatorLens # Measure lens galaxy PDFs + classname: BPZliteEstimator + threads_per_process: 1 + aliases: + model: lens_photoz_model + input: photometry_catalog + output: lens_photoz_pdfs + - name: TXMeanLensSelector # select objects for lens bins from the PDFs + - name: NZDirInformerLens # Prepare the DIR method inputs for the lens sample + classname: NZDirInformer + aliases: + input: spectroscopic_catalog + model: lens_direct_calibration_model + - name: PZRailSummarizeLens # Run the DIR method on the lens sample to find n(z) + classname: PZRailSummarize + aliases: + tomography_catalog: lens_tomography_catalog + photometry_catalog: photometry_catalog + model: lens_direct_calibration_model + photoz_stack: lens_photoz_stack + - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) + classname: PZRailSummarize + aliases: + tomography_catalog: shear_tomography_catalog + photometry_catalog: shear_catalog + model: source_direct_calibration_model + photoz_stack: shear_photoz_stack + - name: TXSourceSelectorMetacal # select and split objects into source bins + - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample + classname: NZDirInformer + aliases: + input: spectroscopic_catalog + model: source_direct_calibration_model + - name: TXShearCalibration # Calibrate and split the source sample tomographically + - name: TXLensCatalogSplitter # Split the lens sample tomographically + - name: TXStarCatalogSplitter # Split the star catalog into separate bins (psf/non-psf) + - name: TXSourceMaps # make source g1 and g2 maps + - name: TXLensMaps # make source lens and n_gal maps + - name: TXAuxiliarySourceMaps # make PSF and flag maps + - name: TXAuxiliaryLensMaps # make depth and bright object maps + - name: TXSimpleMask # combine maps to make a simple mask + - name: TXLSSWeightsUnit # add systematic weights to the lens sample (weight=1 for this example) + - name: TXSourceNoiseMaps # Compute shear noise using rotations + - name: TXLensNoiseMaps # Compute lens noise using half-splits + - name: TXDensityMaps # turn mask and ngal maps into overdensity maps + - name: TXMapPlots # make pictures of all the maps + - name: TXTracerMetadata # collate metadata + - name: TXRandomCat # generate lens bin random catalogs + - name: TXJackknifeCenters # Split the area into jackknife regions + - name: TXTwoPoint # Compute real-space 2-point correlations + threads_per_process: 2 + - name: TXBlinding # Blind the data following Muir et al + threads_per_process: 2 + - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later + - name: TXTwoPointPlotsTheory # Make plots of 2pt correlations + - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots + - name: TXLensDiagnosticPlots # Make a suite of diagnostic plots + - name: TXGammaTFieldCenters # Compute and plot gamma_t around center points + threads_per_process: 2 + - name: TXGammaTStars # Compute and plot gamma_t around bright stars + threads_per_process: 2 + - name: TXGammaTRandoms # Compute and plot gamma_t around randoms + threads_per_process: 2 + - name: TXRoweStatistics # Compute and plot Rowe statistics + threads_per_process: 2 + - name: TXGalaxyStarDensity # Compute and plot the star-galaxy density cross-correlation + - name: TXGalaxyStarShear # Compute and plot the star-galaxy shear cross-correlation + - name: TXPSFDiagnostics # Compute and plots other PSF diagnostics + - name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect + - name: TXPhotozPlotSource # Plot the bin n(z) + classname: TXPhotozPlot + aliases: + photoz_stack: shear_photoz_stack + nz_plot: source_nz + - name: TXPhotozPlotLens # Plot the bin n(z) + classname: TXPhotozPlot + aliases: + photoz_stack: lens_photoz_stack + nz_plot: lens_nz + - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps + - name: TXConvergenceMapPlots # Plot the convergence map + - name: TXMapCorrelations # plot the correlations between systematics and data + - name: TXApertureMass # Compute aperture-mass statistics + threads_per_process: 2 + - name: TXTwoPointFourier # Compute power spectra C_ell # Disablig this since not yet synchronised with new Treecorr MPI # - name: TXSelfCalibrationIA # Self-calibration intrinsic alignments of galaxies # Disabling these as they take too long for a quick test. @@ -86,7 +123,7 @@ modules: > # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ + - submodules/WLMassMap/python/desc/ # Where to put outputs output_dir: data/example/outputs_metacal @@ -116,12 +153,12 @@ inputs: fiducial_cosmology: data/fiducial_cosmology.yml # For the self-calibration extension we are not using Random_cat_source for now # So it is set to Null, so the yaml intepreter returns a None value to python. - random_cats_source: Null + random_cats_source: flow: data/example/inputs/example_flow.pkl # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/example/logs_metacal # where to put an overall parsl pipeline log diff --git a/examples/metadetect/config.yml b/examples/metadetect/config.yml index 261bdb320..5d1dffde5 100755 --- a/examples/metadetect/config.yml +++ b/examples/metadetect/config.yml @@ -8,7 +8,7 @@ global: # These mapping options are also read by a range of stages pixelization: healpix nside: 64 - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXGCRTwoCatalogInput: metacal_dir: /global/cscratch1/sd/desc/DC2/data/Run2.2i/dpdd/Run2.2i-t3828/metacal_table_summary @@ -18,14 +18,14 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal TXIngestRedmagic: lens_zbin_edges: [0.1, 0.3, 0.5] @@ -41,9 +41,6 @@ TXLensTrueNumberDensity: PZPrepareEstimatorLens: name: PZPrepareEstimatorLens classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: lens_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -57,7 +54,7 @@ PZPrepareEstimatorLens: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -67,9 +64,6 @@ PZPrepareEstimatorLens: PZPrepareEstimatorSource: name: PZPrepareEstimatorSource classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: source_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -84,7 +78,7 @@ PZPrepareEstimatorSource: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -94,10 +88,6 @@ PZPrepareEstimatorSource: PZEstimatorLens: name: PZEstimatorLens classname: BPZliteEstimator - aliases: - model: lens_photoz_model - input: photometry_catalog - output: lens_photoz_pdfs zmin: 0.0 zmax: 3.0 dz: 0.01 @@ -125,7 +115,7 @@ PZEstimatorLens: mag_r: 29.06 mag_i: 28.62 mag_z: 27.98 - mag_y: 27.05 + mag_y: 27.05 @@ -140,21 +130,19 @@ TXTrueNumberDensity: zmax: 3.0 TXSourceSelectorMetadetect: - input_pz: False - true_z: True + input_pz: false + true_z: true bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 max_rows: 1000 delta_gamma: 0.02 source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0] - shear_prefix: "" - true_z: False - + shear_prefix: '' TXTruthLensSelector: # Mag cuts - input_pz: False - true_z: True + input_pz: false + true_z: true lens_zbin_edges: [0.1, 0.3, 0.5] cperp_cut: 0.2 r_cpar_cut: 13.5 @@ -167,7 +155,7 @@ TXTruthLensSelector: TXMeanLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -178,7 +166,7 @@ TXMeanLensSelector: TXModeLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -193,15 +181,15 @@ TXRandomCat: TXTwoPoint: bin_slop: 0.1 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: True # use true when using metacal shears + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 60.0 nbins: 10 verbose: 0 - subtract_mean_shear: True + subtract_mean_shear: true TXGammaTBrightStars: {} @@ -220,7 +208,7 @@ TXBlinding: sigma8: [0.801, 0.01] n_s: [0.971, 0.03] b0: 0.95 ## we use bias of the form b0/g - delete_unblinded: True + delete_unblinded: true TXSourceDiagnosticPlots: psf_prefix: 00/mcal_psf_ @@ -238,7 +226,7 @@ TXSourceDiagnosticPlots: TXLensDiagnosticPlots: {} TXDiagnosticMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas snr_threshold: 10.0 snr_delta: 1.0 # pixelization: gnomonic @@ -251,21 +239,19 @@ TXDiagnosticMaps: psf_prefix: mcal_psf_ TXSourceMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXLensMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas # For redmagic mapping TXExternalLensMaps: chunk_rows: 100000 - sparse: True + sparse: true pixelization: healpix nside: 512 -TXSourceMaps: {} -TXLensMaps: {} TXAuxiliarySourceMaps: flag_exponent_max: 8 @@ -274,7 +260,7 @@ TXAuxiliarySourceMaps: TXAuxiliaryLensMaps: flag_exponent_max: 8 bright_obj_threshold: 22.0 # The magnitude threshold for a object to be counted as bright - depth_band : i + depth_band: i snr_threshold: 10.0 # The S/N value to generate maps for (e.g. 5 for 5-sigma depth) snr_delta: 1.0 # The range threshold +/- delta is used for finding objects at the boundary @@ -285,26 +271,26 @@ TXRealGaussianCovariance: pickled_wigner_transform: data/example/inputs/wigner.pkl TXFourierTJPCovariance: - cov_type: ["FourierGaussianFsky"] + cov_type: [FourierGaussianFsky] TXJackknifeCenters: npatch: 5 TXTwoPointFourier: - flip_g2: True + flip_g2: true ell_min: 30 ell_max: 100 bandwidth: 20 cache_dir: ./cache/workspaces - do_shear_shear: True - do_shear_pos: True - do_pos_pos: True - compute_theory: True + do_shear_shear: true + do_shear_pos: true + do_pos_pos: true + compute_theory: true TXSimpleMaskFrac: supreme_map_file: data/example/inputs/supreme_dc2_dr6d_v2_g_nexp_sum_1deg2.hs - depth_cut : 23.5 + depth_cut: 23.5 bright_object_max: 10.0 TXMapCorrelations: @@ -319,14 +305,8 @@ PZRailSummarizeLens: zmax: 3.0 nzbins: 50 name: PZRailSummarizeLens - catalog_group: "photometry" - tomography_name: "lens" - aliases: - tomography_catalog: lens_tomography_catalog - photometry_catalog: photometry_catalog - model: lens_direct_calibration_model - photoz_stack: lens_photoz_stack - photoz_realizations: lens_photoz_realizations + catalog_group: photometry + tomography_name: lens model: None @@ -336,36 +316,15 @@ PZRailSummarizeSource: zmax: 3.0 nzbins: 50 nsamples: 100 - mag_prefix: "/shear/00/mag_" - tomography_name: "source" + mag_prefix: /shear/00/mag_ + tomography_name: source name: PZRailSummarizeSource - aliases: - tomography_catalog: shear_tomography_catalog - photometry_catalog: shear_catalog - model: source_direct_calibration_model - photoz_stack: shear_photoz_stack - photoz_realizations: source_photoz_realizations - - FlowCreator: n_samples: 1000000 seed: 5763248 - aliases: - # This input was generated using get_example_flow in pzflow, - # not something specific. - output: ideal_specz_catalog - model: flow - InvRedshiftIncompleteness: pivot_redshift: 0.8 - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq - GridSelection: - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq redshift_cut: 5.1 ratio_file: data/example/inputs/hsc_ratios_and_specz.hdf5 settings_file: data/example/inputs/HSC_grid_settings.pkl @@ -377,47 +336,19 @@ GridSelection: TXParqetToHDF: hdf_group: photometry - aliases: - input: specz_catalog_pq - output: spectroscopic_catalog - - NZDirInformerSource: name: NZDirInformerSource usecols: [r, i, z] hdf5_groupname: photometry - aliases: - input: spectroscopic_catalog - model: source_direct_calibration_model - NZDirInformerLens: name: NZDirInformerLens - usecols: [u, g, r, i, z, "y"] + usecols: [u, g, r, i, z, y] hdf5_groupname: photometry - aliases: - input: spectroscopic_catalog - model: lens_direct_calibration_model - PZRealizationsPlotSource: name: PZRealizationsPlotSource - aliases: - photoz_realizations: source_photoz_realizations - photoz_realizations_plot: source_photoz_realizations_plot - PZRealizationsPlotLens: name: PZRealizationsPlotLens - aliases: - photoz_realizations: lens_photoz_realizations - photoz_realizations_plot: lens_photoz_realizations_plot - TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: nz_source - TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: nz_lens diff --git a/examples/metadetect/pipeline.yml b/examples/metadetect/pipeline.yml index 9ec1ca95e..9c3df35f8 100755 --- a/examples/metadetect/pipeline.yml +++ b/examples/metadetect/pipeline.yml @@ -1,75 +1,120 @@ - # Stages to run stages: - - name: FlowCreator # Simulate a spectroscopic population - - name: GridSelection # Simulate a spectroscopic sample - - name: TXParqetToHDF # Convert the spec sample format - - name: PZPrepareEstimatorLens # Prepare the p(z) estimator - classname: BPZliteInformer - - name: PZEstimatorLens # Measure lens galaxy PDFs - classname: BPZliteEstimator - threads_per_process: 1 - - name: TXMeanLensSelector # select objects for lens bins from the PDFs - - name: NZDirInformerLens # Prepare the DIR method inputs for the lens sample - classname: NZDirInformer - - name: PZRailSummarizeLens # Run the DIR method on the lens sample to find n(z) - classname: PZRailSummarize - - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) - classname: PZRailSummarize - - name: TXSourceSelectorMetadetect # select and split objects into source bins - - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample - classname: NZDirInformer - - name: TXShearCalibration # Calibrate and split the source sample tomographically - - name: TXLensCatalogSplitter # Split the lens sample tomographically - - name: TXStarCatalogSplitter # Split the star catalog into separate bins (psf/non-psf) - - name: TXLSSWeightsUnit # add systematic weights to the lens sample (weight=1 for this example) - - name: TXSourceMaps # make source g1 and g2 maps - - name: TXLensMaps # make source lens and n_gal maps - - name: TXAuxiliarySourceMaps # make PSF and flag maps - - name: TXAuxiliaryLensMaps # make depth and bright object maps - - name: TXSimpleMaskFrac # combine maps to make a simple mask - - name: TXSourceNoiseMaps # Compute shear noise using rotations - - name: TXLensNoiseMaps # Compute lens noise using half-splits - - name: TXDensityMaps # turn mask and ngal maps into overdensity maps - - name: TXMapPlots # make pictures of all the maps - - name: TXTracerMetadata # collate metadata - - name: TXRandomCat # generate lens bin random catalogs - - name: TXJackknifeCenters # Split the area into jackknife regions - - name: TXTwoPoint # Compute real-space 2-point correlations - threads_per_process: 2 - - name: TXBlinding # Blind the data following Muir et al - threads_per_process: 2 - - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later - - name: TXTwoPointPlots # Make plots of 2pt correlations - - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots - - name: TXLensDiagnosticPlots # Make a suite of diagnostic plots - - name: TXGammaTFieldCenters # Compute and plot gamma_t around center points - threads_per_process: 2 - - name: TXGammaTStars # Compute and plot gamma_t around bright stars - threads_per_process: 2 - - name: TXGammaTRandoms # Compute and plot gamma_t around randoms - threads_per_process: 2 - - name: TXRoweStatistics # Compute and plot Rowe statistics - threads_per_process: 2 - - name: TXGalaxyStarDensity # Compute and plot the star-galaxy density cross-correlation - - name: TXGalaxyStarShear # Compute and plot the star-galaxy shear cross-correlation - - name: TXPSFDiagnostics # Compute and plots other PSF diagnostics - - name: TXTauStatistics # Compute and plot the tau statistics - - name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect - - name: TXPhotozPlotSource # Plot the bin n(z) - classname: TXPhotozPlot - - name: TXPhotozPlotLens # Plot the bin n(z) - classname: TXPhotozPlot - - name: PZRealizationsPlotSource # Plot n(z) realizations - classname: PZRealizationsPlot - - name: PZRealizationsPlotLens # Plot n(z) realizations - classname: PZRealizationsPlot - - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - - name: TXConvergenceMapPlots # Plot the convergence map - - name: TXMapCorrelations # plot the correlations between systematics and data - - name: TXApertureMass # Compute aperture-mass statistics - threads_per_process: 2 - - name: TXTwoPointFourier # Compute power spectra C_ell + - name: FlowCreator # Simulate a spectroscopic population + aliases: + output: ideal_specz_catalog + model: flow + - name: GridSelection # Simulate a spectroscopic sample + aliases: + input: ideal_specz_catalog + output: specz_catalog_pq + - name: TXParqetToHDF # Convert the spec sample format + aliases: + input: specz_catalog_pq + output: spectroscopic_catalog + - name: PZPrepareEstimatorLens # Prepare the p(z) estimator + classname: BPZliteInformer + aliases: + input: spectroscopic_catalog + model: lens_photoz_model + - name: PZEstimatorLens # Measure lens galaxy PDFs + classname: BPZliteEstimator + threads_per_process: 1 + aliases: + model: lens_photoz_model + input: photometry_catalog + output: lens_photoz_pdfs + - name: TXMeanLensSelector # select objects for lens bins from the PDFs + - name: NZDirInformerLens # Prepare the DIR method inputs for the lens sample + classname: NZDirInformer + aliases: + input: spectroscopic_catalog + model: lens_direct_calibration_model + - name: PZRailSummarizeLens # Run the DIR method on the lens sample to find n(z) + classname: PZRailSummarize + aliases: + tomography_catalog: lens_tomography_catalog + photometry_catalog: photometry_catalog + model: lens_direct_calibration_model + photoz_stack: lens_photoz_stack + photoz_realizations: lens_photoz_realizations + - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) + classname: PZRailSummarize + aliases: + tomography_catalog: shear_tomography_catalog + photometry_catalog: shear_catalog + model: source_direct_calibration_model + photoz_stack: shear_photoz_stack + photoz_realizations: source_photoz_realizations + - name: TXSourceSelectorMetadetect # select and split objects into source bins + - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample + classname: NZDirInformer + aliases: + input: spectroscopic_catalog + model: source_direct_calibration_model + - name: TXShearCalibration # Calibrate and split the source sample tomographically + - name: TXLensCatalogSplitter # Split the lens sample tomographically + - name: TXStarCatalogSplitter # Split the star catalog into separate bins (psf/non-psf) + - name: TXLSSWeightsUnit # add systematic weights to the lens sample (weight=1 for this example) + - name: TXSourceMaps # make source g1 and g2 maps + - name: TXLensMaps # make source lens and n_gal maps + - name: TXAuxiliarySourceMaps # make PSF and flag maps + - name: TXAuxiliaryLensMaps # make depth and bright object maps + - name: TXSimpleMaskFrac # combine maps to make a simple mask + - name: TXSourceNoiseMaps # Compute shear noise using rotations + - name: TXLensNoiseMaps # Compute lens noise using half-splits + - name: TXDensityMaps # turn mask and ngal maps into overdensity maps + - name: TXMapPlots # make pictures of all the maps + - name: TXTracerMetadata # collate metadata + - name: TXRandomCat # generate lens bin random catalogs + - name: TXJackknifeCenters # Split the area into jackknife regions + - name: TXTwoPoint # Compute real-space 2-point correlations + threads_per_process: 2 + - name: TXBlinding # Blind the data following Muir et al + threads_per_process: 2 + - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later + - name: TXTwoPointPlots # Make plots of 2pt correlations + - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots + - name: TXLensDiagnosticPlots # Make a suite of diagnostic plots + - name: TXGammaTFieldCenters # Compute and plot gamma_t around center points + threads_per_process: 2 + - name: TXGammaTStars # Compute and plot gamma_t around bright stars + threads_per_process: 2 + - name: TXGammaTRandoms # Compute and plot gamma_t around randoms + threads_per_process: 2 + - name: TXRoweStatistics # Compute and plot Rowe statistics + threads_per_process: 2 + - name: TXGalaxyStarDensity # Compute and plot the star-galaxy density cross-correlation + - name: TXGalaxyStarShear # Compute and plot the star-galaxy shear cross-correlation + - name: TXPSFDiagnostics # Compute and plots other PSF diagnostics + - name: TXTauStatistics # Compute and plot the tau statistics + - name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect + - name: TXPhotozPlotSource # Plot the bin n(z) + classname: TXPhotozPlot + aliases: + photoz_stack: shear_photoz_stack + nz_plot: nz_source + - name: TXPhotozPlotLens # Plot the bin n(z) + classname: TXPhotozPlot + aliases: + photoz_stack: lens_photoz_stack + nz_plot: nz_lens + - name: PZRealizationsPlotSource # Plot n(z) realizations + classname: PZRealizationsPlot + aliases: + photoz_realizations: source_photoz_realizations + photoz_realizations_plot: source_photoz_realizations_plot + - name: PZRealizationsPlotLens # Plot n(z) realizations + classname: PZRealizationsPlot + aliases: + photoz_realizations: lens_photoz_realizations + photoz_realizations_plot: lens_photoz_realizations_plot + - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps + - name: TXConvergenceMapPlots # Plot the convergence map + - name: TXMapCorrelations # plot the correlations between systematics and data + - name: TXApertureMass # Compute aperture-mass statistics + threads_per_process: 2 + - name: TXTwoPointFourier # Compute power spectra C_ell # Disabled since not yet synchronised with new Treecorr MPI # - name: TXSelfCalibrationIA # Self-calibration intrinsic alignments of galaxies @@ -93,7 +138,7 @@ modules: > # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ + - submodules/WLMassMap/python/desc/ # Where to put outputs output_dir: data/example/outputs_metadetect @@ -125,12 +170,12 @@ inputs: fiducial_cosmology: data/fiducial_cosmology.yml # For the self-calibration extension we are not using Random_cat_source for now # So it is set to Null, so the yaml intepreter returns a None value to python. - random_cats_source: Null + random_cats_source: flow: data/example/inputs/example_flow.pkl # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/example/logs_metadetect # where to put an overall parsl pipeline log diff --git a/examples/metadetect_source_only/config.yml b/examples/metadetect_source_only/config.yml index 06a23eac1..60b0752e4 100755 --- a/examples/metadetect_source_only/config.yml +++ b/examples/metadetect_source_only/config.yml @@ -8,7 +8,7 @@ global: # These mapping options are also read by a range of stages pixelization: healpix nside: 64 - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXGCRTwoCatalogInput: metacal_dir: /global/cscratch1/sd/desc/DC2/data/Run2.2i/dpdd/Run2.2i-t3828/metacal_table_summary @@ -18,14 +18,14 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal TXIngestRedmagic: lens_zbin_edges: [0.1, 0.3, 0.5] @@ -39,9 +39,6 @@ PZPDFMLZ: PZPrepareEstimatorSource: name: PZPrepareEstimatorSource classname: BPZliteInformer - aliases: - input: spectroscopic_catalog - model: source_photoz_model zmin: 0.0 zmax: 3.0 nzbins: 301 @@ -56,7 +53,7 @@ PZPrepareEstimatorSource: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] @@ -70,32 +67,30 @@ TXSourceTrueNumberDensity: TXSourceSelectorMetadetect: - input_pz: False - true_z: True + input_pz: false + true_z: true bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 max_rows: 1000 delta_gamma: 0.02 source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0] - shear_prefix: "" - true_z: False - + shear_prefix: '' TXRandomCat: density: 10 # gals per sq arcmin TXTwoPoint: bin_slop: 0.1 delta_gamma: 0.02 - do_pos_pos: False - do_shear_shear: True - do_shear_pos: False - flip_g2: True # use true when using metacal shears + do_pos_pos: false + do_shear_shear: true + do_shear_pos: false + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 60.0 nbins: 10 verbose: 0 - subtract_mean_shear: True + subtract_mean_shear: true TXGammaTBrightStars: {} @@ -114,7 +109,7 @@ TXBlinding: sigma8: [0.801, 0.01] n_s: [0.971, 0.03] b0: 0.95 ## we use bias of the form b0/g - delete_unblinded: True + delete_unblinded: true TXSourceDiagnosticPlots: psf_prefix: 00/mcal_psf_ @@ -131,7 +126,7 @@ TXSourceDiagnosticPlots: TXDiagnosticMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas snr_threshold: 10.0 snr_delta: 1.0 # pixelization: gnomonic @@ -144,7 +139,7 @@ TXDiagnosticMaps: psf_prefix: mcal_psf_ TXSourceMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXMainMaps: {} @@ -160,14 +155,14 @@ TXRealGaussianCovariance: pickled_wigner_transform: data/example/inputs/wigner.pkl TXFourierTJPCovariance: - cov_type: ["FourierGaussianFsky"] + cov_type: [FourierGaussianFsky] TXJackknifeCenters: npatch: 5 TXTwoPointFourier: - flip_g2: True + flip_g2: true ell_min: 30 ell_max: 100 bandwidth: 20 @@ -175,7 +170,7 @@ TXTwoPointFourier: TXSimpleMaskFrac: supreme_map_file: data/example/inputs/supreme_dc2_dr6d_v2_g_nexp_sum_1deg2.hs - depth_cut : 23.5 + depth_cut: 23.5 bright_object_max: 10.0 TXMapCorrelations: @@ -190,36 +185,15 @@ PZRailSummarizeSource: zmin: 0.0 zmax: 3.0 nzbins: 50 - mag_prefix: "/shear/00/mag_" - tomography_name: "source" + mag_prefix: /shear/00/mag_ + tomography_name: source name: PZRailSummarizeSource - aliases: - tomography_catalog: shear_tomography_catalog - photometry_catalog: shear_catalog - model: source_direct_calibration_model - photoz_stack: shear_photoz_stack - photoz_realizations: source_photoz_realizations - - FlowCreator: n_samples: 1000000 seed: 5763248 - aliases: - # This input was generated using get_example_flow in pzflow, - # not something specific. - output: ideal_specz_catalog - model: flow - InvRedshiftIncompleteness: pivot_redshift: 0.8 - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq - GridSelection: - aliases: - input: ideal_specz_catalog - output: specz_catalog_pq redshift_cut: 5.1 ratio_file: data/example/inputs/hsc_ratios_and_specz.hdf5 settings_file: data/example/inputs/HSC_grid_settings.pkl @@ -231,27 +205,11 @@ GridSelection: TXParqetToHDF: hdf_group: photometry - aliases: - input: specz_catalog_pq - output: spectroscopic_catalog - - NZDirInformerSource: name: NZDirInformerSource usecols: [r, i, z] hdf5_groupname: photometry - aliases: - input: spectroscopic_catalog - model: source_direct_calibration_model - PZRealizationsPlotSource: name: PZRealizationsPlotSource - aliases: - photoz_realizations: source_photoz_realizations - photoz_realizations_plot: source_photoz_realizations_plot - TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: nz_source diff --git a/examples/metadetect_source_only/pipeline.yml b/examples/metadetect_source_only/pipeline.yml index dc4a1e4f7..049052011 100755 --- a/examples/metadetect_source_only/pipeline.yml +++ b/examples/metadetect_source_only/pipeline.yml @@ -1,38 +1,61 @@ - # Stages to run stages: - - name: FlowCreator # Simulate a spectroscopic population - - name: GridSelection # Simulate a spectroscopic sample - - name: TXParqetToHDF # Convert the spec sample format - - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) - classname: PZRailSummarize - - name: TXSourceSelectorMetadetect # select and split objects into source bins - - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample - classname: NZDirInformer - - name: TXShearCalibration # Calibrate and split the source sample tomographically - - name: TXSourceMaps # make source g1 and g2 maps - - name: TXAuxiliarySourceMaps # make PSF and flag maps - - name: TXSimpleMaskSource # combine maps to make a simple mask - - name: TXSourceNoiseMaps # Compute shear noise using rotations - - name: TXMapPlots # make pictures of all the maps - - name: TXTracerMetadata # collate metadata - - name: TXJackknifeCentersSource # Split the area into jackknife regions - - name: TXTwoPoint # Compute real-space 2-point correlations - threads_per_process: 2 - - name: TXBlinding # Blind the data following Muir et al - threads_per_process: 2 - - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later - - name: TXTwoPointPlots # Make plots of 2pt correlations - - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots - - name: TXRoweStatistics # Compute and plot Rowe statistics - threads_per_process: 2 - - name: TXGalaxyStarShear # Compute and plot the star-galaxy shear cross-correlation - - name: TXPhotozPlotSource # Plot the bin n(z) - classname: TXPhotozPlot - - name: PZRealizationsPlotSource # Plot n(z) realizations - classname: PZRealizationsPlot - - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - - name: TXConvergenceMapPlots # Plot the convergence map + - name: FlowCreator # Simulate a spectroscopic population + aliases: + output: ideal_specz_catalog + model: flow + - name: GridSelection # Simulate a spectroscopic sample + aliases: + input: ideal_specz_catalog + output: specz_catalog_pq + - name: TXParqetToHDF # Convert the spec sample format + aliases: + input: specz_catalog_pq + output: spectroscopic_catalog + - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) + classname: PZRailSummarize + aliases: + tomography_catalog: shear_tomography_catalog + photometry_catalog: shear_catalog + model: source_direct_calibration_model + photoz_stack: shear_photoz_stack + photoz_realizations: source_photoz_realizations + - name: TXSourceSelectorMetadetect # select and split objects into source bins + - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample + classname: NZDirInformer + aliases: + input: spectroscopic_catalog + model: source_direct_calibration_model + - name: TXShearCalibration # Calibrate and split the source sample tomographically + - name: TXSourceMaps # make source g1 and g2 maps + - name: TXAuxiliarySourceMaps # make PSF and flag maps + - name: TXSimpleMaskSource # combine maps to make a simple mask + - name: TXSourceNoiseMaps # Compute shear noise using rotations + - name: TXMapPlots # make pictures of all the maps + - name: TXTracerMetadata # collate metadata + - name: TXJackknifeCentersSource # Split the area into jackknife regions + - name: TXTwoPoint # Compute real-space 2-point correlations + threads_per_process: 2 + - name: TXBlinding # Blind the data following Muir et al + threads_per_process: 2 + - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later + - name: TXTwoPointPlots # Make plots of 2pt correlations + - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots + - name: TXRoweStatistics # Compute and plot Rowe statistics + threads_per_process: 2 + - name: TXGalaxyStarShear # Compute and plot the star-galaxy shear cross-correlation + - name: TXPhotozPlotSource # Plot the bin n(z) + classname: TXPhotozPlot + aliases: + photoz_stack: shear_photoz_stack + nz_plot: nz_source + - name: PZRealizationsPlotSource # Plot n(z) realizations + classname: PZRealizationsPlot + aliases: + photoz_realizations: source_photoz_realizations + photoz_realizations_plot: source_photoz_realizations_plot + - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps + - name: TXConvergenceMapPlots # Plot the convergence map # Disabled since not yet synchronised with new Treecorr MPI # - name: TXSelfCalibrationIA # Self-calibration intrinsic alignments of galaxies @@ -56,7 +79,7 @@ modules: > # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ + - submodules/WLMassMap/python/desc/ # Where to put outputs output_dir: data/example/metadetect_source_only @@ -100,7 +123,7 @@ inputs: # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/example/metadetect_source_only # where to put an overall parsl pipeline log diff --git a/examples/redmagic/config.yml b/examples/redmagic/config.yml index 067e1b6ec..69d69876d 100644 --- a/examples/redmagic/config.yml +++ b/examples/redmagic/config.yml @@ -8,7 +8,7 @@ global: # These mapping options are also read by a range of stages pixelization: healpix nside: 512 - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXGCRTwoCatalogInput: metacal_dir: /global/cscratch1/sd/desc/DC2/data/Run2.2i/dpdd/Run2.2i-t3828/metacal_table_summary @@ -18,14 +18,14 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal TXIngestRedmagic: lens_zbin_edges: [0.1, 0.3, 0.5] @@ -49,28 +49,28 @@ PZRailTrainSource: p_min: 0.005 gauss_kernel: 0.0 mag_err_min: 0.005 - inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'} + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} madau_reddening: no bands: riz zp_errors: [0.01, 0.01, 0.01] PZRailEstimateSource: - convert_unseen: True # needed for BPZ + convert_unseen: true # needed for BPZ FlexZPipe: chunk_rows: 1000 - bands: ["u","g","r","i","z","y"] + bands: [u, g, r, i, z, y] sigma_intrins: 0.05 #"intrinsic" assumed scatter, used in ODDS odds_int: 0.99445 #number of sigma_intrins to integrate +/- around peak # note that 1.95993 is the number of sigma you get for old "ODDS" =0.95 #in old BPZ, 0.68 is 0.99445 - has_redshift: True #does the test file have redshift? + has_redshift: true #does the test file have redshift? #if so, read in and append to output file. nz: 300 #Number of grid points that FZboost will calculate - model_picklefile: "data/example/inputs/flexcode_model_sqderr.pkl" + model_picklefile: data/example/inputs/flexcode_model_sqderr.pkl #the pickle file containing the trained flexzbooxt model. - metacal_fluxes: False #switch for whether or not to run metacal suffices + metacal_fluxes: false #switch for whether or not to run metacal suffices # Mock version of stacking: @@ -84,8 +84,8 @@ TXTrueNumberDensity: zmax: 3.0 TXSourceSelectorMetacal: - input_pz: False - true_z: True + input_pz: false + true_z: true bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -96,8 +96,8 @@ TXSourceSelectorMetacal: TXTruthLensSelector: # Mag cuts - input_pz: False - true_z: True + input_pz: false + true_z: true lens_zbin_edges: [0.1, 0.3, 0.5] cperp_cut: 0.2 r_cpar_cut: 13.5 @@ -110,7 +110,7 @@ TXTruthLensSelector: TXMeanLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -121,7 +121,7 @@ TXMeanLensSelector: TXModeLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -136,15 +136,15 @@ TXRandomCat: TXTwoPoint: bin_slop: 0.1 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: True # use true when using metacal shears + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 60.0 nbins: 10 verbose: 0 - subtract_mean_shear: True + subtract_mean_shear: true TXGammaTBrightStars: {} @@ -163,7 +163,7 @@ TXBlinding: sigma8: [0.801, 0.01] n_s: [0.971, 0.03] b0: 0.95 ## we use bias of the form b0/g - delete_unblinded: True + delete_unblinded: true TXSourceDiagnosticPlots: psf_prefix: mcal_psf_ @@ -181,7 +181,7 @@ TXSourceDiagnosticPlots: TXLensDiagnosticPlots: {} TXDiagnosticMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas snr_threshold: 10.0 snr_delta: 1.0 # pixelization: gnomonic @@ -194,21 +194,19 @@ TXDiagnosticMaps: psf_prefix: mcal_psf_ TXSourceMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXLensMaps: - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas # For redmagic mapping TXExternalLensMaps: chunk_rows: 100000 - sparse: True + sparse: true pixelization: healpix nside: 512 -TXSourceMaps: {} -TXLensMaps: {} TXAuxiliarySourceMaps: flag_exponent_max: 8 @@ -217,7 +215,7 @@ TXAuxiliarySourceMaps: TXAuxiliaryLensMaps: flag_exponent_max: 8 bright_obj_threshold: 22.0 # The magnitude threshold for a object to be counted as bright - depth_band : i + depth_band: i snr_threshold: 10.0 # The S/N value to generate maps for (e.g. 5 for 5-sigma depth) snr_delta: 1.0 # The range threshold +/- delta is used for finding objects at the boundary @@ -233,11 +231,11 @@ TXJackknifeCenters: TXTwoPointFourier: - flip_g2: True + flip_g2: true bandwidth: 100 TXSimpleMask: - depth_cut : 23.5 + depth_cut: 23.5 bright_object_max: 10.0 TXMapCorrelations: @@ -247,24 +245,11 @@ TXMapCorrelations: TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: nz_source - TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: nz_lens - TXTruePhotozStackSource: name: TXTruePhotozStackSource - aliases: - tomography_catalog: shear_tomography_catalog - catalog: shear_catalog - weights_catalog: shear_catalog - photoz_stack: shear_photoz_stack weight_col: metacal/weight redshift_group: metacal zmax: 2.0 - nz: 201 \ No newline at end of file + nz: 201 diff --git a/examples/redmagic/pipeline.yml b/examples/redmagic/pipeline.yml index 2e4f05743..0d883fb7e 100644 --- a/examples/redmagic/pipeline.yml +++ b/examples/redmagic/pipeline.yml @@ -1,52 +1,62 @@ - # Stages to run stages: - - name: TXSourceSelectorMetacal - - name: TXShearCalibration - - name: TXExternalLensCatalogSplitter - - name: TXStarCatalogSplitter - - name: TXTruePhotozStackSource - classname: TXTruePhotozStack - - name: TXIngestRedmagic - - name: TXTracerMetadata - - name: TXSourceMaps - - name: TXExternalLensMaps - - name: TXAuxiliarySourceMaps - - name: TXAuxiliaryLensMaps - - name: TXSimpleMask - - name: TXLSSWeightsUnit - - name: TXDensityMaps - - name: TXMapPlots - - name: TXRandomCat - - name: TXJackknifeCenters - - name: TXTwoPoint - threads_per_process: 2 - - name: TXTwoPointRLens - threads_per_process: 2 - - name: TXBlinding # Blind the data following Muir et al - threads_per_process: 2 - - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later - - name: TXPhotozPlotSource # Plot the bin n(z) - classname: TXPhotozPlot - - name: TXPhotozPlotLens # Plot the bin n(z) - classname: TXPhotozPlot - - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots - - name: TXLensDiagnosticPlots # Make a suite of diagnostic plots - - name: TXGammaTFieldCenters # Compute and plot gamma_t around center points - threads_per_process: 2 - - name: TXGammaTRandoms # Compute and plot gamma_t around randoms - threads_per_process: 2 - - name: TXGammaTStars # Compute and plot gamma_t around dim stars - threads_per_process: 2 - - name: TXRoweStatistics # Compute and plot Rowe statistics - threads_per_process: 2 - - name: TXPSFDiagnostics # Compute and plots other PSF diagnostics - - name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect - - name: TXSourceNoiseMaps - - name: TXExternalLensNoiseMaps - - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - - name: TXConvergenceMapPlots # Plot the convergence map - - name: TXTwoPointPlots + - name: TXSourceSelectorMetacal + - name: TXShearCalibration + - name: TXExternalLensCatalogSplitter + - name: TXStarCatalogSplitter + - name: TXTruePhotozStackSource + classname: TXTruePhotozStack + aliases: + tomography_catalog: shear_tomography_catalog + catalog: shear_catalog + weights_catalog: shear_catalog + photoz_stack: shear_photoz_stack + - name: TXIngestRedmagic + - name: TXTracerMetadata + - name: TXSourceMaps + - name: TXExternalLensMaps + - name: TXAuxiliarySourceMaps + - name: TXAuxiliaryLensMaps + - name: TXSimpleMask + - name: TXLSSWeightsUnit + - name: TXDensityMaps + - name: TXMapPlots + - name: TXRandomCat + - name: TXJackknifeCenters + - name: TXTwoPoint + threads_per_process: 2 + - name: TXTwoPointRLens + threads_per_process: 2 + - name: TXBlinding # Blind the data following Muir et al + threads_per_process: 2 + - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later + - name: TXPhotozPlotSource # Plot the bin n(z) + classname: TXPhotozPlot + aliases: + photoz_stack: shear_photoz_stack + nz_plot: nz_source + - name: TXPhotozPlotLens # Plot the bin n(z) + classname: TXPhotozPlot + aliases: + photoz_stack: lens_photoz_stack + nz_plot: nz_lens + - name: TXSourceDiagnosticPlots # Make a suite of diagnostic plots + - name: TXLensDiagnosticPlots # Make a suite of diagnostic plots + - name: TXGammaTFieldCenters # Compute and plot gamma_t around center points + threads_per_process: 2 + - name: TXGammaTRandoms # Compute and plot gamma_t around randoms + threads_per_process: 2 + - name: TXGammaTStars # Compute and plot gamma_t around dim stars + threads_per_process: 2 + - name: TXRoweStatistics # Compute and plot Rowe statistics + threads_per_process: 2 + - name: TXPSFDiagnostics # Compute and plots other PSF diagnostics + - name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect + - name: TXSourceNoiseMaps + - name: TXExternalLensNoiseMaps + - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps + - name: TXConvergenceMapPlots # Plot the convergence map + - name: TXTwoPointPlots # - name: TXTwoPointFourier # threads_per_process: 2 # - name: TXRealGaussianCovariance # Compute covariance of real-space correlations @@ -71,7 +81,7 @@ modules: txpipe # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ + - submodules/WLMassMap/python/desc/ # configuration settings config: examples/redmagic/config.yml @@ -93,7 +103,7 @@ inputs: # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/example/logs_redmagic # where to put an overall parsl pipeline log diff --git a/examples/redmagic/pipeline_complete.yml b/examples/redmagic/pipeline_complete.yml index a2d78da7d..1928f2c4c 100644 --- a/examples/redmagic/pipeline_complete.yml +++ b/examples/redmagic/pipeline_complete.yml @@ -1,48 +1,47 @@ - # Stages to run stages: - - name: FlexZPipe - - name: TXSourceSelector - - name: TXPhotozSourceStack - - name: TXIngestRedmagic - - name: TXSourceMaps - - name: TXExternalLensMaps - - name: TXAuxiliarySourceMaps - - name: TXAuxiliaryLensMaps - - name: TXSimpleMask - - name: TXDensityMaps - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXRandomCat - - name: TXJackknifeCenters - - name: TXTwoPointLensCat - threads_per_process: 2 - - name: TXBlinding - threads_per_process: 2 - - name: TXTwoPointPlots - - name: TXSourceDiagnosticPlots - - name: TXLensDiagnosticPlots - - name: TXGammaTFieldCenters - threads_per_process: 2 - - name: TXGammaTBrightStars - threads_per_process: 2 - - name: TXGammaTDimStars - threads_per_process: 2 - - name: TXRoweStatistics - threads_per_process: 2 - - name: TXPSFDiagnostics - - name: TXRealGaussianCovariance - threads_per_process: 2 - - name: TXSourceNoiseMaps - threads_per_process: 2 - - name: TXExternalLensNoiseMaps - threads_per_process: 2 - - name: TXTwoPointFourier - threads_per_process: 2 - - name: TXFourierGaussianCovariance - threads_per_process: 2 - - name: TXConvergenceMaps - - name: TXConvergenceMapPlots + - name: FlexZPipe + - name: TXSourceSelector + - name: TXPhotozSourceStack + - name: TXIngestRedmagic + - name: TXSourceMaps + - name: TXExternalLensMaps + - name: TXAuxiliarySourceMaps + - name: TXAuxiliaryLensMaps + - name: TXSimpleMask + - name: TXDensityMaps + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXRandomCat + - name: TXJackknifeCenters + - name: TXTwoPointLensCat + threads_per_process: 2 + - name: TXBlinding + threads_per_process: 2 + - name: TXTwoPointPlots + - name: TXSourceDiagnosticPlots + - name: TXLensDiagnosticPlots + - name: TXGammaTFieldCenters + threads_per_process: 2 + - name: TXGammaTBrightStars + threads_per_process: 2 + - name: TXGammaTDimStars + threads_per_process: 2 + - name: TXRoweStatistics + threads_per_process: 2 + - name: TXPSFDiagnostics + - name: TXRealGaussianCovariance + threads_per_process: 2 + - name: TXSourceNoiseMaps + threads_per_process: 2 + - name: TXExternalLensNoiseMaps + threads_per_process: 2 + - name: TXTwoPointFourier + threads_per_process: 2 + - name: TXFourierGaussianCovariance + threads_per_process: 2 + - name: TXConvergenceMaps + - name: TXConvergenceMapPlots # Where to put outputs output_dir: data/example/outputs @@ -64,8 +63,8 @@ modules: txpipe # where to find any modules that are not in this repo, # and any other code we need. python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/FlexZPipe + - submodules/WLMassMap/python/desc/ + - submodules/FlexZPipe # configuration settings config: examples/redmagic/config.yml @@ -87,7 +86,7 @@ inputs: # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: True +resume: true # where to put output logs for individual stages log_dir: data/example/logs # where to put an overall parsl pipeline log diff --git a/examples/skysim/config.yml b/examples/skysim/config.yml index aa31cd474..d3b8e09da 100644 --- a/examples/skysim/config.yml +++ b/examples/skysim/config.yml @@ -9,14 +9,14 @@ TXMetacalGCRInput: cat_name: dc2_object_run2.1i_dr1b_with_metacal_griz TXExposureInfo: - dc2_name: '1.2p' + dc2_name: 1.2p TXCosmoDC2Mock: cat_name: cosmoDC2_v1.1.4_image visits_per_band: 16 extra_cols: redshift_true size_true shear_1 shear_2 - flip_g2: True # to match metacal + flip_g2: true # to match metacal TXIngestRedmagic: lens_zbin_edges: [0.1, 0.3, 0.5, 0.7, 0.9] @@ -25,11 +25,6 @@ TXSourceTrueNumberDensity: nz: 301 zmax: 3.0 name: TXSourceTrueNumberDensity - aliases: - tomography_catalog: shear_tomography_catalog - catalog: shear_catalog - weights_catalog: shear_catalog - photoz_stack: shear_photoz_stack weight_col: shear/00/weight redshift_group: shear/00 @@ -38,42 +33,37 @@ TXLensTrueNumberDensity: nz: 301 zmax: 3.0 name: TXLensTrueNumberDensity - aliases: - tomography_catalog: lens_tomography_catalog - catalog: photometry_catalog - weights_catalog: none - photoz_stack: lens_photoz_stack redshift_group: photometry TXLensMaps: pixelization: healpix nside: 512 - sparse: True + sparse: true TXSourceMaps: nside: 512 - sparse: True + sparse: true pixelization: healpix - true_shear: False + true_shear: false TXExternalLensMaps: nside: 512 - sparse: True + sparse: true pixelization: healpix TXAuxiliarySourceMaps: - sparse: True - psf_prefix: psf_ + sparse: true + psf_prefix: psf_ TXAuxiliaryLensMaps: - sparse: True - bright_obj_threshold: 22.0 + sparse: true + bright_obj_threshold: 22.0 TXSimpleMask: depth_cut: 23.0 - bright_object_max: 10.0 + bright_object_max: 10.0 PZPDFMLZ: @@ -85,7 +75,7 @@ TXPhotozStack: {} TXSourceSelectorMetacal: - input_pz: False + input_pz: false bands: riz #used for selection T_cut: 0.5 s2n_cut: 10.0 @@ -93,7 +83,7 @@ TXSourceSelectorMetacal: delta_gamma: 0.02 source_zbin_edges: [0.3, 0.55, 0.8, 1.05, 2.0] # may also need one for r_cpar_cut - true_z: False + true_z: false shear_prefix: mcal_ TXRandomCat: @@ -113,38 +103,35 @@ TXRealGaussianCovariance: TXTwoPoint: bin_slop: 0.0 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: True # use true when using metacal shears + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 250 nbins: 20 verbose: 0 - use_true_shear: False + use_true_shear: false TXTwoPointLensCat: bin_slop: 0.0 delta_gamma: 0.02 - do_pos_pos: True - do_shear_shear: True - do_shear_pos: True - flip_g2: True # use true when using metacal shears + do_pos_pos: true + do_shear_shear: true + do_shear_pos: true + flip_g2: true # use true when using metacal shears min_sep: 2.5 max_sep: 250 nbins: 20 verbose: 0 - use_true_shear: False - + use_true_shear: false + TXTwoPointFourier: - flip_g2: True - flip_g1: True + flip_g2: true + flip_g1: true apodization_size: 0.0 cache_dir: ./cache - true_shear: False - -TXRealGaussianCovariance: - use_true_shear: False + true_shear: false TXClusteringNoiseMaps: n_realization: 30 @@ -154,7 +141,7 @@ TXLensingNoiseMaps: TXTruthLensSelector: # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -166,12 +153,5 @@ TXTruthLensSelector: TXPhotozPlotLens: name: TXPhotozPlotLens - aliases: - photoz_stack: lens_photoz_stack - nz_plot: lens_nz - TXPhotozPlotSource: name: TXPhotozPlotSource - aliases: - photoz_stack: shear_photoz_stack - nz_plot: source_nz diff --git a/examples/skysim/pipeline.yml b/examples/skysim/pipeline.yml index caf69ed85..986722b18 100644 --- a/examples/skysim/pipeline.yml +++ b/examples/skysim/pipeline.yml @@ -9,78 +9,94 @@ site: modules: txpipe python_paths: - - submodules/WLMassMap/python/desc/ - - submodules/FlexZPipe + - submodules/WLMassMap/python/desc/ + - submodules/FlexZPipe stages: - - name: TXRandomCat - - name: TXSourceSelectorMetacal - nodes: 8 - nprocess: 256 - - name: TXTruthLensSelector - nodes: 8 - nprocess: 256 - - name: TXShearCalibration - nodes: 1 - nprocess: 8 - - name: TXTruthLensCatalogSplitter - nodes: 1 - nprocess: 8 - - name: TXSourceTrueNumberDensity - classname: TXTruePhotozStack - nodes: 8 - nprocess: 64 - - name: TXLensTrueNumberDensity - classname: TXTruePhotozStack - nodes: 8 - nprocess: 64 - - name: TXPhotozPlotSource - classname: TXPhotozPlot - - name: TXPhotozPlotLens - classname: TXPhotozPlot - - name: TXJackknifeCenters - - name: TXDensityMaps - - name: TXSourceMaps - nodes: 8 - nprocess: 64 - - name: TXLensMaps - nodes: 8 - nprocess: 64 - - name: TXAuxiliarySourceMaps - nodes: 2 - nprocess: 16 - - name: TXAuxiliaryLensMaps - nodes: 2 - nprocess: 16 - - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later - - name: TXTwoPointTheoryFourier - - name: TXSimpleMask - - name: TXLSSWeightsUnit - - name: TXMapPlots - - name: TXTracerMetadata - - name: TXNullBlinding - - name: TXTwoPoint - threads_per_process: 32 - nprocess: 8 - nodes: 8 - - name: TXTwoPointPlots - - name: TXRealGaussianCovariance - threads_per_process: 32 - - name: TXTwoPointFourier - nprocess: 8 - nodes: 8 - threads_per_process: 32 - - name: TXSourceNoiseMaps - nprocess: 32 - nodes: 8 - threads_per_process: 1 - - name: TXLensNoiseMaps - nprocess: 32 - nodes: 8 - threads_per_process: 1 - - name: TXFourierGaussianCovariance - threads_per_process: 32 - - name: TXTwoPointPlotsFourier + - name: TXRandomCat + - name: TXSourceSelectorMetacal + nodes: 8 + nprocess: 256 + - name: TXTruthLensSelector + nodes: 8 + nprocess: 256 + - name: TXShearCalibration + nodes: 1 + nprocess: 8 + - name: TXTruthLensCatalogSplitter + nodes: 1 + nprocess: 8 + - name: TXSourceTrueNumberDensity + classname: TXTruePhotozStack + nodes: 8 + nprocess: 64 + aliases: + tomography_catalog: shear_tomography_catalog + catalog: shear_catalog + weights_catalog: shear_catalog + photoz_stack: shear_photoz_stack + - name: TXLensTrueNumberDensity + classname: TXTruePhotozStack + nodes: 8 + nprocess: 64 + aliases: + tomography_catalog: lens_tomography_catalog + catalog: photometry_catalog + weights_catalog: none + photoz_stack: lens_photoz_stack + - name: TXPhotozPlotSource + classname: TXPhotozPlot + aliases: + photoz_stack: shear_photoz_stack + nz_plot: source_nz + - name: TXPhotozPlotLens + classname: TXPhotozPlot + aliases: + photoz_stack: lens_photoz_stack + nz_plot: lens_nz + - name: TXJackknifeCenters + - name: TXDensityMaps + - name: TXSourceMaps + nodes: 8 + nprocess: 64 + - name: TXLensMaps + nodes: 8 + nprocess: 64 + - name: TXAuxiliarySourceMaps + nodes: 2 + nprocess: 16 + - name: TXAuxiliaryLensMaps + nodes: 2 + nprocess: 16 + - name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later + - name: TXTwoPointTheoryFourier + - name: TXSimpleMask + - name: TXLSSWeightsUnit + - name: TXMapPlots + - name: TXTracerMetadata + - name: TXNullBlinding + - name: TXTwoPoint + threads_per_process: 32 + nprocess: 8 + nodes: 8 + - name: TXTwoPointPlots + - name: TXRealGaussianCovariance + threads_per_process: 32 + - name: TXTwoPointFourier + nprocess: 8 + nodes: 8 + threads_per_process: 32 + - name: TXSourceNoiseMaps + nprocess: 32 + nodes: 8 + threads_per_process: 1 + - name: TXLensNoiseMaps + nprocess: 32 + nodes: 8 + threads_per_process: 1 + - name: TXFourierGaussianCovariance + threads_per_process: 32 + - name: TXTwoPointPlotsFourier output_dir: data/skysim5000_v1.1.1/outputs config: examples/skysim/config.yml @@ -90,13 +106,13 @@ config: examples/skysim/config.yml inputs: # See README for paths to download these files - shear_catalog: data/skysim5000_v1.1.1/inputs/shear_catalog.hdf5 + shear_catalog: data/skysim5000_v1.1.1/inputs/shear_catalog.hdf5 photometry_catalog: data/skysim5000_v1.1.1/inputs/photometry_catalog.hdf5 fiducial_cosmology: data/fiducial_cosmology.yml calibration_table: data/example/inputs/sample_cosmodc2_w10year_errors.dat none: none -resume: True +resume: true log_dir: data/skysim5000_v1.1.1/logs pipeline_log: data/skysim5000_v1.1.1/log.txt diff --git a/examples/ssi/config_mag.yml b/examples/ssi/config_mag.yml index 313cf9775..3ccde86bb 100755 --- a/examples/ssi/config_mag.yml +++ b/examples/ssi/config_mag.yml @@ -8,7 +8,7 @@ global: # These mapping options are also read by a range of stages pixelization: healpix nside: 256 - sparse: True # Generate sparse maps - faster if using small areas + sparse: true # Generate sparse maps - faster if using small areas TXMatchSSI: name: TXMatchSSI @@ -18,15 +18,10 @@ TXMatchSSIMag: name: TXMatchSSIMag match_radius: 0.1 magnification: 0.02 - aliases: - injection_catalog: injection_catalog_mag - ssi_photometry_catalog: ssi_photometry_catalog_mag - matched_ssi_photometry_catalog: matched_ssi_photometry_catalog_mag - TXTruthLensSelectorSSI: name: TXTruthLensSelectorSSI # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -34,13 +29,10 @@ TXTruthLensSelectorSSI: i_lo_cut: 17.5 i_hi_cut: 21.9 r_i_cut: 2.0 - aliases: - photometry_catalog: matched_ssi_photometry_catalog - TXTruthLensSelectorSSIMag: name: TXTruthLensSelectorSSIMag # Mag cuts - lens_zbin_edges: [0.0,0.2,0.4] + lens_zbin_edges: [0.0, 0.2, 0.4] cperp_cut: 0.2 r_cpar_cut: 13.5 r_lo_cut: 16.0 @@ -48,26 +40,10 @@ TXTruthLensSelectorSSIMag: i_lo_cut: 17.5 i_hi_cut: 21.9 r_i_cut: 2.0 - aliases: - photometry_catalog: matched_ssi_photometry_catalog_mag - lens_tomography_catalog_unweighted: lens_tomography_catalog_unweighted_mag - -TXTruthLensCatalogSplitterSSI: +TXTruthLensCatalogSplitterSSI: name: TXTruthLensCatalogSplitterSSI - aliases: - photometry_catalog: matched_ssi_photometry_catalog - -TXTruthLensCatalogSplitterSSIMag: +TXTruthLensCatalogSplitterSSIMag: name: TXTruthLensCatalogSplitterSSIMag - aliases: - photometry_catalog: matched_ssi_photometry_catalog_mag - lens_tomography_catalog_unweighted: lens_tomography_catalog_unweighted_mag - binned_lens_catalog_unweighted: binned_lens_catalog_unweighted_mag - TXSSIMagnification: name: TXSSIMagnification applied_magnification: 1.02 - aliases: - binned_lens_catalog_nomag: binned_lens_catalog_unweighted - binned_lens_catalog_mag: binned_lens_catalog_unweighted_mag - diff --git a/examples/ssi/pipeline_mag.yml b/examples/ssi/pipeline_mag.yml index 28306b67a..152c04053 100755 --- a/examples/ssi/pipeline_mag.yml +++ b/examples/ssi/pipeline_mag.yml @@ -1,26 +1,43 @@ - # Stages to run stages: - - name: TXMatchSSI # process the SSI inputs - - name: TXMatchSSIMag # process the SSI inputs - classname: TXMatchSSI + - name: TXMatchSSI # process the SSI inputs + - name: TXMatchSSIMag # process the SSI inputs + classname: TXMatchSSI + + aliases: + injection_catalog: injection_catalog_mag + ssi_photometry_catalog: ssi_photometry_catalog_mag + matched_ssi_photometry_catalog: matched_ssi_photometry_catalog_mag + - name: TXTruthLensSelectorSSI #make SSI run lens samples + classname: TXTruthLensSelector + aliases: + photometry_catalog: matched_ssi_photometry_catalog + - name: TXTruthLensCatalogSplitterSSI + classname: TXTruthLensCatalogSplitter - - name: TXTruthLensSelectorSSI #make SSI run lens samples - classname: TXTruthLensSelector - - name: TXTruthLensCatalogSplitterSSI - classname: TXTruthLensCatalogSplitter - - - name: TXTruthLensSelectorSSIMag #make magnified SSI run lens samples - classname: TXTruthLensSelector - - name: TXTruthLensCatalogSplitterSSIMag - classname: TXTruthLensCatalogSplitter + aliases: + photometry_catalog: matched_ssi_photometry_catalog + - name: TXTruthLensSelectorSSIMag #make magnified SSI run lens samples + classname: TXTruthLensSelector + aliases: + photometry_catalog: matched_ssi_photometry_catalog_mag + lens_tomography_catalog_unweighted: lens_tomography_catalog_unweighted_mag + - name: TXTruthLensCatalogSplitterSSIMag + classname: TXTruthLensCatalogSplitter - - name: TXSSIMagnification # compute magnification coefficients for lens sample + aliases: + photometry_catalog: matched_ssi_photometry_catalog_mag + lens_tomography_catalog_unweighted: lens_tomography_catalog_unweighted_mag + binned_lens_catalog_unweighted: binned_lens_catalog_unweighted_mag + - name: TXSSIMagnification # compute magnification coefficients for lens sample # modules and packages to import that have pipeline # stages defined in them + aliases: + binned_lens_catalog_nomag: binned_lens_catalog_unweighted + binned_lens_catalog_mag: binned_lens_catalog_unweighted_mag modules: txpipe # modules: txpipe rail tjpcov @@ -42,7 +59,6 @@ config: examples/ssi/config_mag.yml inputs: # See README for paths to download these files - #use photometry catalog for everything, just while setting up injection_catalog: data/example/inputs/photometry_catalog.hdf5 ssi_photometry_catalog: data/example/inputs/photometry_catalog.hdf5 @@ -55,7 +71,7 @@ inputs: # if supported by the launcher, restart the pipeline where it left off # if interrupted -resume: False +resume: false # where to put output logs for individual stages log_dir: data/example/outputs_ssi_mag/logs/ # where to put an overall parsl pipeline log diff --git a/txpipe/source_selector.py b/txpipe/source_selector.py index dc2c90bd8..10f483d9b 100755 --- a/txpipe/source_selector.py +++ b/txpipe/source_selector.py @@ -631,14 +631,15 @@ def data_iterator(self): shear_cols += band_variants( bands, "mag", "mag_err", shear_catalog_type="metadetect" ) + renames = {} # We need truth shears and/or PZ point-estimates for each shear too if self.config["input_pz"]: shear_cols += metadetect_variants("mean_z") elif self.config["true_z"]: - shear_cols += ["redshift_true"] + shear_cols += ["00/redshift_true"] + renames["00/redshift_true"] = "redshift_true" - renames = {} for prefix in ["00", "1p", "1m", "2p", "2m"]: renames[f"{prefix}/mcal_psf_T_mean"] = f"{prefix}/psf_T_mean" From 182085d5a41506abd6625fffb622971045332e55 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Fri, 9 Aug 2024 10:37:27 +0100 Subject: [PATCH 06/15] switch to dev image --- .github/workflows/ci.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4a97647e..c1ebca36f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ env: EXAMPLE_DATA_FILE_VERSION: v2 # Setting this did not appear to work. Instead will need to # find/replace when changing it. - # CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe:latest + # CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe-dev:latest jobs: # Run a download step first so that it is in @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-latest container: - image: ghcr.io/lsstdesc/txpipe:latest + image: ghcr.io/lsstdesc/txpipe-dev:latest steps: - name: Checkout repository @@ -55,6 +55,7 @@ jobs: - name: Test with pytest run: | + ceci --version pytest txpipe @@ -64,7 +65,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe:latest + image: ghcr.io/lsstdesc/txpipe-dev:latest steps: - name: Checkout repository @@ -101,7 +102,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe:latest + image: ghcr.io/lsstdesc/txpipe-dev:latest steps: - name: Checkout repository @@ -148,7 +149,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe:latest + image: ghcr.io/lsstdesc/txpipe-dev:latest steps: - name: Checkout repository @@ -187,7 +188,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe:latest + image: ghcr.io/lsstdesc/txpipe-dev:latest steps: - name: Checkout repository @@ -226,7 +227,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe:latest + image: ghcr.io/lsstdesc/txpipe-dev:latest steps: - name: Checkout repository @@ -262,7 +263,7 @@ jobs: runs-on: ubuntu-latest container: - image: ghcr.io/lsstdesc/txpipe:latest + image: ghcr.io/lsstdesc/txpipe-dev:latest steps: - name: Checkout repository From c82246e8fd95e605f57be017e04268bfc0bbb99d Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Fri, 9 Aug 2024 13:44:23 +0100 Subject: [PATCH 07/15] require tables>3.8 --- environment-nopip.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-nopip.yml b/environment-nopip.yml index 2b6fa9743..5e574b241 100644 --- a/environment-nopip.yml +++ b/environment-nopip.yml @@ -25,6 +25,7 @@ dependencies: - python=3.10.* - scikit-learn=1.5.* - scipy=1.13.* + - tables=3.9.* - tables-io-full=0.9.* - threadpoolctl=3.5.* - tjpcov=0.4.* From 3507eb0c5f0aa04587a17d42ddcac82edcf59e3c Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Fri, 9 Aug 2024 13:49:40 +0100 Subject: [PATCH 08/15] correct name to pytables --- environment-nopip.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-nopip.yml b/environment-nopip.yml index 5e574b241..1e5259584 100644 --- a/environment-nopip.yml +++ b/environment-nopip.yml @@ -25,7 +25,7 @@ dependencies: - python=3.10.* - scikit-learn=1.5.* - scipy=1.13.* - - tables=3.9.* + - pytables=3.9.* - tables-io-full=0.9.* - threadpoolctl=3.5.* - tjpcov=0.4.* From 4d10b22a66e0f5c516e5af0c7c031cf50cc75f85 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Fri, 9 Aug 2024 13:54:41 +0100 Subject: [PATCH 09/15] try to loosen pandas version --- environment-nopip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment-nopip.yml b/environment-nopip.yml index 1e5259584..de6d0b3a0 100644 --- a/environment-nopip.yml +++ b/environment-nopip.yml @@ -19,13 +19,13 @@ dependencies: - mpich>=4.* - namaster=2.0.* - numpy=1.26.* - - pandas=2.2.* + - pandas>=2.0 - psutil=5.9.* - pyccl=3.0.* - python=3.10.* - scikit-learn=1.5.* - scipy=1.13.* - - pytables=3.9.* + - pytables>=3.8.* - tables-io-full=0.9.* - threadpoolctl=3.5.* - tjpcov=0.4.* From cf57f29d47309ef6c15b3807c1208369bec0c6a9 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Fri, 9 Aug 2024 14:51:30 +0100 Subject: [PATCH 10/15] remove tables again --- environment-nopip.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment-nopip.yml b/environment-nopip.yml index de6d0b3a0..f19540ad9 100644 --- a/environment-nopip.yml +++ b/environment-nopip.yml @@ -25,7 +25,6 @@ dependencies: - python=3.10.* - scikit-learn=1.5.* - scipy=1.13.* - - pytables>=3.8.* - tables-io-full=0.9.* - threadpoolctl=3.5.* - tjpcov=0.4.* From f338b88b753e73d218d0e169e12fc7ca4d8e02a7 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Wed, 28 Aug 2024 15:10:08 +0100 Subject: [PATCH 11/15] update source-only yml --- examples/metadetect_source_only/pipeline.yml | 38 ++++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/examples/metadetect_source_only/pipeline.yml b/examples/metadetect_source_only/pipeline.yml index dc4a1e4f7..73aecf72d 100755 --- a/examples/metadetect_source_only/pipeline.yml +++ b/examples/metadetect_source_only/pipeline.yml @@ -1,14 +1,32 @@ # Stages to run stages: - - name: FlowCreator # Simulate a spectroscopic population - - name: GridSelection # Simulate a spectroscopic sample - - name: TXParqetToHDF # Convert the spec sample format - - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) + - name: FlowCreator # Simulate a spectroscopic population + aliases: + output: ideal_specz_catalog + model: flow + - name: GridSelection # Simulate a spectroscopic sample + aliases: + input: ideal_specz_catalog + output: specz_catalog_pq + - name: TXParqetToHDF # Convert the spec sample format + aliases: + input: specz_catalog_pq + output: spectroscopic_catalog + - name: PZRailSummarizeSource # Run the DIR method on the lens sample to find n(z) classname: PZRailSummarize + aliases: + tomography_catalog: shear_tomography_catalog + binned_catalog: binned_shear_catalog + model: source_direct_calibration_model + photoz_stack: shear_photoz_stack + photoz_realizations: source_photoz_realizations - name: TXSourceSelectorMetadetect # select and split objects into source bins - - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample + - name: NZDirInformerSource # Prepare the DIR method inputs for the source sample classname: NZDirInformer + aliases: + input: spectroscopic_catalog + model: source_direct_calibration_model - name: TXShearCalibration # Calibrate and split the source sample tomographically - name: TXSourceMaps # make source g1 and g2 maps - name: TXAuxiliarySourceMaps # make PSF and flag maps @@ -27,10 +45,16 @@ stages: - name: TXRoweStatistics # Compute and plot Rowe statistics threads_per_process: 2 - name: TXGalaxyStarShear # Compute and plot the star-galaxy shear cross-correlation - - name: TXPhotozPlotSource # Plot the bin n(z) + - name: TXPhotozPlotSource # Plot the bin n(z) classname: TXPhotozPlot - - name: PZRealizationsPlotSource # Plot n(z) realizations + aliases: + photoz_stack: shear_photoz_stack + nz_plot: nz_source + - name: PZRealizationsPlotSource # Plot n(z) realizations classname: PZRealizationsPlot + aliases: + photoz_realizations: source_photoz_realizations + photoz_realizations_plot: source_photoz_realizations_plot - name: TXConvergenceMaps # Make convergence kappa maps from g1, g2 maps - name: TXConvergenceMapPlots # Plot the convergence map # Disabled since not yet synchronised with new Treecorr MPI From 6b0aaf80ae8b6b4d72227bd0a5dece1483c5d859 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Wed, 28 Aug 2024 15:30:31 +0100 Subject: [PATCH 12/15] add cmake so lephare can build --- environment-nopip.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-nopip.yml b/environment-nopip.yml index f19540ad9..28bcad6a4 100644 --- a/environment-nopip.yml +++ b/environment-nopip.yml @@ -4,6 +4,7 @@ dependencies: - astropy=6.1.* - camb=1.5.* - cosmosis=3.7.* + - cmake - dask=2024.5.2 - dm-tree=0.1.8 - fitsio=1.2.* From 868daab515986bb23bf286ed74a3455de42f4f95 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Wed, 28 Aug 2024 15:53:52 +0100 Subject: [PATCH 13/15] Try newest macos --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37cbe9d44..5366b000e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,11 +284,11 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12] + os: [ubuntu-latest, macos-14] include: - os: ubuntu-latest INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget - - os: macos-12 + - os: macos-14 INSTALL_DEPS: brew update-reset && brew install wget steps: - name: Checkout repository From 01552d718ca48a95d25574bae697a2d061a5c456 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Thu, 29 Aug 2024 15:18:28 +0100 Subject: [PATCH 14/15] update two rail package versions --- environment-piponly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment-piponly.yml b/environment-piponly.yml index 765f617a5..d0d52d53c 100644 --- a/environment-piponly.yml +++ b/environment-piponly.yml @@ -16,7 +16,7 @@ dependencies: - pz-rail-astro-tools==1.0.4 - pz-rail-base==1.0.4 - pz-rail-bpz==1.0.1 - - pz-rail-cmnn==1.0.0 # no release for ceci2 yet + - pz-rail-cmnn==1.0.1 - pz-rail-dsps==0.0.3 # no pypi release for more than 0.3 - pz-rail-flexzboost==1.0.1 - pz-rail-fsps==1.0.1 @@ -24,4 +24,4 @@ dependencies: - pz-rail-lephare==0.2 # no release for ceci2 yet - cannot use - pz-rail-pzflow==1.0.1 - pz-rail-sklearn==1.0.1 - - pz-rail-som==1.0.0 # no release for ceci2 yet - cannot use + - pz-rail-som==1.0.1 From 2fe79a35bebb697eaf00d201c68e3495f719b519 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Tue, 3 Sep 2024 09:01:41 +0100 Subject: [PATCH 15/15] Switch to v1.0 of image in CI --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5366b000e..4b634a184 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-latest container: - image: ghcr.io/lsstdesc/txpipe-dev:latest + image: ghcr.io/lsstdesc/txpipe:v1.0 steps: - name: Checkout repository @@ -65,7 +65,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe-dev:latest + image: ghcr.io/lsstdesc/txpipe:v1.0 steps: - name: Checkout repository @@ -102,7 +102,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe-dev:latest + image: ghcr.io/lsstdesc/txpipe:v1.0 steps: - name: Checkout repository @@ -149,7 +149,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe-dev:latest + image: ghcr.io/lsstdesc/txpipe:v1.0 steps: - name: Checkout repository @@ -188,7 +188,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe-dev:latest + image: ghcr.io/lsstdesc/txpipe:v1.0 steps: - name: Checkout repository @@ -227,7 +227,7 @@ jobs: needs: Download_Data container: - image: ghcr.io/lsstdesc/txpipe-dev:latest + image: ghcr.io/lsstdesc/txpipe:v1.0 steps: - name: Checkout repository @@ -263,7 +263,7 @@ jobs: runs-on: ubuntu-latest container: - image: ghcr.io/lsstdesc/txpipe-dev:latest + image: ghcr.io/lsstdesc/txpipe:v1.0 steps: - name: Checkout repository