diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 95f7e2389..cc4ae4e93 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -13,24 +13,27 @@ jobs: name: Full Test Suite runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} + strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.8] + python-version: [3.9] steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v3 - name: Install Linux dependencies if: runner.os == 'Linux' run: sudo apt-get install libopenblas-dev - name: Install macOS Dependencies - shell: bash -l {0} if: runner.os == 'macOS' - run: | - brew install libomp + run: brew install libomp - name: Set up conda uses: conda-incubator/setup-miniconda@v2 @@ -40,12 +43,9 @@ jobs: auto-activate-base: true - name: Install package - shell: bash -l {0} - run: | - ./install_shapepipe --develop + run: ./install_shapepipe --develop - name: Run tests - shell: bash -l {0} run: | conda activate shapepipe python setup.py test diff --git a/docs/source/configuration.md b/docs/source/configuration.md index 2c3d2e99e..3336123c1 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -200,6 +200,9 @@ where ``X`` is an integer greater than or equal to ``1``. This feature can be co ```ini +[EXECUTION] +MODULE = module_a_runner, module_b_runner, module_b_runner + [MODULE_A_RUNNER_RUN_1] ... diff --git a/environment.yml b/environment.yml index 24f6dfd5f..9279402ef 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: dependencies: - python=3.9 - pip>=21.2.4 + - numpy==1.21.6 - astropy==5.0 - automake==1.16.2 - autoconf==2.69 @@ -15,6 +16,7 @@ dependencies: - numba==0.54.1 - pandas==1.4.1 - pip: + - cs_util==0.0.5 - mccd==1.2.3 - modopt==1.6.0 - PyQt5==5.15.6 diff --git a/example/cfis/config_GitFeGie_symlink.ini b/example/cfis/config_GitFeGie_symlink.ini index bc39a1e02..22e834533 100644 --- a/example/cfis/config_GitFeGie_symlink.ini +++ b/example/cfis/config_GitFeGie_symlink.ini @@ -66,7 +66,7 @@ NUMBERING_SCHEME = # Input path where original images are stored. Can be local path or vos url. # Single string or list of strings -INPUT_PATH = $HOME/astro/data/CFIS/tiles_DR2, $HOME/astro/data/CFIS/tiles_DR2 +INPUT_PATH = $SP_RUN/data_tiles, $SP_RUN/data_tiles # Input file pattern including tile number as dummy template INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight @@ -98,6 +98,7 @@ FILE_EXT = .fits # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 +COLNUM = 3 # Get exposures [GET_IMAGES_RUNNER_RUN_2] @@ -119,7 +120,7 @@ NUMBERING_SCHEME = -000-000 # Input path where original images are stored. Can be local path or vos url. # Single string or list of strings -INPUT_PATH = $HOME/astro/data/CFIS/pitcairn_DR2, $HOME/astro/data/CFIS/weights_DR2, $HOME/astro/data/CFIS/flags_DR2 +INPUT_PATH = $SP_RUN/data_exp, $SP_RUN/data_exp, $SP_RUN/data_exp # Input file pattern including tile number as dummy template INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag diff --git a/example/cfis/config_MaMa_onthefly.ini b/example/cfis/config_MaMa_onthefly.ini new file mode 100644 index 000000000..bde813940 --- /dev/null +++ b/example/cfis/config_MaMa_onthefly.ini @@ -0,0 +1,105 @@ +# ShapePipe configuration file for masking of tiles and exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_MaMa + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner, mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask tiles +[MASK_RUNNER_RUN_1] + +# Input directory, containing input files, single string or list of names +INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = CFIS_image, CFIS_weight + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = False + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = False + +# File name suffix for the output flag files (optional) +PREFIX = pipeline + +### Mask exposures +[MASK_RUNNER_RUN_2] + +# Parent module +INPUT_DIR = last:split_exp_runner + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = False + +# File name suffix for the output flag files (optional) +PREFIX = pipeline diff --git a/example/cfis/config_MaMa_save.ini b/example/cfis/config_MaMa_save.ini new file mode 100644 index 000000000..4bd1b00ef --- /dev/null +++ b/example/cfis/config_MaMa_save.ini @@ -0,0 +1,109 @@ +# ShapePipe configuration file for masking of tiles and exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_MaMa + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner, mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask tiles +[MASK_RUNNER_RUN_1] + +# Input directory, containing input files, single string or list of names +INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, star_cat_tiles + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = CFIS_image, CFIS_weight, star_cat + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .fits, .cat + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_tile_save.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = False + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = True + +# File name suffix for the output flag files (optional) +PREFIX = pipeline + +### Mask exposures +[MASK_RUNNER_RUN_2] + +# Parent module +INPUT_DIR = last:split_exp_runner, star_cat_exp + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +FILE_PATTERN = image, weight, flag, star_cat + +FILE_EXT = .fits, .fits, .fits, .cat + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_save.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = True + +# File name suffix for the output flag files (optional) +PREFIX = pipeline diff --git a/example/cfis/config_MsPl_mccd.ini b/example/cfis/config_MsPl_mccd.ini index ef8698e0f..5a2cc42c6 100644 --- a/example/cfis/config_MsPl_mccd.ini +++ b/example/cfis/config_MsPl_mccd.ini @@ -13,7 +13,7 @@ VERBOSE = True RUN_NAME = run_sp_MsPl # Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False +RUN_DATETIME = True ## ShapePipe execution options diff --git a/example/cfis/config_Pl_mccd.ini b/example/cfis/config_Pl_mccd.ini index 2739b8cde..a666d09dd 100644 --- a/example/cfis/config_Pl_mccd.ini +++ b/example/cfis/config_Pl_mccd.ini @@ -65,9 +65,12 @@ PLOT_MEANSHAPES = True X_GRID = 5 Y_GRID = 10 -# Optional: max values for elliptity and residual ellipticities -MAX_E = 0.05 -MAX_DE = 0.005 +# Optional: max values for focal plan plots (meanshape) +MAX_E = 0.1 +MAX_DE = 0.01 +MIN_R2 = 4.5 +MAX_R2 = 7 +MAX_DR2 = 0.03 PLOT_HISTOGRAMS = True REMOVE_OUTLIERS = False diff --git a/example/cfis/config_Pl_psfex.ini b/example/cfis/config_Pl_psfex.ini index bf1a062c1..c561f15a5 100644 --- a/example/cfis/config_Pl_psfex.ini +++ b/example/cfis/config_Pl_psfex.ini @@ -66,8 +66,11 @@ X_GRID = 5 Y_GRID = 10 # Optional: max values for elliptity and residual ellipticities -MAX_E = 0.05 -MAX_DE = 0.005 +MAX_E = 0.1 +MAX_DE = 0.01 +MIN_R2 = 4.5 +MAX_R2 = 7 +MAX_DR2 = 0.03 PLOT_HISTOGRAMS = True REMOVE_OUTLIERS = False diff --git a/example/cfis/config.mask b/example/cfis/config_onthefly.mask similarity index 100% rename from example/cfis/config.mask rename to example/cfis/config_onthefly.mask diff --git a/example/cfis/config_save.mask b/example/cfis/config_save.mask new file mode 100644 index 000000000..887110ff5 --- /dev/null +++ b/example/cfis/config_save.mask @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = ww +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +#CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_tile_Sx_exp_mccd.ini b/example/cfis/config_tile_Sx_exp_mccd.ini index 66e041e10..c195a8c80 100644 --- a/example/cfis/config_tile_Sx_exp_mccd.ini +++ b/example/cfis/config_tile_Sx_exp_mccd.ini @@ -253,13 +253,6 @@ NUMBERING_SCHEME = -0000000 [MCCD_PLOTS_RUNNER] -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - # Now MCCD has created a focal-plane PSF model, including all CCDS per images, # thus single-exposure files NUMBERING_SCHEME = -0000000 diff --git a/example/cfis/config_tile_Uz.ini b/example/cfis/config_tile_Uz.ini index aef899d0f..0bfe1a5cb 100644 --- a/example/cfis/config_tile_Uz.ini +++ b/example/cfis/config_tile_Uz.ini @@ -44,7 +44,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 +SMP_BATCH_SIZE = 16 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 diff --git a/example/cfis/config_tile.mask b/example/cfis/config_tile_onthefly.mask similarity index 100% rename from example/cfis/config_tile.mask rename to example/cfis/config_tile_onthefly.mask diff --git a/example/cfis/config_tile_save.mask b/example/cfis/config_tile_save.mask new file mode 100644 index 000000000..4b5fbb7f8 --- /dev/null +++ b/example/cfis/config_tile_save.mask @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = ww +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +#CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = False + +BORDER_WIDTH = 0 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis_simu/config.mask_simu b/example/cfis_simu/config.mask_simu new file mode 100644 index 000000000..c06a14a02 --- /dev/null +++ b/example/cfis_simu/config.mask_simu @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = ww +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = False + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = False + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = False + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = False + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis_simu/config_GitFeGie_symlink.ini b/example/cfis_simu/config_GitFeGie_symlink.ini new file mode 100644 index 000000000..533e6a959 --- /dev/null +++ b/example/cfis_simu/config_GitFeGie_symlink.ini @@ -0,0 +1,146 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_GitFeGie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner, find_exposures_runner, get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER_RUN_1] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_RUN/input_tiles, $SP_RUN/input_tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS_simu_image-000-000, CFIS_simu_weight-000-000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}-\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_simu_image-, CFIS_simu_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# Copy command options, optional +RETRIEVE_OPTIONS = -L + + +[FIND_EXPOSURES_RUNNER] + +INPUT_MODULE = get_images_runner_run_1 + +FILE_PATTERN = CFIS_simu_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +COLNUM = 2 + +EXP_PREFIX = simu_image- +# Get exposures +[GET_IMAGES_RUNNER_RUN_2] + +INPUT_MODULE = find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_RUN/input_exp/images,$SP_RUN/input_exp/images,$SP_RUN/input_exp/images + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = simu_image-0000000,simu_weight-0000000,simu_flag-0000000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits,.fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{7} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-,weight-,flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = -L diff --git a/example/cfis_simu/config_MCCD.ini b/example/cfis_simu/config_MCCD.ini new file mode 100644 index 000000000..bf5e6852e --- /dev/null +++ b/example/cfis_simu/config_MCCD.ini @@ -0,0 +1,120 @@ +# Configuration file for the MCCD method + +[INPUTS] +INPUT_DIR = . +PREPROCESSED_OUTPUT_DIR = ./output +OUTPUT_DIR = ./output +INPUT_REGEX_FILE_PATTERN = star_split_ratio_80-*-*.fits +INPUT_SEPARATOR = - +MIN_N_STARS = 20 +OUTLIER_STD_MAX = 100. +USE_SNR_WEIGHTS = False + +[INSTANCE] +N_COMP_LOC = 8 +D_COMP_GLOB = 8 +KSIG_LOC = 0.00 +KSIG_GLOB = 0.00 +FILTER_PATH = None +D_HYB_LOC = 2 +MIN_D_COMP_GLOB = None +RMSE_THRESH = 1.25 +CCD_STAR_THRESH = 0.15 +FP_GEOMETRY = CFIS + +[FIT] +LOC_MODEL = hybrid +PSF_SIZE = 6.2 +PSF_SIZE_TYPE = R2 +N_EIGENVECTS = 5 +N_ITER_RCA = 1 +N_ITER_GLOB = 2 +N_ITER_LOC = 2 +NB_SUBITER_S_LOC = 300 +NB_SUBITER_A_LOC = 400 +NB_SUBITER_S_GLOB = 100 +NB_SUBITER_A_GLOB = 200 + +[VALIDATION] +VAL_DATA_INPUT_DIR = . +VAL_PREPROCESSED_OUTPUT_DIR = ./output +VAL_MODEL_INPUT_DIR = ./output +VAL_OUTPUT_DIR = ./output +VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits +VAL_SEPARATOR = - +APPLY_DEGRADATION = True +MCCD_DEBUG = False +GLOBAL_POL_INTERP = False + + +# Parameter description: +# +# +# [INPUTS] +# INPUT_DIR : (Required) Must be a valid directory containing the input +# MCCD files. +# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should +# follow regex (regular expression) standards. +# INPUT_SEPARATOR : Separator of the different fields in the filename, +# ie sexcat[SEP]catalog_id[SEP]CCD_id.fits +# MIN_N_STARS : Minimum number of stars to keep a CCD for the training. +# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection. +# Should not be too low as a hihg quantity of low quality +# stars will be rejected. ie 9 is a conservative rejection. +# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will +# be used. +# For now, it needs the SNR estimations from SExtractor. +# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the +# preprocessed input files. +# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. +# The constructed models will be saved. +# +# +# [INSTANCE] +# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly, +# will be the max degree D of the polynomial. +# D_COMP_GLOB : Max degree of the global polynomial model. +# KSIG_LOC : Denoising parameter of the local model. +# ie 1 is a normal denoising, 3 is a hard denoising. +# KSIG_GLOB : Denoising parameter of the global model. +# ie 1 is a normal denoising, 3 is a hard denoising. +# FILTER_PATH : Path for predefined filters. +# +# +# [FIT] +# LOC_MODEL : Defines the type of local model to use, it can be: 'rca', +# 'poly' or 'hybrid'. +# When the poly model is used, N_COMP_LOC should be used +# as the D_LOC (max degree of the poly model) +# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways. +# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma +# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint +# construction. +# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between +# global and local estimations. +# N_ITER_GLOB : Number of epochs for each global optimization. Alternations +# between A_GLOB and S_GLOB. +# N_ITER_LOC : Number of epochs for each local optimization. Alternations +# between the different A_LOC and S_LOC. +# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC. +# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC. +# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB. +# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB. +# +# +# [VALIDATION] +# MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the +# saved trained models. +# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the +# validation input data (test dataset). +# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation. +# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation. +# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the +# validation outputs, test PSFs and interpolated PSFs. +# APPLY_DEGRADATION : Whether the PSF models should be degraded +# (sampling/shifts/flux) to match stars; use True if you +# plan on making pixel-based comparisons (residuals etc.). +# MCCD_DEBUG : Debug mode. Returns the local and global contributions. +# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model +# instead of RBF kernel interpolation. +# diff --git a/example/cfis/config_MaMa.ini b/example/cfis_simu/config_MaMa.ini similarity index 91% rename from example/cfis/config_MaMa.ini rename to example/cfis_simu/config_MaMa.ini index cc4b7f9c4..4a13e7946 100644 --- a/example/cfis/config_MaMa.ini +++ b/example/cfis_simu/config_MaMa.ini @@ -56,20 +56,20 @@ TIMEOUT = 96:00:00 [MASK_RUNNER_RUN_1] # Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner +INPUT_DIR = last:get_images_runner_run_1, last:get_images_runner_run_2 # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 # Input file pattern(s), list of strings with length matching number of expected input file types # Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight +FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight # FILE_EXT (optional) list of string extensions to identify input files FILE_EXT = .fits, .fits # Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile.mask +MASK_CONFIG_PATH = $SP_CONFIG/config_tile.mask_simu # External mask file flag, use if True, otherwise ignore USE_EXT_FLAG = False @@ -92,7 +92,7 @@ INPUT_DIR = last:split_exp_runner NUMBERING_SCHEME = -0000000-0 # Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config.mask +MASK_CONFIG_PATH = $SP_CONFIG/config.mask_simu # External mask file flag, use if True, otherwise ignore USE_EXT_FLAG = True diff --git a/example/cfis_simu/config_exp_SpMh.ini b/example/cfis_simu/config_exp_SpMh.ini new file mode 100644 index 000000000..58dc56e6e --- /dev/null +++ b/example/cfis_simu/config_exp_SpMh.ini @@ -0,0 +1,84 @@ +# ShapePipe configuration file for single-exposures, +# split images, merge headers + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SpMh + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner, merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = last:get_images_runner_run_2 + +# Matches compressed single-exposure files +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are save as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 + + +[MERGE_HEADERS_RUNNER] + +FILE_PATTERN = headers + +FILE_EXT = .npy + +# Single-exposure numbering scheme +NUMBERING_SCHEME = -0000000 + +OUTPUT_PATH = $SP_RUN/output diff --git a/example/cfis_simu/config_make_cat_mccd.ini b/example/cfis_simu/config_make_cat_mccd.ini new file mode 100644 index 000000000..5aa33d272 --- /dev/null +++ b/example/cfis_simu/config_make_cat_mccd.ini @@ -0,0 +1,77 @@ +# ShapePipe post-run configuration file: create final catalogs + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Mc + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = last:sextractor_runner_run_1, last:spread_model_runner, last:mccd_interp_runner, last:merge_sep_cats_runner + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix +#, galsim + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .fits, .sqlite, .fits +#, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SM_DO_CLASSIFICATION = True +SM_STAR_THRESH = 0.003 +SM_GAL_THRESH = 0.01 + +SHAPE_MEASUREMENT_TYPE = ngmix +#, galsim diff --git a/example/cfis_simu/config_merge_sep_cats_template.ini b/example/cfis_simu/config_merge_sep_cats_template.ini new file mode 100644 index 000000000..fb848475f --- /dev/null +++ b/example/cfis_simu/config_merge_sep_cats_template.ini @@ -0,0 +1,75 @@ +# ShapePipe post-run configuration file: merge separated catalogues + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Ms + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_sep_cats_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output +#, ./output/run_sp_tile_ngmix_Ng1u/galsim_shapes_v2_runner/output + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_SEP_CATS_RUNNER] + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = ngmix +#, galsim + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits +#, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +NUMBERING_SCHEME = -000-000 + +# WARNING (optional, default is 'error'). Use 'always'/'ignore' to +# display/ignore warnings, and not raise error +WARNING = always + +# Maximum number of separated catalogues per input +N_SPLIT_MAX = 0 diff --git a/example/cfis_simu/config_tile.mask_simu b/example/cfis_simu/config_tile.mask_simu new file mode 100644 index 000000000..2c0a53e04 --- /dev/null +++ b/example/cfis_simu/config_tile.mask_simu @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = ww +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 1 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = False + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = False + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = False + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = False + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis_simu/config_tile_MiViSmVi.ini b/example/cfis_simu/config_tile_MiViSmVi.ini new file mode 100644 index 000000000..dfb584bda --- /dev/null +++ b/example/cfis_simu/config_tile_MiViSmVi.ini @@ -0,0 +1,181 @@ +# ShapePipe configuration file for tiles, from detection up to shape measurement. +# MCCD PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_MiViSmVi + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mccd_interp_runner, + vignetmaker_runner, spread_model_runner, + vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 12 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MCCD_INTERP_RUNNER] + +INPUT_DIR = last:sextractor_runner_run_1 + +FILE_PATTERN = sexcat + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Column names of position parameters +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# If True, measure and store ellipticity of the PSF +GET_SHAPES = True + +PSF_MODEL_DIR = mccd_fit_val_runner + +PSF_MODEL_PATTERN = fitted_model + +PSF_MODEL_SEPARATOR = - + +# Multi-epoch mode: Path to file with single-exposure WCS header information +ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + + +[VIGNETMAKER_RUNNER_RUN_1] + +# Create vignets for tile weights + +INPUT_DIR = last:sextractor_runner_run_1, last:get_images_runner_run_1 + +FILE_PATTERN = sexcat, CFIS_simu_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[SPREAD_MODEL_RUNNER] + +INPUT_DIR = last:sextractor_runner_run_1, last:mccd_interp_runner, last:vignetmaker_runner_run_1 + +FILE_PATTERN = sexcat, galaxy_psf, weight_vignet + +FILE_EXT = .fits, .sqlite, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +# Output mode: +# new: create a new catalog with: [number, mag, sm, sm_err] +# add: create a copy of the input SExtractor with the column sm and sm_err +OUTPUT_MODE = new + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = last:sextractor_runner_run_1 + +FILE_PATTERN = sexcat + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs +ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2 +ME_IMAGE_PATTERN = flag, image, weight, background +ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite diff --git a/example/cfis_simu/config_tile_Ng_template.ini b/example/cfis_simu/config_tile_Ng_template.ini new file mode 100644 index 000000000..f97e8bc9d --- /dev/null +++ b/example/cfis_simu/config_tile_Ng_template.ini @@ -0,0 +1,99 @@ +# ShapePipe configuration file for tiles: ngmix + KSB + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_ngmix_NgXu + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +INPUT_DIR = last:sextractor_runner_run_1,last:MCCD_interp_runner,last:vignetmaker_runner_run_2 + +FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Multi-epoch mode: Path to file with single-exposure WCS header information +LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + +# Magnitude zero-point +MAG_ZP = 30.0 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +ID_OBJ_MIN = -1 +ID_OBJ_MAX = -1 + + +# Moment-based (KSB) shapes with galsim +[GALSIM_SHAPES_V2_RUNNER] + +INPUT_DIR = last:sextractor_runner_run_2, last:vignetmaker_runner_run_1, last:X_interp_runner,last:vignetmaker_runner_run_2 + +FILE_PATTERN = sexcat, weight_vignet, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet + +FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Multi-epoch mode: Path to file with single-exposure WCS header information +LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + +# Magnitude zero-point +MAG_ZP = 30.0 + +ID_OBJ_MIN = -1 +ID_OBJ_MAX = -1 diff --git a/example/cfis_simu/config_tile_Sx_exp_mccd.ini b/example/cfis_simu/config_tile_Sx_exp_mccd.ini new file mode 100644 index 000000000..d78a72f5f --- /dev/null +++ b/example/cfis_simu/config_tile_Sx_exp_mccd.ini @@ -0,0 +1,285 @@ +# ShapePipe configuration file for single-exposures, MCCD PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, sextractor_runner, setools_runner, + mccd_preprocessing_runner, mccd_fit_val_runner, + merge_starcat_runner, mccd_plots_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 4 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +## Detection on tile +[SEXTRACTOR_RUNNER_RUN_1] + +INPUT_DIR = last:get_images_runner_run_1, last:get_images_runner_run_1, last:mask_runner_run_1 + +FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight, pipeline_flag + +FILE_EXT = .fits, .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +#CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# Multi-epoch mode: Path to file with single-exposure WCS header information +LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 + + +## Detection on single exposures +[SEXTRACTOR_RUNNER_RUN_2] + +INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2 + +# Input from two modules +INPUT_MODULE = split_exp_runner, mask_runner_run_2 + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, flag + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) if True +DETECTION_WEIGHT = False + +# Se to True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_MODULE = sextractor_runner_run_2 + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[MCCD_PREPROCESSING_RUNNER] + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +INPUT_DIR = last:setools_runner + +# Input are individual CCDs, thus single-exposure single-HDU images +NUMBERING_SCHEME = -0000000-0 + +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 + +FILE_EXT = .fits, .fits + + +[MCCD_FIT_VAL_RUNNER] + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +NUMBERING_SCHEME = -0000000 + + +[MERGE_STARCAT_RUNNER] + +INPUT_DIR = last:mccd_fit_val_runner + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +PSF_MODEL = mccd + +NUMBERING_SCHEME = -0000000 + + +[MCCD_PLOTS_RUNNER] + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +# Now MCCD has created a focal-plane PSF model, including all CCDS per images, +# thus single-exposure files +NUMBERING_SCHEME = -0000000 + +PSF = mccd + +PLOT_MEANSHAPES = True + +# X_GRID, Y_GRID: correspond to the number of bins in each direction of each +# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. +X_GRID = 5 +Y_GRID = 10 + +PLOT_HISTOGRAMS = True + +# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape +# before drawing the plots. +REMOVE_OUTLIERS = False + +PLOT_RHO_STATS = False + +# RHO_STATS_STYLE: can be 'HSC' or 'DES' +RHO_STATS_STYLE = HSC diff --git a/example/cfis_simu/config_tile_Sx_exp_psfex.ini b/example/cfis_simu/config_tile_Sx_exp_psfex.ini new file mode 100644 index 000000000..1271eb012 --- /dev/null +++ b/example/cfis_simu/config_tile_Sx_exp_psfex.ini @@ -0,0 +1,248 @@ +# ShapePipe configuration file for single-exposures. PSFex PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 40 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER_RUN_1] + +INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1 + +INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 + +FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag + +FILE_EXT = .fits, .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# Multi-epoch mode: Path to file with single-exposure WCS header information +LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 + + +[SEXTRACTOR_RUNNER_RUN_2] + +# Somehow this works but not +# - omitting +# - $SP_RUN/output +#INPUT_DIR = . + +# Input from two modules +INPUT_MODULE = split_exp_runner, mask_runner + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_MODULE = sextractor_runner_run_2 + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[PSFEX_RUNNER] + +# Use 80% sample for PSF model +FILE_PATTERN = star_split_ratio_80 + +NUMBERING_SCHEME = -0000000-0 + +# Path to executable for the PSF model (optional) +EXEC_PATH = psfex + +# Default psfex configuration file +DOT_PSFEX_FILE = $SP_CONFIG/default.psfex + + +[PSFEX_INTERP_RUNNER] + +# Use 20% sample for PSF validation +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat + +FILE_EXT = .psf, .fits, .cat + +NUMBERING_SCHEME = -0000000-0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = VALIDATION + +# Column names of position parameters +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# If True, measure and store ellipticity of the PSF (using moments) +GET_SHAPES = True + +# Minimum number of stars per CCD for PSF model to be computed +STAR_THRESH = 22 + +# Maximum chi^2 for PSF model to be computed on CCD +CHI2_THRESH = 2 diff --git a/example/cfis_simu/default.conv b/example/cfis_simu/default.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/example/cfis_simu/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/example/cfis_simu/default.param b/example/cfis_simu/default.param new file mode 100644 index 000000000..6521a94d5 --- /dev/null +++ b/example/cfis_simu/default.param @@ -0,0 +1,69 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +#!!! REQUIRE FLAG_IMAGE !!! +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +#VECTOR_ASSOC #ASSOCiated parameter vector +#NUMBER_ASSOC #Number of ASSOCiated IDs + +#SPREAD_MODEL +#SPREADERR_MODEL +#FWHMPSF_IMAGE diff --git a/example/cfis_simu/default.psfex b/example/cfis_simu/default.psfex new file mode 100644 index 000000000..15af2dc29 --- /dev/null +++ b/example/cfis_simu/default.psfex @@ -0,0 +1,85 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2017-11-30 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +BASIS_NAME basis.fits # Basis filename (FITS data-cube) +BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter +NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT + # or PCA_COMMON +NEWBASIS_NUMBER 8 # Number of new basis vectors +PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) +PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 51,51 # Image size of the PSF model +PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? +MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 2 # Polynom degree for each group +PSFVAR_NSNAP 9 # Number of PSF snapshots per axis +HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON +STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? + +BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? +BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed + +#----------------------- PSF homogeneisation kernel -------------------------- + +HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE +HOMOBASIS_NUMBER 10 # Kernel basis number or parameter +HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter +HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF +HOMOKERNEL_DIR # Where to write kernels (empty=same as input) +HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels + +#----------------------------- Output catalogs ------------------------------- + +OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV PNG # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS +CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +# Note: Check-image types can be set the ShapePipe config file, psfex_runner section +####### +#CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames +#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? + +#----------------------------- Miscellaneous --------------------------------- + +PSF_SUFFIX .psf # Filename extension for output PSF filename +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic diff --git a/example/cfis_simu/default_exp.sex b/example/cfis_simu/default_exp.sex new file mode 100644 index 000000000..b87275ecb --- /dev/null +++ b/example/cfis_simu/default_exp.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE AUTO # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/example/cfis_simu/default_tile.sex b/example/cfis_simu/default_tile.sex new file mode 100644 index 000000000..ff3b25213 --- /dev/null +++ b/example/cfis_simu/default_tile.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE MANUAL # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/example/cfis_simu/job_sp_simu.bash b/example/cfis_simu/job_sp_simu.bash new file mode 100755 index 000000000..09915cae3 --- /dev/null +++ b/example/cfis_simu/job_sp_simu.bash @@ -0,0 +1,462 @@ +#!/usr/bin/env bash + +# Name: job_sp_simu.bash +# Description: General script to process one or more tiles +# with all contributing exposures. +# This works as job submission script. +# called in interactive mode on a virtual +# machine. +# Author: Martin Kilbinger +# Date: v1.0 11/2020 +# v1.1 01/2021 + + +# VM home, required for canfar run. +## On other machines set to $HOME +export VM_HOME=/home/ubuntu +if [ ! -d "$VM_HOME" ]; then + export VM_HOME=$HOME +fi + +# Command line arguments +## Default values +job=255 +#config_dir='vos:cfis/cosmostat/kilbinger/cfis' +config_dir="$VM_HOME/SP_simu" +psf='mccd' +#retrieve='vos' +retrieve=symlink +results="$VM_HOME/SP_simu" +nsh_step=3200 +nsh_max=-1 +nsh_jobs=8 + +## Help string +usage="Usage: $(basename "$0") [OPTIONS] TILE_ID_1 [TILE_ID_2 [...]] +\n\nOptions:\n + -h\tthis message\n + -j, --job JOB\tRunning JOB, bit-coded\n + \t 1: retrieve images (online if method=vos)\n + \t 2: prepare images (offline)\n + \t 4: mask (online)\n + \t 8: detection of galaxies on tiles; processing of stars on exposures (offline)\n + \t 16: galaxy selection on tiles (offline)\n + \t 32: shapes and morphology (offline)\n + \t 64: paste catalogues (offline)\n + \t 128: upload results (online)\n + -c, --config_dir DIR\n + \t config file directory, default='$config_dir'\n + -p, --psf MODEL\n + \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n + -r, --retrieve METHOD\n + \tmethod to retrieve images, one in ['vos'|'symlink]', default='$retrieve'\n + -o, --output_dir\n + \toutput (upload) directory on vos:cfis, default='$results'\n + --nsh_jobs NJOB\n + \tnumber of shape measurement parallel jobs, default=$nsh_jobs\n + --nsh_step NSTEP\n + \tnumber of objects per parallel shape module call, \n + \t default: $nsh_step\n + --nsh_max NMAX\n + \tmax number of objects per parallel shape module call, \n + \t default: unlimited; has precedent over --nsh_step\n + TILE_ID_i\n + \ttile ID(s), e.g. 283.247 214.242\n +" + +## Help if no arguments +if [ -z $1 ]; then + echo -ne $usage + exit 1 +fi + +## Parse command line +TILE_ARR=() +while [ $# -gt 0 ]; do + case "$1" in + -h) + echo -ne $usage + exit 0 + ;; + -j|--job) + job="$2" + shift + ;; + -c|--config_dir) + config_dir="$2" + shift + ;; + -p|--psf) + psf="$2" + shift + ;; + -r|--retrieve) + retrieve="$2" + shift + ;; + -o|--output_dir) + results="$2" + shift + ;; + --nsh_max) + nsh_max="$2" + shift + ;; + --nsh_step) + nsh_step="$2" + shift + ;; + --nsh_jobs) + nsh_jobs="$2" + shift + ;; + *) + TILE_ARR+=("$1") + ;; + esac + shift +done + +## Check options +if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then + echo "PSF (option -p) needs to be 'psfex' or 'mccd'" + exit 2 +fi +n_tile=${#TILE_ARR[@]} +if [ "$n_tile" == "0" ]; then + echo "No tile ID given" + exit 3 +fi +if [ $nsh_max != -1 ]; then + nsh_step=$nsh_max +fi + +# For tar archives. Should be unique to each job +export ID=`echo ${TILE_ARR[@]} | tr ' ' '_'` + +## Paths + +# SExtractor library bug work-around +export PATH="$PATH:$VM_HOME/bin" + +## Path variables used in shapepipe config files + +# Run path and location of input image directories +export SP_RUN=`pwd` + +# Config file path +export SP_CONFIG=$SP_RUN +export SP_CONFIG_MOD=$SP_RUN + + +## Other variables + +# Input tile numbers ASCII file +export TILE_NUMBERS_PATH=tile_numbers.txt + +# Output +OUTPUT=$SP_RUN/output + +# For tar archives +output_rel=`realpath --relative-to=. $OUTPUT` + +# Stop on error, default=1 +STOP=1 + +# Verbose mode (1: verbose, 0: quiet) +VERBOSE=1 + +# VCP options +export CERTFILE=$VM_HOME/.ssl/cadcproxy.pem +export VCP="vcp --certfile=$CERTFILE" + + +## Functions + +# Print string, executes command, and prints return value. +function command () { + cmd=$1 + str=$2 + + #RED='\033[0;31m' + #GREEN='\033[0;32m' + #NC='\033[0m' # No Color + # Color escape characters show up in log files + RED='' + GREEN='' + NC='' + + + if [ $# == 2 ]; then + if [ $VERBOSE == 1 ]; then + echo "$str: running '$cmd'" + fi + $cmd + else + if [ $VERBOSE == 1 ]; then + echo "$str: running '$cmd $4 \"$5 $6\"'" + fi + $cmd $4 "$5 $6" + fi + res=$? + + if [ $VERBOSE == 1 ]; then + if [ $res == 0 ]; then + echo -e "${GREEN}success, return value = $res${NC}" + else + echo -e "${RED}error, return value = $res${NC}" + if [ $STOP == 1 ]; then + echo "${RED}exiting 'canfar_sp.bash', error in command '$cmd'${NC}" + exit $res + else + echo "${RED}continuing 'canfar_sp.bash', error in command '$cmd'${NC}" + fi + fi + fi + + #return $res +} + +# Run shapepipe command. If error occurs, upload sp log files before stopping script. +command_sp() { + cmd=$1 + str=$2 + + command "$1" "$2" +} + +# Tar and upload files to vos +function upload() { + base=$1 + shift + ID=$1 + shift + verbose=$1 + shift + upl=("$@") + + echo "Counting upload files" + n_upl=(`ls -l ${upl[@]} | wc`) + if [ $n_upl == 0 ]; then + if [ $STOP == 1 ]; then + echo "Exiting script, no file found for '$base' tar ball" + exit 3 + fi + fi + tar czf ${base}_${ID}.tgz ${upl[@]} + command "$VCP ${base}_${ID}.tgz vos:cfis/$results" "Upload tar ball" +} + +# Upload log files +function upload_logs() { + id=$1 + verbose=$2 + + upl="$output_rel/*/*/logs $output_rel/*/logs" + upload "logs" "$id" "$verbose" "${upl[@]}" +} + +# Print script variables +function print_env() { + echo "*** Environment ***" + echo "Data:" + echo " TILE_ARR=${TILE_ARR[@]}" + echo "Paths:" + echo " VM_HOME=$VM_HOME" + echo " SP_RUN=$SP_RUN" + echo " TILE_NUMBERS_PATH=$TILE_NUMBERS_PATH" + echo " OUTPUT=$OUTPUT" + echo " SP_CONFIG=$SP_CONFIG" + echo "Other variables:" + echo " VCP=$VCP" + echo " CERTFILE=$CERTFILE" + echo "***" +} + + +### Start ### + +echo "Start" + +echo "Processing $n_tile tile(s)" + +# Create input and output directories +mkdir -p $SP_RUN +cd $SP_RUN +mkdir -p $OUTPUT + +# Processing + +## Retrieve config files and images (online if retrieve=vos) +(( do_job= $job & 1 )) +if [[ $do_job != 0 ]]; then + + # Write tile numbers to ASCII input file + rm -rf $TILE_NUMBERS_PATH + for TILE in ${TILE_ARR[@]}; do + echo $TILE >> $TILE_NUMBERS_PATH + done + + ### Retrieve config files + if [[ $config_dir == *"vos:"* ]]; then + command_sp "$VCP $config_dir ." "Retrieve shapepipe config files" + else + if [[ ! -L cfis ]]; then + command_sp "ln -s $config_dir cfis" "Retrieve shapepipe config files" + fi + fi + + ### Retrieve files + command_sp "shapepipe_run -c $SP_CONFIG/config_GitFeGie_$retrieve.ini" "Retrieve images" + +fi + +## Prepare images (offline) +(( do_job= $job & 2 )) +if [[ $do_job != 0 ]]; then + + ### Uncompress tile weights + #command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Uz.ini" "Run shapepipe (uncompress tile weights)" + + ### Split images into single-HDU files, merge headers for WCS info + command_sp "shapepipe_run -c $SP_CONFIG/config_exp_SpMh.ini" "Run shapepipe (split images, merge headers)" + +fi + +## Mask tiles and exposures: add star, halo, and Messier object masks (online) +(( do_job= $job & 4 )) +if [[ $do_job != 0 ]]; then + + ### Mask tiles and exposures + command_sp "shapepipe_run -c $SP_CONFIG/config_MaMa.ini" "Run shapepipe (mask)" + +fi + + +## Remaining exposure processing (offline) +(( do_job= $job & 8 )) +if [[ $do_job != 0 ]]; then + + ### Star detection, selection, PSF model. setools can exit with an error for CCD with insufficient stars, + ### the script should continue + STOP=0 + command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Sx_exp_${psf}.ini" "Run shapepipe (tile detection, exp $psf)" + STOP=1 + +fi + +## Process tiles up to shape measurement +(( do_job= $job & 16 )) +if [[ $do_job != 0 ]]; then + + ### PSF model letter: 'P' (psfex) or 'M' (mccd) + letter=${psf:0:1} + Letter=${letter^} + command_sp "shapepipe_run -c $SP_CONFIG/config_tile_${Letter}iViSmVi.ini" "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" + +fi + +## Shape measurement (offline) +(( do_job= $job & 32 )) +if [[ $do_job != 0 ]]; then + + ### Prepare config files + mkdir -p $SP_CONFIG_MOD + n_min=0 + n_max=$((nsh_step - 1)) + for k in $(seq 1 $nsh_jobs); do + cat $SP_CONFIG/config_tile_Ng_template.ini | \ + perl -ane \ + 's/(ID_OBJ_MIN =) X/$1 '$n_min'/; s/(ID_OBJ_MAX =) X/$1 '$n_max'/; s/NgXu/Ng'$k'u/; s/X_interp/'$psf'_interp/g; print' \ + > $SP_CONFIG_MOD/config_tile_Ng${k}u.ini + n_min=$((n_min + nsh_step)) + if [ "$k" == $((nsh_jobs - 1)) ] && [ $nsh_max == -1 ]; then + n_max=-1 + else + n_max=$((n_min + nsh_step - 1)) + fi + done + + ### Shapes, run $nsh_jobs parallel processes + VERBOSE=0 + for k in $(seq 1 $nsh_jobs); do + command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix+galsim $k)" & + done + wait + VERBOSE=1 + +fi + +## Create final catalogues (offline) +(( do_job= $job & 64 )) +if [[ $do_job != 0 ]]; then + + cat $SP_CONFIG/config_merge_sep_cats_template.ini | \ + perl -ane \ + 's/(N_SPLIT_MAX =) X/$1 '$nsh_jobs'/; print' \ + > $SP_CONFIG_MOD/config_merge_sep_cats.ini + + ### Merge separated shapes catalogues + command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" "Run shapepipe (tile: merge sep cats)" "$VERBOSE" "$ID" + + ### Merge all relevant information into final catalogue + command_sp "shapepipe_run -c $SP_CONFIG/config_make_cat_$psf.ini" "Run shapepipe (tile: create final cat $psf)" "$VERBOSE" "$ID" + +fi + +## Upload results (online) +(( do_job= $job & 128 )) +if [[ $do_job != 0 ]]; then + + ### module and pipeline log files + upload_logs "$ID" "$VERBOSE" + + ### Final shape catalog + ### pipeline_flags are the tile masks, for random cats + ### SETools masks (selection), stats and plots + ### ${psf}_interp_exp for diagnostics, validation with leakage, + ### validation with residuals, rho stats + + NAMES=( + "final_cat" + "pipeline_flag" + "setools_mask" + "setools_stat" + "setools_plot" + ) + DIRS=( + "*/make_cat_runner/output" + "*/mask_runner_run_1/output" + "*/setools_runner/output/mask" + "*/setools_runner/output/stat" + "*/setools_runner/output/plot" + ) + PATTERNS=( + "final_cat-*" + "pipeline_flag-???-???*" + "*" + "*" + "*" + ) + + # PSF validation + pattern="validation_psf-*" + if [ "$psf" == "psfex" ]; then + name="psfex_interp_exp" + dir="*/psfex_interp_runner/output" + else + name="mccd_fit_val_runner" + dir="*/mccd_fit_val_runner/output" + fi + upl=$output_rel/$dir/$pattern + upload "$name" "$ID" "$VERBOSE" "${upl[@]}" + + for n in "${!NAMES[@]}"; do + name=${NAMES[$n]} + dir=${DIRS[$n]} + pattern=${PATTERNS[$n]} + upl=$output_rel/$dir/$pattern + upload "$name" "$ID" "$VERBOSE" "${upl[@]}" + done + +fi diff --git a/example/cfis_simu/mask_default/MEGAPRIME_star_i_13.8.reg b/example/cfis_simu/mask_default/MEGAPRIME_star_i_13.8.reg new file mode 100644 index 000000000..4e4164aaf --- /dev/null +++ b/example/cfis_simu/mask_default/MEGAPRIME_star_i_13.8.reg @@ -0,0 +1,24 @@ +-11.5 68 +-6 186.5 +7 188 +10 64.5 +31 55 +50 38.5 +56.5 11.5 +188 8 +192 -4 +59.5 -11.5 +45 -33 +13.5 -64 +5 -154 +-6 -155 +-11 -64.5 +-40 -44.5 +-51.5 -30.5 +-62.5 -22.5 +-68 -9.5 +-177 -2 +-176 3 +-78 12.5 +-67.5 14.5 +-38.5 50 diff --git a/example/cfis_simu/mask_default/Messier_catalog.npy b/example/cfis_simu/mask_default/Messier_catalog.npy new file mode 100644 index 000000000..ef07eb032 Binary files /dev/null and b/example/cfis_simu/mask_default/Messier_catalog.npy differ diff --git a/example/cfis_simu/mask_default/Messier_catalog_updated.fits b/example/cfis_simu/mask_default/Messier_catalog_updated.fits new file mode 100644 index 000000000..6a9f00096 Binary files /dev/null and b/example/cfis_simu/mask_default/Messier_catalog_updated.fits differ diff --git a/example/cfis_simu/mask_default/default.ww b/example/cfis_simu/mask_default/default.ww new file mode 100644 index 000000000..c2797f904 --- /dev/null +++ b/example/cfis_simu/mask_default/default.ww @@ -0,0 +1,40 @@ +#--------------------------------- Weights ------------------------------------ + +WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) + +WEIGHT_MIN 0. # Pixel below those thresholds will be flagged +WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged +WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels + +#---------------------------------- Flags ------------------------------------- + +FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) + +FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels +FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs +FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits + +#---------------------------------- Polygons ---------------------------------- + +POLY_NAMES "" # Filename(s) of input DS9 regions +POLY_OUTFLAGS # FLAG values for polygon masks +POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks +POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? + +#---------------------------------- Output ------------------------------------ + +OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename +OUTFLAG_NAME flag.fits # Output FLAG-map filename + +#----------------------------- Miscellaneous --------------------------------- + +GETAREA N # Compute area for flags and weights (Y/N)? +GETAREA_WEIGHT 0.0 # Weight threshold for area computation +GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area +MEMORY_BUFSIZE 256 # Buffer size in lines +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME ww.xml # Filename for XML output +XSL_URL file:///usr/local/share/weightwatcher/ww.xsl + # Filename for XSL style-sheet +NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/example/cfis_simu/mask_default/halo_mask.reg b/example/cfis_simu/mask_default/halo_mask.reg new file mode 100644 index 000000000..c44f25167 --- /dev/null +++ b/example/cfis_simu/mask_default/halo_mask.reg @@ -0,0 +1,50 @@ + 274.66813 -1.25966 + 272.54579 32.47406 + 266.21222 65.67579 + 255.76731 97.82190 + 241.37579 128.40544 + 223.26462 156.94408 + 201.71942 182.98775 + 177.07997 206.12573 + 149.73486 225.99312 + 120.11532 242.27660 + 88.68848 254.71937 + 55.94996 263.12519 + 22.41606 267.36151 + -11.38436 267.36151 + -44.91826 263.12519 + -77.65678 254.71937 +-109.08362 242.27660 +-138.70315 225.99312 +-166.04827 206.12573 +-190.68772 182.98775 +-212.23292 156.94408 +-230.34409 128.40544 +-244.73561 97.82190 +-255.18052 65.67579 +-261.51409 32.47406 +-263.63643 -1.25966 +-261.51409 -34.99339 +-255.18052 -68.19511 +-244.73561 -100.34123 +-230.34409 -130.92476 +-212.23292 -159.46341 +-190.68772 -185.50708 +-166.04827 -208.64506 +-138.70315 -228.51245 +-109.08362 -244.79593 + -77.65678 -257.23870 + -44.91826 -265.64452 + -11.38436 -269.88084 + 22.41606 -269.88084 + 55.94996 -265.64452 + 88.68848 -257.23870 + 120.11532 -244.79593 + 149.73486 -228.51245 + 177.07997 -208.64506 + 201.71942 -185.50708 + 223.26462 -159.46341 + 241.37579 -130.92476 + 255.76731 -100.34123 + 266.21222 -68.19511 + 272.54579 -34.99339 diff --git a/example/cfis_simu/mask_default/ngc_cat.fits b/example/cfis_simu/mask_default/ngc_cat.fits new file mode 100644 index 000000000..f51546da7 Binary files /dev/null and b/example/cfis_simu/mask_default/ngc_cat.fits differ diff --git a/example/cfis_simu/readme.txt b/example/cfis_simu/readme.txt new file mode 100644 index 000000000..719ca3052 --- /dev/null +++ b/example/cfis_simu/readme.txt @@ -0,0 +1,7 @@ +This folder contains the configuration files used to run + shapepipe on simulated images. +The $SP_RUN and $SP_CONFIG can point to this directory. +A link to the exposures and tiles can be called "input_exp" +and "input_tiles" and added to this directory. + + diff --git a/example/cfis_simu/star_selection.setools b/example/cfis_simu/star_selection.setools new file mode 100644 index 000000000..424457853 --- /dev/null +++ b/example/cfis_simu/star_selection.setools @@ -0,0 +1,104 @@ +## SETools configuration file for star/galaxy separation based on size/mag properties + +[MASK:preselect] +MAG_AUTO > 0 +MAG_AUTO < 21 +FWHM_IMAGE > 0.3 / 0.187 +FWHM_IMAGE < 1.5 / 0.187 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +[MASK:flag] +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + + +[MASK:star_selection] +# Star selection using the FWHM mode +MAG_AUTO > 18. +MAG_AUTO < 22. +FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 +FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 +FLAGS == 0 +IMAFLAGS_ISO == 0 +#CLASS_STAR != 0 + +[MASK:fwhm_mag_cut] +FWHM_IMAGE > 0 +FWHM_IMAGE < 40 +MAG_AUTO < 35 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +# Split the 'star_selection' sample into +# two random sub-samples with ratio 80/20 +[RAND_SPLIT:star_split] +RATIO = 20 +MASK = star_selection + +# The following selection is only used for plotting + +[PLOT:size_mag] +TYPE = plot +FORMAT = png +X_1 = FWHM_IMAGE{fwhm_mag_cut} +Y_1 = MAG_AUTO{fwhm_mag_cut} +X_2 = FWHM_IMAGE{star_selection} +Y_2 = MAG_AUTO{star_selection} +MARKER_1 = + +MARKER_2 = . +MARKERSIZE_1 = 3 +MARKERSIZE_2 = 3 +LABEL_1 = All +LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" +TITLE = "Stellar locus" +XLABEL = "FWHM (pix)" +YLABEL = Mag + +[PLOT:hist_mag_stars] +TYPE = hist +FORMAT = png +Y = MAG_AUTO{star_selection} +BIN = 20 +LABEL = "stars" +XLABEL = "Magnitude" +YLABEL = "Number" +TITLE = "Magnitude of stars" + +[PLOT:fwhm_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = FWHM_IMAGE{star_selection}*0.186 +MARKER = . +LABEL = "FWHM (arcsec)" +TITLE = "FWHM of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[PLOT:mag_star_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = MAG_AUTO{star_selection} +MARKER = . +LABEL = "Magnitude" +TITLE = "Magnitude of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[STAT:star_stat] +"Nb objects full cat" = len(FWHM_IMAGE) +"Nb objects not masked" = len(FWHM_IMAGE{flag}) +"Nb stars" = len(FWHM_IMAGE{star_selection}) +"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. +"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 +"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 +"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 +"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 +"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 diff --git a/example/cfis_simu/tile_numbers.txt b/example/cfis_simu/tile_numbers.txt new file mode 100644 index 000000000..3c0ce178a --- /dev/null +++ b/example/cfis_simu/tile_numbers.txt @@ -0,0 +1 @@ +224-295 diff --git a/example/pbs/config_smp.ini b/example/pbs/config_smp.ini index bb61e78f5..d47a8c9d6 100644 --- a/example/pbs/config_smp.ini +++ b/example/pbs/config_smp.ini @@ -2,7 +2,7 @@ ## ShapePipe execution options [EXECUTION] -MODULE = python_example, serial_example, execute_example +MODULE = python_example_runner, serial_example_runner, execute_example_runner MODE = smp ## ShapePipe file handling options @@ -16,8 +16,8 @@ SMP_BATCH_SIZE = 4 TIMEOUT = 00:01:35 ## Module options -[PYTHON_EXAMPLE] +[PYTHON_EXAMPLE_RUNNER] MESSAGE = The obtained value is: -[SERIAL_EXAMPLE] +[SERIAL_EXAMPLE_RUNNER] ADD_INPUT_DIR = $SPDIR/example/data/numbers, $SPDIR/example/data/letters diff --git a/scripts/jupyter/plot_spectro_areas.ipynb b/scripts/jupyter/plot_spectro_areas.ipynb index 7d4f5aed1..b881d208c 100644 --- a/scripts/jupyter/plot_spectro_areas.ipynb +++ b/scripts/jupyter/plot_spectro_areas.ipynb @@ -74,6 +74,10 @@ "# - AGES:\n", "# http://vizier.cfa.harvard.edu/viz-bin/VizieR-3?-source=J/ApJS/200/8/sources\n", "\n", + "# Other data\n", + "# - HSC Hectomap field, star catalogue\n", + "# From Wentao Luo\n", + "\n", "# Download directory\n", "cat_home = '{}/astro/data/CFIS/spectro_surveys'.format(os.environ['HOME'])" ] @@ -189,10 +193,14 @@ "#use = ['Test', '3DHST-AEGIS', '3DHST-GOODS-N', 'DEEP2+3', 'AGES',\n", "# 'HectoMap-groups', 'DECaLS-groups']\n", "\n", - "use = ['DEEP2+3', 'AGES', 'HectoMap-groups',\n", - " 'CMASSLOWZ12-N', 'CMASSLOWZ12-S', 'eBOSSLRG14-N', 'eBOSSLRG14-S', 'NYUVAGC-N', \n", - " 'eBOSSLRGCMASS16-N', 'eBOSSLRGCMASS16-S', 'eBOSSELG16-N',\n", - " 'eBOSSQSO16-N', 'eBOSSQSO16-S']\n", + "#use = ['DEEP2+3', 'AGES', 'HectoMap-groups',\n", + "# 'CMASSLOWZ12-N', 'CMASSLOWZ12-S', 'eBOSSLRG14-N', 'eBOSSLRG14-S', 'NYUVAGC-N', \n", + "# 'eBOSSLRGCMASS16-N', 'eBOSSLRGCMASS16-S', 'eBOSSELG16-N',\n", + "# 'eBOSSQSO16-N', 'eBOSSQSO16-S']\n", + "\n", + "use = ['HectoMap-HSC-stars']\n", + "\n", + "do_SDSS = False\n", "\n", "if 'Test' in use:\n", " surveys.append(Survey.from_array('Test', 'orange',\n", @@ -269,6 +277,11 @@ " '{}/asu.fit'.format(cat_home),\n", " key_ra='RAJ2000', key_dec='DEJ2000'))\n", " \n", + "if 'HectoMap-HSC-stars'in use:\n", + " surveys.append(Survey.from_fits('HectoMap-HSC-stars', 'red',\n", + " '{}/HECTOMAP_stars.fits'.format(cat_home),\n", + " key_ra='ira', key_dec='idec'))\n", + " \n", "if 'DECaLS-groups'in use:\n", " surveys.append(Survey.from_fits('DECaLS-groups', 'cyan',\n", " '{}/DESI_NGC_group_cut.txt.fits'.format(cat_home),\n", @@ -288,61 +301,48 @@ "metadata": {}, "outputs": [], "source": [ - "fraction = {}\n", - "fraction['N'] = {} # np.zeros(shape=(len(surveys), len(surveys)))\n", - "fraction['S'] = {} # np.zeros(shape=(len(surveys), len(surveys)))\n", + "if do_SDSS:\n", + " fraction = {}\n", + " fraction['N'] = {} # np.zeros(shape=(len(surveys), len(surveys)))\n", + " fraction['S'] = {} # np.zeros(shape=(len(surveys), len(surveys)))\n", "\n", - "tolerance = 1.0 * u.arcsec\n", + " tolerance = 1.0 * u.arcsec\n", "\n", - "for s1 in surveys:\n", - " if len(s1._ra) == 0:\n", - " continue\n", + " for s1 in surveys:\n", + " if len(s1._ra) == 0:\n", + " continue\n", " \n", - " c1 = SkyCoord(ra=s1._ra, dec=s1._dec, unit='deg')\n", + " c1 = SkyCoord(ra=s1._ra, dec=s1._dec, unit='deg')\n", "\n", - " for s2 in surveys:\n", - " if len(s2._ra) == 0:\n", - " continue\n", + " for s2 in surveys:\n", + " if len(s2._ra) == 0:\n", + " continue\n", " \n", - " c2 = SkyCoord(ra=s2._ra, dec=s2._dec, unit='deg')\n", + " c2 = SkyCoord(ra=s2._ra, dec=s2._dec, unit='deg')\n", "\n", - " idx, d2d, d3d = match_coordinates_sky(c1, c2, nthneighbor=1)\n", + " idx, d2d, d3d = match_coordinates_sky(c1, c2, nthneighbor=1)\n", "\n", - " idx_close = d2d > tolerance\n", - " id_sub = np.where(idx_close==True)[0]\n", + " idx_close = d2d > tolerance\n", + " id_sub = np.where(idx_close==True)[0]\n", " \n", - " for hemisph in ['N', 'S']:\n", - " if re.search('-{}'.format(hemisph), s1.name) and \\\n", - " re.search('-{}'.format(hemisph), s2.name):\n", + " for hemisph in ['N', 'S']:\n", + " if re.search('-{}'.format(hemisph), s1.name) and \\\n", + " re.search('-{}'.format(hemisph), s2.name):\n", "\n", - " if s1.name not in fraction[hemisph]:\n", - " fraction[hemisph][s1.name] = {}\n", + " if s1.name not in fraction[hemisph]:\n", + " fraction[hemisph][s1.name] = {}\n", " \n", - " fraction[hemisph][s1.name][s2.name] = len(id_sub) / len(idx)\n", + " fraction[hemisph][s1.name][s2.name] = len(id_sub) / len(idx)\n", " \n", - " print(s1.name, s2.name, end='\\t\\t')\n", - " print(' {}/{} = {:.2f}% objects not matched'\n", - " ''.format(len(id_sub), len(idx), 100*fraction[hemisph][s1.name][s2.name]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_N = pd.DataFrame(fraction['N'])\n", - "df_N.style.background_gradient(cmap='coolwarm').set_precision(2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_S = pd.DataFrame(fraction['S'])\n", - "df_S.style.background_gradient(cmap='coolwarm').set_precision(2)" + " print(s1.name, s2.name, end='\\t\\t')\n", + " print(' {}/{} = {:.2f}% objects not matched'\n", + " ''.format(len(id_sub), len(idx), 100*fraction[hemisph][s1.name][s2.name]))\n", + "\n", + " df_N = pd.DataFrame(fraction['N'])\n", + " df_N.style.background_gradient(cmap='coolwarm').set_precision(2)\n", + " \n", + " df_S = pd.DataFrame(fraction['S'])\n", + " df_S.style.background_gradient(cmap='coolwarm').set_precision(2)" ] }, { @@ -376,8 +376,8 @@ "d = 0.5\n", "\n", "# Input tiles ID files\n", - "sp_home = '{}/astro/repositories/github/shapepipe'.format(os.environ['HOME'])\n", - "tiles_dir = '{}/aux/CFIS/tiles_202011'.format(sp_home)\n", + "sp_home = '{}/shapepipe'.format(os.environ['HOME'])\n", + "tiles_dir = '{}/auxdir/CFIS/tiles_202011'.format(sp_home)\n", "tiles_ID = {}\n", "for band in bands:\n", " tiles_ID[band] = '{}/tiles_{}.txt'.format(tiles_dir, band)\n", @@ -474,13 +474,24 @@ " print('{}: {} common tiles in {}'.format(s.name, len(ID[s.name]['all']), bands_str))\n", " \n", "# Write to text files\n", + "\n", "for s in surveys:\n", + " ## All bands are present\n", " if len(ID[s.name]['all']) > 0:\n", " #out_path = '{}_{}.txt'.format(tiles_all_ID_base, s.name)\n", " out_path = '{}{}_{}.txt'.format(tiles_ID_base, bands_str, s.name)\n", " f = open(out_path, 'w')\n", " f.write('\\n'.join(ID[s.name]['all']))\n", " f.write('\\n')\n", + " f.close()\n", + " \n", + " ## Only r-band is present\n", + " this_band = 'r'\n", + " if len(ID[s.name][this_band]) > 0:\n", + " out_path = '{}{}_{}.txt'.format(tiles_ID_base, this_band, s.name)\n", + " f = open(out_path, 'w')\n", + " f.write('\\n'.join(ID[s.name][this_band]))\n", + " f.write('\\n')\n", " f.close()" ] }, @@ -575,9 +586,23 @@ " ra = s._ra\n", " dec = s._dec\n", " map.scatter(ra, dec, marker='o', color=s.color, edgecolors='none', alpha=1)\n", - " \n", - " # Draw grid\n", - " map.grid(sep=sep)\n", + "\n", + " # Get limits\n", + " xlim = map.ax.get_xlim()\n", + " ylim = map.ax.get_ylim()\n", + " dx = xlim[1] - xlim[0]\n", + " dy = ylim[1] - ylim[0]\n", + "\n", + " while True:\n", + " # Draw grid\n", + " map.grid(sep=sep)\n", + " \n", + " # If plot size allows for one or fewer grid lines and ticks:\n", + " # decrease grid separation and redraw\n", + " if min(dx, dy) > sep:\n", + " break\n", + " else:\n", + " sep = round(sep / 2)\n", " \n", " #ra_min_max = ra_all.min(), ra_all.max()\n", " ra_min_max = s._min_ra, s._max_ra\n", @@ -588,7 +613,7 @@ " map.title(s.name)\n", "\n", " map.focus(np.array(ra_min_max), np.array(dec_min_max), pad=0.25)\n", - "\n", + " \n", " map.savefig('{}_spherical.pdf'.format(s.name))\n", " plt.show()" ] @@ -617,7 +642,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.7" } }, "nbformat": 4, diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py index bc062d808..c44c90e31 100755 --- a/scripts/python/canfar_avail_results.py +++ b/scripts/python/canfar_avail_results.py @@ -19,7 +19,7 @@ from optparse import OptionParser -from shapepipe.utilities.canfar import dir_list +from cs_util.canfar import dir_list from shapepipe.utilities import cfis diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py index feecb7209..2c87d3f8d 100755 --- a/scripts/python/create_star_cat.py +++ b/scripts/python/create_star_cat.py @@ -4,31 +4,30 @@ """Script create_star_cat.py -Create reference star catalogue for masking of -bright stars and diffraction spikes +:Description: Create reference star catalogue for masking of +bright star halos and diffraction spikes :Authors: Axel Guinot, Martin Kilbinger -:Date: 2019, 2020 - -:Package: ShapePipe """ -from shapepipe.pipeline.execute import execute +import re +import os +import sys import numpy as np + from astropy.coordinates import SkyCoord from astropy.wcs import WCS from astropy.io import fits from astropy import units as u -import re -import os -import sys + +from shapepipe.pipeline.execute import execute def _get_wcs(header): - """Get WCS + """Get WCS. Compute the astropy WCS from header manually. (The purpose of this is to avoid possible incompatibility on distortion @@ -41,10 +40,10 @@ def _get_wcs(header): Returns ------- - final_wcs : astropy.wcs.WCS + astropy.wcs.WCS WCS object - """ + """ final_wcs = WCS(naxis=2) final_wcs.wcs.ctype = [header['CTYPE1'], header['CTYPE2']] try: @@ -53,14 +52,16 @@ def _get_wcs(header): final_wcs.wcs.cunit = ['deg', 'deg'] final_wcs.wcs.crpix = [header['CRPIX1'], header['CRPIX2']] final_wcs.wcs.crval = [header['CRVAL1'], header['CRVAL2']] - final_wcs.wcs.cd = [[header['CD1_1'], header['CD1_2']], - [header['CD2_1'], header['CD2_2']]] + final_wcs.wcs.cd = [ + [header['CD1_1'], header['CD1_2']], + [header['CD2_1'], header['CD2_2']] + ] return final_wcs def _get_image_radius(center, wcs): - """Get image radius + """Get Image Radius. Compute the diagonal distance of the image in arcmin. @@ -68,59 +69,68 @@ def _get_image_radius(center, wcs): ---------- center : numpy.ndarray Coordinates of the center of the image (in pixel) + wcs : astropy.wcs.WCS + WCS object Returns ------- float The diagonal distance of the image in arcmin. - """ + """ if center is None: - return SphereDist(self._fieldcenter['pix'], np.zeros(2))/60. + raise ValueError('center cannot be None') else: if type(center) is np.ndarray: - return SphereDist(center, np.zeros(2), wcs)/60. + return SphereDist(center, np.zeros(2), wcs) / 60.0 else: raise TypeError('center has to be a numpy.ndarray') def SphereDist(position1, position2, wcs): - """Compute spherical distance - - Compute spherical distance between 2 points. + """Sphere Dist. + + Compute distance between two points on the sphere. Parameters ---------- position1 : numpy.ndarray - [x,y] first point (in pixel) + [x,y], first point (in pixels) position2 : numpy.ndarray - [x,y] second point (in pixel) + [x,y], second point (in pixels) Returns ------- float - The distance in degree. + distance (in degrees) """ - if (type(position1) is not np.ndarray) & (type(position2) is not np.ndarray): - raise ValueError('Positions need to be a numpy.ndarray') + if ( + (type(position1) is not np.ndarray) + & (type(position2) is not np.ndarray) + ): + raise ValueError('Positions need to be of type numpy.ndarray') - p1 = (np.pi/180.)*np.hstack(wcs.all_pix2world(position1[0], position1[1], 1)) - p2 = (np.pi/180.)*np.hstack(wcs.all_pix2world(position2[0], position2[1], 1)) + rad2deg = np.pi / 180.0 + p1 = rad2deg * np.hstack(wcs.all_pix2world(position1[0], position1[1], 1)) + p2 = rad2deg * np.hstack(wcs.all_pix2world(position2[0], position2[1], 1)) dTheta = p1 - p2 dLong = dTheta[0] dLat = dTheta[1] - dist = 2*np.arcsin(np.sqrt(np.sin(dLat/2.)**2. + np.cos(p1[1])*np.cos(p2[1])*np.sin(dLong/2.)**2.)) + dist = 2 * np.arcsin( + np.sqrt(np.sin(dLat/2.0)**2 + np.cos(p1[1])*np.cos(p2[1])*np.sin(dLong/2.0)**2) + ) return dist*(180./np.pi)*3600. def find_stars(position, output_name, radius=None): - """Find stars + """Find Stars. - Return GSC (Guide Star Catalog) objects for a field with center (ra,dec) and radius r. + Return GSC (Guide Star Catalog) objects for a field with center + (ra,dec) and radius r. Parameters ---------- @@ -129,12 +139,7 @@ def find_stars(position, output_name, radius=None): radius : float Radius in which the query is done (in arcmin) - Returns - ------- - dict - Stars dicotionnary for GSC objects in the field. """ - ra = position[0] dec = position[1] @@ -145,22 +150,14 @@ def find_stars(position, output_name, radius=None): else: sign = '' - cmd_line = '{0} {1} {2}{3} -r {4} -n 1000000'.format('findgsc2.2', ra, sign, dec, radius) + cmd_line = f'findgsc2.2 {ra} {sign}{dec} -r {radius} -n 1000000' - # output=subprocess.check_output(cmd_line, shell=True) CDS_stdout, CDS_stderr = execute(cmd_line) output_file = open(output_name, 'w') output_file.write(CDS_stdout) output_file.close() - # if CDS_stderr != '': - # err = True - # return None - # - # # return self._make_star_cat(output.decode("utf-8")) - # return CDS_stdout - def main(input_dir, output_dir, kind): @@ -170,8 +167,6 @@ def main(input_dir, output_dir, kind): if 'image' not in f: continue - # img = fits.open(input_dir + '/' + f) - if kind == 'exp': list_ind = range(1, 41) else: @@ -184,12 +179,12 @@ def main(input_dir, output_dir, kind): else: exp_suff = '' - output_name = output_dir + '/star_cat' + re.split('image', os.path.splitext(f)[0])[1] + exp_suff + '.cat' + img_number = re.split('image', os.path.splitext(f)[0])[1] + output_name = f'{output_dir}/star_cat{img_number}{exp_suff}.cat' if os.path.isfile(output_name): continue - # h = img[ind].header h = fits.getheader(input_dir + '/' + f, ind) w = _get_wcs(h) @@ -197,11 +192,21 @@ def main(input_dir, output_dir, kind): img_shape = (h['NAXIS2'], h['NAXIS1']) img_center = np.array([img_shape[1]/2., img_shape[0]/2.]) wcs_center = w.all_pix2world([img_center], 1)[0] - astropy_center = SkyCoord(ra=wcs_center[0], dec=wcs_center[1], unit='deg') + astropy_center = SkyCoord( + ra=wcs_center[0], + dec=wcs_center[1], + unit='deg' + ) rad = _get_image_radius(img_center, w) - find_stars(np.array([astropy_center.ra.value, astropy_center.dec.value]), output_name, rad) + find_stars( + np.array([astropy_center.ra.value, astropy_center.dec.value]), + output_name, + rad + ) + + return 0 if __name__ == '__main__': @@ -218,11 +223,9 @@ def main(input_dir, output_dir, kind): except: output_path = '.' - # Temporary fix try: kind = argv[3] except: kind = '' main(input_path, output_path, kind) - diff --git a/scripts/python/get_number_objects.py b/scripts/python/get_number_objects.py new file mode 100755 index 000000000..67cdb294d --- /dev/null +++ b/scripts/python/get_number_objects.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python + +# -*- coding: utf-8 -*- + +"""Script get_number_objects.py + +Get number of objects in a (last-run SExtractor) catalogue. + +:Author: Martin Kilbinger + +""" + +import sys +import copy +import glob + +from optparse import OptionParser +from astropy.io import fits + +from shapepipe.pipeline.run_log import get_last_dir +from shapepipe.utilities import cfis + + +class param: + """General class to store (default) variables""" + def __init__(self, **kwds): + self.__dict__.update(kwds) + + def print(self, **kwds): + print(self.__dict__) + + def var_list(self, **kwds): + return vars(self) + + +def params_default(): + """Set default parameter values. + + Returns + ------- + class param + parameter values + + """ + p_def = param( + input_path='.', + input_name_base='final_cat', + hdu_num=1, + ) + + return p_def + + +def parse_options(p_def): + """Parse command line options. + + Parameters + ---------- + p_def: class param + parameter values + + Returns + ------- + list + command line options + command line str + + """ + usage = "%prog [OPTIONS]" + parser = OptionParser(usage=usage) + + # IO + parser.add_option( + '-i', + '--input_path', + dest='input_path', + type='string', + default=p_def.input_path, + help=f'input path, default=\'{p_def.input_path}\'' + ) + parser.add_option( + '-n', + '--input_name_base', + dest='input_name_base', + type='string', + default=p_def.input_name_base, + help=f'input name base, default=\'{p_def.input_name_base}\'' + ) + parser.add_option( + '-l', + '--list_tile_ID_path', + dest='tile_ID_list_path', + type='string', + default=None, + help=f'tile ID list, default: Use all data in input files' + ) + + # Control + parser.add_option( + '-p', + '--param_path', + dest='param_path', + type='string', + default=None, + help='parameter file path, default=None' + ) + + parser.add_option( + '', + '--hdu_num', + dest='hdu_num', + type='int', + default=p_def.hdu_num, + help=f'input HDU number, default=\'{p_def.hdu_num}\'' + ) + + parser.add_option( + '-v', + '--verbose', + dest='verbose', + action='store_true', + help='verbose output' + ) + + options, args = parser.parse_args() + + return options, args + + +def check_options(options): + """Check command line options. + + Parameters + ---------- + options: tuple + Command line options + + Returns + ------- + bool + Result of option check. False if invalid option value. + + """ + return True + + +def update_param(p_def, options): + """Return default parameter, updated and complemented according to options. + + Parameters + ---------- + p_def: class param + parameter values + optiosn: tuple + command line options + + Returns + ------- + class param + updated paramter values + + """ + param = copy.copy(p_def) + + # Update keys in param according to options values + for key in vars(param): + if key in vars(options): + setattr(param, key, getattr(options, key)) + + # Add remaining keys from options to param + for key in vars(options): + if not key in vars(param): + setattr(param, key, getattr(options, key)) + + # Do extra stuff if necessary + + return param + + +def read_param_file(path, verbose=False): + """Return parameter list read from file + + Parameters + ---------- + path: str + input file name + verbose: bool, optional, default=False + verbose output if True + + Returns + ------- + list of str + parameter names + + """ + param_list = [] + + if path: + + with open(path) as f: + for line in f: + if line.startswith('#'): + continue + entry = line.rstrip() + if not entry or entry == '': + continue + param_list.append(entry) + + if verbose: + if len(param_list) > 0: + print(f'Copying {len(param_list)} columns', end='') + else: + print('Copying all columns', end='') + print(' into merged catalogue') + + # Check for multiples + multiples = [] + for param in param_list: + if param_list.count(param) > 1: + multiples.append(param) + + if len(multiples) > 0: + print('The following parameters are more than one times ' + 'in the parameter file: ', end='') + for m in multiples: + print(m, end=' ') + print() + raise ValueError('Multiple identical keys found') + + return param_list + + +def get_data(path, hdu_num, param_list): + """Return data of selected columns from FITS file. + + Parameters + ---------- + path: str + input file name + hdu_num: int + HDU number + param_list: list of str + parameters to be extracted. If none, copy + all columns + + Returns + ------- + numpy array + data columns + + """ + hdu_list = fits.open(path) + hdu = hdu_list[hdu_num] + + if param_list: + cols = [] + for p in param_list: + cols.append(hdu.columns[p]) + coldefs = fits.ColDefs(cols) + hdu_new = fits.BinTableHDU.from_columns(coldefs) + d = hdu_new.data + else: + d = hdu.data + + return d + + +def main(argv=None): + + # Set default parameters + p_def = params_default() + + # Command line options + options, args = parse_options(p_def) + + if check_options(options) is False: + return 1 + + param = update_param(p_def, options) + + # Save command line arguments to log file + f_log = cfis.log_command(argv, close_no_return=False) + + module = 'sextractor_runner_run_1' + pattern = 'sexcat' + run_log_file = 'output/log_run_sp.txt' + last_dir = get_last_dir(run_log_file, module) + + file_list = glob.glob(f'{last_dir}/{pattern}*.fits') + if len(file_list) == 0: + raise ValueError(f'No files {last_dir}/{pattern}*.fits found') + + n_obj = 0 + hdu_no = -1 + for fpath in file_list: + hdu_list = fits.open(fpath) + header = hdu_list[-1].header + n_obj += int(header['NAXIS2']) + + n_obj = int(n_obj / len(file_list)) + + print(n_obj) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py index ed79b27ee..1a160c449 100755 --- a/scripts/python/merge_final_cat.py +++ b/scripts/python/merge_final_cat.py @@ -316,7 +316,6 @@ def main(argv=None): add_this_l = False # mark to add if correct extension, matches input pattern, - not `.npy` file if ( this_l.endswith(ext) and (f'{param.input_name_base}' in this_l) diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash index 6cbb687e7..15cb1428a 100755 --- a/scripts/sh/job_sp.bash +++ b/scripts/sh/job_sp.bash @@ -11,6 +11,9 @@ # Date: v1.0 11/2020 # v1.1 01/2021 +# MKDEBUG TODO: +# Option to change SMP_BATCH_SIZE, not for MPI + # VM home, required for canfar run. ## On other machines set to $HOME @@ -26,8 +29,10 @@ job=255 config_dir=$VM_HOME/shapepipe/example/cfis psf='mccd' retrieve='vos' +star_cat_for_mask='onthefly' results='cosmostat/kilbinger/results_v1' -nsh_step=3200 +n_smp=-1 +nsh_step=-1 nsh_max=-1 nsh_jobs=8 @@ -49,17 +54,22 @@ usage="Usage: $(basename "$0") [OPTIONS] TILE_ID_1 [TILE_ID_2 [...]] -p, --psf MODEL\n \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n -r, --retrieve METHOD\n - \tmethod to retrieve images, one in ['vos'|'symlink]', default='$retrieve'\n + \tmethod to retrieve images, allowed are 'vos', 'symlink', default='$retrieve'\n + -s, --star_cat_for_mask\n + \tcatalogue for masking bright stars, allowed are 'onthefly', 'save',\n + \tdefault is '${star_cat_for_mask}'\n -o, --output_dir\n \toutput (upload) directory on vos:cfis, default='$results'\n + -n, --n_smp\n + \tnumber of jobs (SMP mode only), default from original config files\n + --nsh_step NSTEP\n --nsh_jobs NJOB\n \tnumber of shape measurement parallel jobs, default=$nsh_jobs\n - --nsh_step NSTEP\n \tnumber of objects per parallel shape module call, \n - \t default: $nsh_step\n + \tdefault: optimal number is computed\n --nsh_max NMAX\n \tmax number of objects per parallel shape module call, \n - \t default: unlimited; has precedent over --nsh_step\n + \tdefault: unlimited; has precedent over --nsh_step\n TILE_ID_i\n \ttile ID(s), e.g. 283.247 214.242\n " @@ -94,10 +104,18 @@ while [ $# -gt 0 ]; do retrieve="$2" shift ;; + -s|--star_cat_for_mask) + star_cat_for_mask="$2" + shift + ;; -o|--output_dir) results="$2" shift ;; + -n|--n_smp) + n_smp="$2" + shift + ;; --nsh_max) nsh_max="$2" shift @@ -122,11 +140,23 @@ if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then echo "PSF (option -p) needs to be 'psfex' or 'mccd'" exit 2 fi + +if [ "$star_cat_for_mask" != "onthefly" ] && [ "$star_cat_for_mask" != "save" ]; then + echo "Star cat for mask (option -s) needs to be 'onthefly' or 'save'" + exit 4 +fi + +if [ "$retrieve" != "vos" ] && [ "$retrieve" != "symlink" ]; then + echo "method to retrieve images (option -r) needs to be 'vos' or 'symlink'" + exit 5 +fi + n_tile=${#TILE_ARR[@]} if [ "$n_tile" == "0" ]; then echo "No tile ID given" exit 3 fi + if [ $nsh_max != -1 ]; then nsh_step=$nsh_max fi @@ -177,13 +207,13 @@ function command () { cmd=$1 str=$2 - #RED='\033[0;31m' - #GREEN='\033[0;32m' - #NC='\033[0m' # No Color + RED='\033[0;31m' + GREEN='\033[0;32m' + NC='\033[0m' # No Color # Color escape characters show up in log files - RED='' - GREEN='' - NC='' + #RED='' + #GREEN='' + #NC='' if [ $# == 2 ]; then @@ -212,18 +242,27 @@ function command () { fi fi fi - - #return $res } # Run shapepipe command. If error occurs, upload sp log files before stopping script. -command_sp() { - cmd=$1 - str=$2 +function command_sp() { + local cmd=$1 + local str=$2 command "$1" "$2" } +# Set up config file and call shapepipe_run +function command_cfg_shapepipe() { + local config_name=$1 + local str=$2 + local _n_smp=$3 + + config_upd=$(set_config_n_smp $config_name $_n_smp) + local cmd="shapepipe_run -c $config_upd" + command_sp "$cmd" "$str" +} + # Tar and upload files to vos function upload() { base=$1 @@ -272,6 +311,35 @@ function print_env() { echo "***" } +function set_config_n_smp() { + local config_name=$1 + local _n_smp=$2 + + local config_orig="$SP_CONFIG/$config_name" + + if [[ $_n_smp != -1 ]]; then + # Update SMP batch size + local config_upd="$SP_CONFIG_MOD/$config_name" + update_config $config_orig $config_upd "SMP_BATCH_SIZE" $_n_smp + else + # Keep original config file + local config_upd=$config_orig + fi + + # Set "return" value (stdout) + echo "$config_upd" +} + +# Update config file +function update_config() { + local config_orig=$1 + local config_upd=$2 + local key=$3 + local val_upd=$4 + + cat $config_orig \ + | perl -ane 's/'$key'\s+=.+/'$key' = '$val_upd'/; print' > $config_upd +} ### Start ### @@ -283,10 +351,12 @@ echo "Processing $n_tile tile(s)" mkdir -p $SP_RUN cd $SP_RUN mkdir -p $OUTPUT +mkdir -p $SP_CONFIG_MOD # Processing ## Retrieve config files and images (online if retrieve=vos) +## Retrieve and save star catalogues for masking (if star_cat_for_mask=save) (( do_job= $job & 1 )) if [[ $do_job != 0 ]]; then @@ -308,6 +378,17 @@ if [[ $do_job != 0 ]]; then ### Retrieve files command_sp "shapepipe_run -c $SP_CONFIG/config_GitFeGie_$retrieve.ini" "Retrieve images" + ### Retrieve and save star catalogues for masking + if [ "$star_cat_for_mask" == "save" ]; then + #### For tiles + mkdir $SP_RUN/star_cat_tiles + command_sp "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_1/output $SP_RUN/star_cat_tiles" "Save star cats for masking (tile)" + + #### For single-exposures + mkdir $SP_RUN/star_cat_exp + command_sp "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_2/output $SP_RUN/star_cat_exp exp" "Save star cats for masking (exp)" + fi + fi ## Prepare images (offline) @@ -315,19 +396,19 @@ fi if [[ $do_job != 0 ]]; then ### Uncompress tile weights - command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Uz.ini" "Run shapepipe (uncompress tile weights)" + command_cfg_shapepipe "config_tile_Uz.ini" "Run shapepipe (uncompress tile weights)" $n_smp ### Split images into single-HDU files, merge headers for WCS info - command_sp "shapepipe_run -c $SP_CONFIG/config_exp_SpMh.ini" "Run shapepipe (split images, merge headers)" + command_cfg_shapepipe "config_exp_SpMh.ini" "Run shapepipe (split images, merge headers)" $n_smp fi -## Mask tiles and exposures: add star, halo, and Messier object masks (online) +## Mask tiles and exposures: add star, halo, and Messier object masks (online if "star_cat_for_mask" is "onthefly") (( do_job= $job & 4 )) if [[ $do_job != 0 ]]; then ### Mask tiles and exposures - command_sp "shapepipe_run -c $SP_CONFIG/config_MaMa.ini" "Run shapepipe (mask)" + command_cfg_shapepipe "config_MaMa_$star_cat_for_mask.ini" "Run shapepipe (mask)" $n_smp fi @@ -360,8 +441,12 @@ fi if [[ $do_job != 0 ]]; then ### Prepare config files - mkdir -p $SP_CONFIG_MOD n_min=0 + if [[ $nsh_step == -1 ]]; then + n_obj=`get_number_objects.py` + nsh_step=`echo "$(($n_obj/$nsh_jobs))"` + fi + n_max=$((nsh_step - 1)) for k in $(seq 1 $nsh_jobs); do cat $SP_CONFIG/config_tile_Ng_template.ini | \ diff --git a/shapepipe/modules/find_exposures_package/__init__.py b/shapepipe/modules/find_exposures_package/__init__.py index bc5bbc591..00653bf0b 100644 --- a/shapepipe/modules/find_exposures_package/__init__.py +++ b/shapepipe/modules/find_exposures_package/__init__.py @@ -25,7 +25,11 @@ Module-specific config file entries =================================== -None +COLNUM : int + Column number to find exposure in fits header of tile image for the HISTORY + string +EXP_PREFIX: str + Prefix of exposures """ __all__ = ['find_exposures.py'] diff --git a/shapepipe/modules/find_exposures_package/find_exposures.py b/shapepipe/modules/find_exposures_package/find_exposures.py index ef0175c5d..a3c8f7928 100644 --- a/shapepipe/modules/find_exposures_package/find_exposures.py +++ b/shapepipe/modules/find_exposures_package/find_exposures.py @@ -25,14 +25,19 @@ class FindExposures(): Output file path w_log : logging.Logger Log file - + colnum: int + column number for exposure name in fits header + prefix: str + prefix for exposures """ - def __init__(self, img_tile_path, output_path, w_log): + def __init__(self, img_tile_path, output_path, w_log, colnum, prefix): self._img_tile_path = img_tile_path self._output_path = output_path self._w_log = w_log + self._colnum = colnum + self.prefix = prefix def process(self): """Process. @@ -83,14 +88,15 @@ def get_exposure_list(self): temp = _hist.split(' ') pattern = r'(.*)\.{1}.*' - pattern_match = re.search(pattern, temp[3]) + pattern_match = re.search(pattern, temp[self._colnum]) if not pattern_match: raise IndexError( - f're match \'{pattern}\' failed for filename \'{temp[3]}\'' + f're match \'{pattern}\' failed for filename' + + f' \'{temp[self._colnum]}\'' ) exp_name = pattern_match.group(1) - + exp_name = exp_name.removeprefix(self.prefix) # LSB exposure names have 's', header still says 'p' # exp_name = re.sub(r'p', 's', exp_name) diff --git a/shapepipe/modules/find_exposures_runner.py b/shapepipe/modules/find_exposures_runner.py index 59531c43b..a7ea6b0be 100644 --- a/shapepipe/modules/find_exposures_runner.py +++ b/shapepipe/modules/find_exposures_runner.py @@ -32,11 +32,18 @@ def find_exposures_runner( # Create output ascii file name output_path = f'{run_dirs["output"]}/exp_numbers{file_number_string}.txt' + # Give column number for exposure name in fits header + colnum = config.getint(module_config_sec, 'COLNUM') + + # Give the prefix of exposures + exp_prefix = config.get(module_config_sec, 'EXP_PREFIX') # Create find exposures class instance find_exp_inst = find_exposures.FindExposures( input_file_name, output_path, - w_log + w_log, + colnum, + exp_prefix, ) # Run processing diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index f0a4220c6..d465d0719 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -12,7 +12,7 @@ import sys from shapepipe.modules.module_decorator import module_runner -from shapepipe.utilities.canfar import vosHandler +from cs_util.canfar import vosHandler # pragma: no cover @@ -62,7 +62,8 @@ def in2out_pattern(number): # remove letters in number number_final = re.sub('[a-zA-Z]', '', number_final) - + # make robust for more generalized file names + number_final = re.sub(r'_', '', number_final) return number_final @@ -185,14 +186,19 @@ def process(self, input_dir, output_dir): input_dir, use_output_file_pattern=False ) - all_outputs = self.get_file_list( + all_outputs_orig = self.get_file_list( + image_number_list, + output_dir, + use_output_file_pattern=False + ) + all_outputs_renamed = self.get_file_list( image_number_list, output_dir, use_output_file_pattern=True ) # Retrieve files - self.retrieve(all_inputs, all_outputs) + self.retrieve(all_inputs, all_outputs_orig, all_outputs_renamed) def get_file_list( self, @@ -260,7 +266,7 @@ def get_file_list( return list_all_files - def retrieve(self, all_inputs, all_outputs): + def retrieve(self, all_inputs, all_outputs_orig, all_outputs_renamed): """Retrieve. Retrieve all files. @@ -273,10 +279,14 @@ def retrieve(self, all_inputs, all_outputs): Output file paths, one list for each input file type """ - for in_per_type, out_per_type in zip(all_inputs, all_outputs): + for in_per_type, out_per_type_orig, out_per_type_renamed in zip( + all_inputs, + all_outputs_orig, + all_outputs_renamed + ): for idx in range(len(in_per_type)): if self._check_existing_dir: - out_base = os.path.basename(out_per_type[idx]) + out_base = os.path.basename(out_per_type_orig[idx]) path = glob.glob( f'{self._check_existing_dir}/**/{out_base}', recursive=True, @@ -300,9 +310,13 @@ def retrieve(self, all_inputs, all_outputs): + f' {self._check_existing_dir},' + ' downloading images' ) - self.retrieve_one(in_per_type[idx], out_per_type[idx]) + self.retrieve_one( + in_per_type[idx], + out_per_type_orig[idx], + out_per_type_renamed[idx] + ) - def retrieve_one(self, in_path, out_path): + def retrieve_one(self, in_path, out_path_orig, out_path_renamed): """Retrieve One. Retrieve one file. @@ -322,12 +336,13 @@ def retrieve_one(self, in_path, out_path): for opt in self._retrieve_options.split(' '): sys.argv.append(opt) sys.argv.append(in_path) - sys.argv.append(out_path) + sys.argv.append(out_path_orig) log_cmd = ' '.join(sys.argv) vcp = vosHandler('vcp') self._w_log.info(log_cmd) + # Download file from VOSpace attempt = 0 while attempt < self._n_try: try: @@ -346,6 +361,10 @@ def retrieve_one(self, in_path, out_path): sys.argv = None + # Create symbolic link to downloaded file with + # link name in ShapePipe numbering format + os.symlink(out_path_orig, out_path_renamed) + elif self._retrieve_method == 'symlink': src = in_path @@ -356,7 +375,7 @@ def retrieve_one(self, in_path, out_path): f'No input file found corresponding to \'{src}\'' ) - dst = out_path + dst = out_path_renamed for src in all_src: if os.path.isdir(dst): # OUTPUT_FILE_PATTERN is '*', so dst is not regular file diff --git a/shapepipe/modules/make_cat_package/make_cat.py b/shapepipe/modules/make_cat_package/make_cat.py index 46203dac2..4e30e3552 100644 --- a/shapepipe/modules/make_cat_package/make_cat.py +++ b/shapepipe/modules/make_cat_package/make_cat.py @@ -16,6 +16,7 @@ from sqlitedict import SqliteDict from shapepipe.pipeline import file_io +from shapepipe.utilities import galaxy def prepare_final_cat_file(output_path, file_number_string): @@ -556,7 +557,9 @@ def _save_psf_data(self, galaxy_psf_path): ) self._add2dict(f'PSF_ELL_{epoch + 1}', e_psf, idx) - psf_fwhm = gpc_data['SHAPES']['SIGMA_PSF_HSM'] * 2.355 + psf_fwhm = galaxy.sigma_to_fwhm( + gpc_data['SHAPES']['SIGMA_PSF_HSM'] + ) self._add2dict(f'PSF_FWHM_{epoch + 1}', psf_fwhm, idx) flag_psf = gpc_data['SHAPES']['FLAG_PSF_HSM'] diff --git a/shapepipe/modules/mask_package/mask.py b/shapepipe/modules/mask_package/mask.py index 17e61b847..69a1c3c97 100644 --- a/shapepipe/modules/mask_package/mask.py +++ b/shapepipe/modules/mask_package/mask.py @@ -1179,6 +1179,7 @@ def _build_final_mask( ) external_flag.open() if final_mask is not None: + final_mask = final_mask.astype(np.int16, copy=False) final_mask += external_flag.get_data()[:, :] else: final_mask = external_flag.get_data()[:, :] diff --git a/shapepipe/modules/mccd_interp_runner.py b/shapepipe/modules/mccd_interp_runner.py index 49780bec6..4856e562f 100644 --- a/shapepipe/modules/mccd_interp_runner.py +++ b/shapepipe/modules/mccd_interp_runner.py @@ -53,7 +53,6 @@ def mccd_interp_runner( module_config_sec, 'PSF_MODEL_SEPARATOR' ) - # psfcat_path, galcat_path = input_file_list galcat_path = input_file_list[0] # verify that the MCCD model exists diff --git a/shapepipe/modules/mccd_package/__init__.py b/shapepipe/modules/mccd_package/__init__.py index 2d7364d7c..dc6416c43 100644 --- a/shapepipe/modules/mccd_package/__init__.py +++ b/shapepipe/modules/mccd_package/__init__.py @@ -123,7 +123,8 @@ GET_SHAPES: bool Calculate PSF model shapes and save to output if ``True`` PSF_MODEL_DIR: str - Input directories for the fitted MCCD PSF model files + Module name of last run producing the fitted MCCD PSF model files. + The specifier "last:" is not required PSF_MODEL_PATTERN: str Pattern of the fitted PSF models PSF_MODEL_SEPARATOR: str diff --git a/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/shapepipe/modules/mccd_package/mccd_interpolation_script.py index 342e20a35..40e644e8b 100644 --- a/shapepipe/modules/mccd_package/mccd_interpolation_script.py +++ b/shapepipe/modules/mccd_package/mccd_interpolation_script.py @@ -378,6 +378,11 @@ def _interpolate_me(self): Interpolate PSFs for multi-epoch run. + Raises + ------ + KeyError + If 'N_EPOCH' key not in input catalogue + Returns ------- output_dict: dict @@ -389,7 +394,14 @@ def _interpolate_me(self): cat.open() all_id = np.copy(cat.get_data()['NUMBER']) - n_epoch = np.copy(cat.get_data()['N_EPOCH']) + key_ne = 'N_EPOCH' + if key_ne not in cat.get_data(): + raise KeyError( + f'Key {key_ne} not found in input galaxy catalogue, needed for' + + ' PSF interpolation to multi-epoch data; run previous module' + + ' (SExtractor) in multi-epoch mode' + ) + n_epoch = np.copy(cat.get_data()[key_ne]) list_ext_name = cat.get_ext_name() hdu_ind = [i for i in range(len(list_ext_name)) if diff --git a/shapepipe/modules/mccd_package/mccd_plot_utilities.py b/shapepipe/modules/mccd_package/mccd_plot_utilities.py index 972f73714..1d3eb9f14 100644 --- a/shapepipe/modules/mccd_package/mccd_plot_utilities.py +++ b/shapepipe/modules/mccd_package/mccd_plot_utilities.py @@ -206,7 +206,12 @@ def plot_meanshapes( remove_outliers=False, plot_meanshapes=True, plot_histograms=True, - psf_model_type='mccd' + psf_model_type='mccd', + max_e=None, + max_de=None, + min_r2=None, + max_r2=None, + max_dr2=None, ): """Plot Mean Shapes. @@ -234,6 +239,21 @@ def plot_meanshapes( default is ``True`` psf_model_type : str, optional PSF model type, options are ``mccd`` or ``psfex``; defualt is ``mccd`` + max_e : float, optional + maximum value for focal plane ellipticity plots; default is ``None``, + set according to from data + max_de : float, optional + maximum value for focal plane residual ellipticity plots; default is + ``None``, set according to from data + min_r2 : float, optional + minimum value for focal plane size plots, default is ``None``; set + according to data + max_r2 : float, optional + maximum value for focal plane size plots, default is ``None``; set + according to data + max_dr2 : float, optional + maximum value for focal plane residual size plots, default is ``None``; + set according to data """ # READ FULL STARCAT @@ -357,13 +377,17 @@ def plot_meanshapes( xs_loc, ys_loc = all_X[ccd_mask] - x_shift, all_Y[ccd_mask] - y_shift - # swap axes to match CCD orientation and origin convention - ys_loc = loc2glob.y_npix - ys_loc + 1 + if psf_model_type == 'mccd': + # swap axes to match CCD orientation and origin convention + ys_loc = loc2glob.y_npix - ys_loc + 1 # digitalize into bins xbins = np.digitize(xs_loc, grid[0]) ybins = np.digitize(ys_loc, grid[1]) + if psf_model_type == 'psfex': + xbins, ybins = megacam_flip(xbins, ybins, ccd_nb, nb_pixel) + for xb in range(nb_pixel[0]): for yb in range(nb_pixel[1]): bin_star_shapes = star_shapes[ @@ -378,8 +402,13 @@ def plot_meanshapes( if plot_meanshapes: # e_1 - vmax = max(np.nanmax(ccd_maps[:, :, 0]), - np.abs(np.nanmin(ccd_maps[:, :, 0]))) + if max_e: + vmax = max_e + else: + vmax = max( + np.nanmax(ccd_maps[:, :, 0]), + np.abs(np.nanmin(ccd_maps[:, :, 0])) + ) vmin = -vmax wind = [vmin, vmax] title = ( @@ -410,7 +439,10 @@ def plot_meanshapes( e1_res = e1_res[~np.isnan(e1_res)] rmse_e1 = np.sqrt(np.mean(e1_res ** 2)) w_log.info(f'Bins: e1 residual RMSE: {rmse_e1:.6f}\n') - vmax = np.nanmax(abs(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0])) + if max_de: + vmax = max_de + else: + vmax = np.nanmax(abs(ccd_maps[:, 0, 0] - ccd_maps[:, 1, 0])) vmin = -vmax wind = [vmin, vmax] title = ( @@ -426,10 +458,13 @@ def plot_meanshapes( ) # e_2 - vmax = max( - np.nanmax(ccd_maps[:, :, 1]), - np.abs(np.nanmin(ccd_maps[:, :, 1])) - ) + if max_e: + vmax = max_e + else: + vmax = max( + np.nanmax(ccd_maps[:, :, 1]), + np.abs(np.nanmin(ccd_maps[:, :, 1])) + ) vmin = -vmax wind = [vmin, vmax] title = ( @@ -461,7 +496,10 @@ def plot_meanshapes( e2_res = e2_res[~np.isnan(e2_res)] rmse_e2 = np.sqrt(np.mean(e2_res ** 2)) w_log.info(f'Bins: e2 residual RMSE: {rmse_e2:.6f}\n') - vmax = np.nanmax(abs(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1])) + if max_de: + vmax = max_de + else: + vmax = np.nanmax(abs(ccd_maps[:, 0, 1] - ccd_maps[:, 1, 1])) vmin = -vmax wind = [vmin, vmax] title = ( @@ -477,7 +515,15 @@ def plot_meanshapes( ) # R^2 - wind = [0, np.nanmax(ccd_maps[:, :, 2])] + if min_r2: + vmin = min_r2 + else: + vmin = 0 + if max_r2: + vmax = max_r2 + else: + vmax = np.nanmax(ccd_maps[:, :, 2]) + wind = [vmin, vmax] colorbar_ampl = 1 title = ( f'R_2 (stars), std={np.nanstd(ccd_maps[:, 0, 2]):.5e}\n' @@ -514,8 +560,14 @@ def plot_meanshapes( R2_res = R2_res[~np.isnan(R2_res)] rmse_r2 = np.sqrt(np.mean(R2_res ** 2)) w_log.info(f"Bins: R2 residual RMSE: {rmse_r2:.6f}\n") - vmax = np.nanmax( - abs((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2])) + if max_dr2: + vmax = max_dr2 + else: + vmax = np.nanmax( + abs( + (ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2] + ) + ) wind = [0, vmax] std_title = np.nanstd( (ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2] diff --git a/shapepipe/modules/mccd_plots_runner.py b/shapepipe/modules/mccd_plots_runner.py index 7fd95b691..1c80505b7 100644 --- a/shapepipe/modules/mccd_plots_runner.py +++ b/shapepipe/modules/mccd_plots_runner.py @@ -70,6 +70,32 @@ def mccd_plots_runner( # Get parameters for meanshapes plots psf_model_type = config.get(module_config_sec, 'PSF') + + if config.has_option(module_config_sec, 'MAX_E'): + max_e = config.getfloat(module_config_sec, "MAX_E") + else: + max_e = None + + if config.has_option(module_config_sec, 'MAX_DE'): + max_de = config.getfloat(module_config_sec, "MAX_DE") + else: + max_de = None + + if config.has_option(module_config_sec, 'MIN_R2'): + min_r2 = config.getfloat(module_config_sec, "MIN_R2") + else: + min_r2 = None + + if config.has_option(module_config_sec, 'MAX_R2'): + max_r2 = config.getfloat(module_config_sec, "MAX_R2") + else: + max_r2 = None + + if config.has_option(module_config_sec, 'MAX_DR2'): + max_dr2 = config.getfloat(module_config_sec, "MAX_DR2") + else: + max_dr2 = None + x_nb_bins = config.getint(module_config_sec, 'X_GRID') y_nb_bins = config.getint(module_config_sec, 'Y_GRID') remove_outliers = config.getboolean(module_config_sec, 'REMOVE_OUTLIERS') @@ -107,6 +133,11 @@ def mccd_plots_runner( plot_meanshapes=plot_meanshapes, plot_histograms=plot_histograms, psf_model_type=psf_model_type, + max_e=max_e, + max_de=max_de, + min_r2=min_r2, + max_r2=max_r2, + max_dr2=max_dr2, ) else: msg = ( @@ -129,9 +160,9 @@ def mccd_plots_runner( ) warnings.warn(msg) w_log.info(msg) - elif rho_stat_plot_style != 'HSC' and rho_stat_plot_style != 'DEC': + elif rho_stat_plot_style != 'HSC' and rho_stat_plot_style != 'DES': msg = ( - 'The rho stat definition should be HSC or DEC. An unknown' + 'The rho stat definition should be HSC or DES. An unknown' + ' definition was used. Rho stat calculation aborted.' ) warnings.warn(msg) diff --git a/shapepipe/modules/psfex_interp_package/__init__.py b/shapepipe/modules/psfex_interp_package/__init__.py index 5569be894..afda4d2e2 100644 --- a/shapepipe/modules/psfex_interp_package/__init__.py +++ b/shapepipe/modules/psfex_interp_package/__init__.py @@ -32,10 +32,11 @@ Threshold of stars under which the PSF is not interpolated CHI2_THRESH : int Threshold for chi squared (:math:`\chi^2`) -ME_DOT_PSF_DIR : list - Input directories for PSFEx PSF model files, for multi-epoch processing -ME_DOT_PSF_PATTERN : list - Input file name patterns for PSFEx PSF model files, for multi-epoch +ME_DOT_PSF_DIR : str + Module name of last run producing PSFEx PSF model files, for multi-epoch + processing. The specifier "last:" is not required +ME_DOT_PSF_PATTERN : str + Input file name pattern for PSFEx PSF model files, for multi-epoch processing ME_LOG_WCS : str Path to world coordinate system log file (``*sqlite``) diff --git a/shapepipe/modules/psfex_interp_package/psfex_interp.py b/shapepipe/modules/psfex_interp_package/psfex_interp.py index 767c43b1b..3ac1b25ea 100644 --- a/shapepipe/modules/psfex_interp_package/psfex_interp.py +++ b/shapepipe/modules/psfex_interp_package/psfex_interp.py @@ -551,6 +551,11 @@ def _interpolate_me(self): Interpolate PSFs for multi-epoch run. + Raises + ------ + KeyError + If 'N_EPOCH' key not in input catalogue + Returns ------- dict @@ -562,7 +567,14 @@ def _interpolate_me(self): cat.open() all_id = np.copy(cat.get_data()['NUMBER']) - n_epoch = np.copy(cat.get_data()['N_EPOCH']) + key_ne = 'N_EPOCH' + if key_ne not in cat.get_data(): + raise KeyError( + f'Key {key_ne} not found in input galaxy catalogue, needed for' + + ' PSF interpolation to multi-epoch data; run previous module' + + ' (SExtractor) in multi-epoch mode' + ) + n_epoch = np.copy(cat.get_data()[key_me]) list_ext_name = cat.get_ext_name() hdu_ind = [ diff --git a/shapepipe/modules/vignetmaker_package/__init__.py b/shapepipe/modules/vignetmaker_package/__init__.py index 56a09b226..a61e83c27 100644 --- a/shapepipe/modules/vignetmaker_package/__init__.py +++ b/shapepipe/modules/vignetmaker_package/__init__.py @@ -36,8 +36,9 @@ PREFIX : str or list Output file name prefix(es) ME_IMAGE_DIR : list - Input directories for single-exposure flags, images, weights, and - SExtractor background images, for multi-epoch processing + Module names of last run producing single-exposure flags, images, weights, + and SExtractor background images, for multi-epoch processing. The specifier + "last:" is not required ME_IMAGE_PATTERN : list Input file name patterns for flag, image, weight, and SExtractor background files, for multi-epoch processing diff --git a/shapepipe/pipeline/file_io.py b/shapepipe/pipeline/file_io.py index a768f0c06..52e847147 100644 --- a/shapepipe/pipeline/file_io.py +++ b/shapepipe/pipeline/file_io.py @@ -1549,7 +1549,7 @@ def _get_fits_col_type(self, col_data): col_type = 'D' elif type(col_data[0]) is bool: col_type = 'L' - elif type(col_data[0]) in [str, np.str, np.str_, np.str0]: + elif type(col_data[0]) in [str, np.str_, np.str0]: col_type = 'A' else: col_type = 'D' diff --git a/shapepipe/utilities/__init__.py b/shapepipe/utilities/__init__.py index 2c27cd7df..63b425136 100644 --- a/shapepipe/utilities/__init__.py +++ b/shapepipe/utilities/__init__.py @@ -7,4 +7,4 @@ """ -__all__ = ['canfar', 'file_system', 'cfis', 'galaxy'] +__all__ = ['file_system', 'cfis', 'galaxy'] diff --git a/shapepipe/utilities/canfar.py b/shapepipe/utilities/canfar.py deleted file mode 100644 index bf67c8574..000000000 --- a/shapepipe/utilities/canfar.py +++ /dev/null @@ -1,190 +0,0 @@ -"""CANFAR TOOLS. - -This module defines methods for managing CANFAR specific actions. - -:Author: Samuel Farrens - Martin Kilbinger - -""" - -import os -import sys -from contextlib import redirect_stdout -from io import StringIO - -try: - import vos.commands as vosc -except ImportError: # pragma: no cover - import_fail = True -else: - import_fail = False - - -class vosError(Exception): - """VOS Error. - - Generic error that is raised by the vosHandler. - - """ - - pass - - -class vosHandler: - """VOS Handler. - - This class manages the use of VOS commands. - - Parameters - ---------- - command : str - VOS command name - - """ - - def __init__(self, command): - - self._check_vos_install() - self._avail_commands = tuple(vosc.__all__) - self.command = command - - @staticmethod - def _check_vos_install(): - """Check VOS Install. - - Check if VOS is correctly installed. - - Raises - ------ - ImportError - if vos package cannot be imported - - """ - if import_fail: - raise ImportError( - 'vos package not found, re-install ShapePipe ' - + 'with \'./install_shapepipe --vos\'' - ) - - @property - def command(self): - """Set Command. - - This method sets the VOS command property. - - Raises - ------ - ValueError - if value is not valid vos command - - """ - return self._command - - @command.setter - def command(self, value): - - if value not in self._avail_commands: - raise ValueError( - f'vos command must be one of {self._avail_commands}' - ) - - self._command = getattr(vosc, value) - - def __call__(self, *args, **kwargs): - """Call Method. - - This method allows class instances to be called as functions. - - Raises - ------ - vosError - if error in vos command occurs - - """ - try: - self._command() - - except Exception: - raise vosError( - f'Error in VOs command: {self._command.__name__}' - ) - - -def download(source, target, verbose=False): - """Download. - - Download file from vos. - - Parameters - ---------- - source : str - source path on vos - target : str - target path - verbose : bool, optional, default=False - verbose output if True - - Returns - ------- - status : bool - status, True/False or success/failure - - """ - cmd = 'vcp' - - if not os.path.exists(target): - sys.argv = [cmd, source, target] - if verbose: - print(f'Downloading file {source} to {target}...') - vcp = vosHandler(cmd) - - vcp() - if verbose: - print('Download finished.') - else: - if verbose: - print(f'Target file {target} exists, skipping download.') - - -def dir_list(path, verbose=False): - """Set Directory List. - - List content of path on vos. - - Parameters - ---------- - path : str - path on vos, starts with 'vos:cfis/...' - verbose : bool, optional, default=False - verbose output if True - - Raises - ------ - HTTPError, KeyError - if error occurs during vos command - - Returns - ------- - vls_out : array of str - file or directory at path - - """ - cmd = 'vls' - sys.argv = [cmd, path] - vls = vosHandler(cmd) - - if verbose: - print('Getting vos directory content from vls...') - - f = StringIO() - - try: - with redirect_stdout(f): - vls() - except Exception: - print('Error during vls command') - raise - - vls_out = f.getvalue() - - return vls_out.split('\n') diff --git a/shapepipe/utilities/cfis.py b/shapepipe/utilities/cfis.py index 9c9da9bcf..c8cdcb7a5 100644 --- a/shapepipe/utilities/cfis.py +++ b/shapepipe/utilities/cfis.py @@ -14,9 +14,7 @@ import astropy.coordinates as coords import numpy as np import pylab as plt -from astropy import units from astropy.io import ascii -from astropy.wcs import WCS from shapepipe.utilities.file_system import mkdir