Merge branch 'CosmoStat:develop' into hp_test

.github/workflows/ci-tests.yml

@@ -13,24 +13,27 @@ jobs:
     name: Full Test Suite
     runs-on: ${{ matrix.os }}
 
+    defaults:
+      run:
+        shell: bash -l {0}
+
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.8]
+        python-version: [3.9]
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v3
 
       - name: Install Linux dependencies
         if: runner.os == 'Linux'
         run: sudo apt-get install libopenblas-dev
 
       - name: Install macOS Dependencies
-        shell: bash -l {0}
         if: runner.os == 'macOS'
-        run: |
-          brew install libomp
+        run: brew install libomp
 
       - name: Set up conda
         uses: conda-incubator/setup-miniconda@v2
@@ -40,12 +43,9 @@ jobs:
           auto-activate-base: true
 
       - name: Install package
-        shell: bash -l {0}
-        run: |
-          ./install_shapepipe --develop
+        run: ./install_shapepipe --develop
 
       - name: Run tests
-        shell: bash -l {0}
         run: |
           conda activate shapepipe
           python setup.py test

docs/source/configuration.md

@@ -200,6 +200,9 @@ where ``X`` is an integer greater than or equal to ``1``. This feature can be co
 
 ```ini
 
+[EXECUTION]
+MODULE = module_a_runner, module_b_runner, module_b_runner
+
 [MODULE_A_RUNNER_RUN_1]
 ...
 

environment.yml

@@ -4,6 +4,7 @@ channels:
 dependencies:
   - python=3.9
   - pip>=21.2.4
+  - numpy==1.21.6
   - astropy==5.0
   - automake==1.16.2
   - autoconf==2.69
@@ -15,6 +16,7 @@ dependencies:
   - numba==0.54.1
   - pandas==1.4.1
   - pip:
+    - cs_util==0.0.5
     - mccd==1.2.3
     - modopt==1.6.0
     - PyQt5==5.15.6

example/cfis/config_GitFeGie_symlink.ini

@@ -66,7 +66,7 @@ NUMBERING_SCHEME =
 
 # Input path where original images are stored. Can be local path or vos url.
 # Single string or list of strings
-INPUT_PATH = $HOME/astro/data/CFIS/tiles_DR2, $HOME/astro/data/CFIS/tiles_DR2
+INPUT_PATH = $SP_RUN/data_tiles, $SP_RUN/data_tiles
 
 # Input file pattern including tile number as dummy template
 INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight
@@ -98,6 +98,7 @@ FILE_EXT = .fits
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
 
+COLNUM = 3
 
 # Get exposures
 [GET_IMAGES_RUNNER_RUN_2]
@@ -119,7 +120,7 @@ NUMBERING_SCHEME = -000-000
 
 # Input path where original images are stored. Can be local path or vos url.
 # Single string or list of strings
-INPUT_PATH = $HOME/astro/data/CFIS/pitcairn_DR2, $HOME/astro/data/CFIS/weights_DR2, $HOME/astro/data/CFIS/flags_DR2
+INPUT_PATH = $SP_RUN/data_exp, $SP_RUN/data_exp, $SP_RUN/data_exp
 
 # Input file pattern including tile number as dummy template
 INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag

example/cfis/config_MaMa_onthefly.ini

@@ -0,0 +1,105 @@
+# ShapePipe configuration file for masking of tiles and exposures
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_MaMa
+
+# Add date and time to RUN_NAME, optional, default: False
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = mask_runner, mask_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 8
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+### Mask tiles
+[MASK_RUNNER_RUN_1]
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Input file pattern(s), list of strings with length matching number of expected input file types
+# Cannot contain wild cards
+FILE_PATTERN = CFIS_image, CFIS_weight
+
+# FILE_EXT (optional) list of string extensions to identify input files
+FILE_EXT = .fits, .fits
+
+# Path of mask config file
+MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask
+
+# External mask file flag, use if True, otherwise ignore
+USE_EXT_FLAG = False
+
+# External star catalogue flag, use external cat if True,
+# obtain from online catalogue if False
+USE_EXT_STAR = False
+
+# File name suffix for the output flag files (optional)
+PREFIX = pipeline
+
+### Mask exposures
+[MASK_RUNNER_RUN_2]
+
+# Parent module
+INPUT_DIR = last:split_exp_runner
+
+# Update numbering convention, accounting for HDU number of
+# single-exposure single-HDU files
+NUMBERING_SCHEME = -0000000-0
+
+# Path of mask config file
+MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask
+
+# External mask file flag, use if True, otherwise ignore
+USE_EXT_FLAG = True
+
+# External star catalogue flag, use external cat if True,
+# obtain from online catalogue if False
+USE_EXT_STAR = False
+
+# File name suffix for the output flag files (optional)
+PREFIX = pipeline

example/cfis/config_MaMa_save.ini

@@ -0,0 +1,109 @@
+# ShapePipe configuration file for masking of tiles and exposures
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_MaMa
+
+# Add date and time to RUN_NAME, optional, default: False
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = mask_runner, mask_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 8
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+### Mask tiles
+[MASK_RUNNER_RUN_1]
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, star_cat_tiles
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Input file pattern(s), list of strings with length matching number of expected input file types
+# Cannot contain wild cards
+FILE_PATTERN = CFIS_image, CFIS_weight, star_cat
+
+# FILE_EXT (optional) list of string extensions to identify input files
+FILE_EXT = .fits, .fits, .cat
+
+# Path of mask config file
+MASK_CONFIG_PATH = $SP_CONFIG/config_tile_save.mask
+
+# External mask file flag, use if True, otherwise ignore
+USE_EXT_FLAG = False
+
+# External star catalogue flag, use external cat if True,
+# obtain from online catalogue if False
+USE_EXT_STAR = True
+
+# File name suffix for the output flag files (optional)
+PREFIX = pipeline
+
+### Mask exposures
+[MASK_RUNNER_RUN_2]
+
+# Parent module
+INPUT_DIR = last:split_exp_runner, star_cat_exp
+
+# Update numbering convention, accounting for HDU number of
+# single-exposure single-HDU files
+NUMBERING_SCHEME = -0000000-0
+
+FILE_PATTERN = image, weight, flag, star_cat
+
+FILE_EXT = .fits, .fits, .fits, .cat
+
+# Path of mask config file
+MASK_CONFIG_PATH = $SP_CONFIG/config_save.mask
+
+# External mask file flag, use if True, otherwise ignore
+USE_EXT_FLAG = True
+
+# External star catalogue flag, use external cat if True,
+# obtain from online catalogue if False
+USE_EXT_STAR = True
+
+# File name suffix for the output flag files (optional)
+PREFIX = pipeline

example/cfis/config_MsPl_mccd.ini

@@ -13,7 +13,7 @@ VERBOSE = True
 RUN_NAME = run_sp_MsPl
 
 # Add date and time to RUN_NAME, optional, default: False
-RUN_DATETIME = False
+RUN_DATETIME = True
 
 
 ## ShapePipe execution options

example/cfis/config_Pl_mccd.ini

@@ -65,9 +65,12 @@ PLOT_MEANSHAPES = True
 X_GRID = 5
 Y_GRID = 10
 
-# Optional: max values for elliptity and residual ellipticities
-MAX_E = 0.05
-MAX_DE = 0.005
+# Optional: max values for focal plan plots (meanshape)
+MAX_E = 0.1
+MAX_DE = 0.01
+MIN_R2 = 4.5
+MAX_R2 = 7
+MAX_DR2 = 0.03
 
 PLOT_HISTOGRAMS = True
 REMOVE_OUTLIERS = False

example/cfis/config_Pl_psfex.ini

@@ -66,8 +66,11 @@ X_GRID = 5
 Y_GRID = 10
 
 # Optional: max values for elliptity and residual ellipticities
-MAX_E = 0.05
-MAX_DE = 0.005
+MAX_E = 0.1
+MAX_DE = 0.01
+MIN_R2 = 4.5
+MAX_R2 = 7
+MAX_DR2 = 0.03
 
 PLOT_HISTOGRAMS = True
 REMOVE_OUTLIERS = False