Merge pull request #365 from LSSTDESC/ceci2

Change to ceci version 2
LSSTDESC · Sep 3, 2024 · 3710fee · 3710fee
2 parents 79aa3e5 + 2fe79a3
commit 3710fee
Show file tree

Hide file tree

Showing 47 changed files with 1,613 additions and 1,693 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,7 +13,7 @@ env:
   EXAMPLE_DATA_FILE_VERSION: v3
   # Setting this did not appear to work. Instead will need to
   # find/replace when changing it.
-  # CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe:latest
+  # CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe-dev:latest
 
 jobs:
   # Run a download step first so that it is in
@@ -45,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:latest
+      image: ghcr.io/lsstdesc/txpipe:v1.0
 
     steps:
     - name: Checkout repository
@@ -55,6 +55,7 @@ jobs:
 
     - name: Test with pytest
       run: |
+        ceci --version
         pytest txpipe
 
 
@@ -64,7 +65,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:latest
+      image: ghcr.io/lsstdesc/txpipe:v1.0
 
     steps:
     - name: Checkout repository
@@ -101,7 +102,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:latest
+      image: ghcr.io/lsstdesc/txpipe:v1.0
 
     steps:
     - name: Checkout repository
@@ -148,7 +149,7 @@ jobs:
       needs: Download_Data
 
       container:
-        image: ghcr.io/lsstdesc/txpipe:latest
+        image: ghcr.io/lsstdesc/txpipe:v1.0
 
       steps:
       - name: Checkout repository
@@ -187,7 +188,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:latest
+      image: ghcr.io/lsstdesc/txpipe:v1.0
 
     steps:
     - name: Checkout repository
@@ -226,7 +227,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:latest
+      image: ghcr.io/lsstdesc/txpipe:v1.0
 
     steps:
     - name: Checkout repository
@@ -262,7 +263,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:latest
+      image: ghcr.io/lsstdesc/txpipe:v1.0
 
     steps:
     - name: Checkout repository
@@ -283,11 +284,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-12]
+        os: [ubuntu-latest, macos-14]
         include:
           - os: ubuntu-latest
             INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget
-          - os: macos-12
+          - os: macos-14
             INSTALL_DEPS: brew update-reset && brew install wget
     steps:
     - name: Checkout repository

diff --git a/environment-nopip.yml b/environment-nopip.yml
@@ -4,6 +4,7 @@ dependencies:
   - astropy=6.1.*
   - camb=1.5.*
   - cosmosis=3.7.*
+  - cmake
   - dask=2024.5.2
   - dm-tree=0.1.8
   - fitsio=1.2.*
@@ -19,7 +20,7 @@ dependencies:
   - mpich>=4.*
   - namaster=2.0.*
   - numpy=1.26.*
-  - pandas=2.2.*
+  - pandas>=2.0
   - psutil=5.9.*
   - pyccl=3.0.*
   - python=3.10.*

diff --git a/environment-piponly.yml b/environment-piponly.yml
@@ -1,6 +1,6 @@
 dependencies:
   - pip:
-    - ceci==1.17
+    - ceci==2.0.1
     - git+https://github.com/jlvdb/hyperbolic@b88b107a291fa16c2006cf971ce610248d58e94c
     - dask-mpi>=2022.4.0
     - git+https://github.com/LSSTDESC/[email protected]
@@ -10,4 +10,18 @@ dependencies:
     - jaxlib==0.4.23
     - glass==2023.7
     - git+https://github.com/beckermr/hybrideb@a3198ea5bc8175542b058e5c1289d910f05fd99d
-    - pz-rail[algos] @ git+https://github.com/LSSTDESC/[email protected]
+    # we must specifically request versions of rail algorithms
+    # to ensure that we get the ceci v2 versions
+    - pz-rail[algos] @ git+https://github.com/LSSTDESC/[email protected]
+    - pz-rail-astro-tools==1.0.4
+    - pz-rail-base==1.0.4
+    - pz-rail-bpz==1.0.1
+    - pz-rail-cmnn==1.0.1
+    - pz-rail-dsps==0.0.3 # no pypi release for more than 0.3
+    - pz-rail-flexzboost==1.0.1
+    - pz-rail-fsps==1.0.1
+    - pz-rail-gpz-v1==1.0.2
+    - pz-rail-lephare==0.2 # no release for ceci2 yet - cannot use
+    - pz-rail-pzflow==1.0.1
+    - pz-rail-sklearn==1.0.1
+    - pz-rail-som==1.0.1
diff --git a/examples/2.2i/config.yml b/examples/2.2i/config.yml
@@ -6,7 +6,7 @@ TXMetacalGCRInput:
     cat_name: dc2_object_run2.2i_dr6c_with_metacal
 
 TXExposureInfo:
-    dc2_name: '1.2p'
+    dc2_name: 1.2p
 
 TXIngestStars:
     cat_name: dc2_object_run2.2i_dr6c
@@ -15,12 +15,12 @@ TXCosmoDC2Mock:
     cat_name: cosmoDC2_v1.1.4_image
     visits_per_band: 16
     extra_cols: redshift_true size_true shear_1 shear_2
-    flip_g2: True # to match metacal
+    flip_g2: true # to match metacal
 
 FlexZPipe:
-    has_redshift: False
+    has_redshift: false
     nz: 300
-    metacal_fluxes: False
+    metacal_fluxes: false
     chunk_rows: 100
 
 
@@ -32,36 +32,17 @@ PZPDFMLZ:
 
 TXPhotozPlotLens:
     name: TXPhotozPlotLens
-    aliases:
-        photoz_stack: lens_photoz_stack
-        nz_plot: source_nz
-
 TXPhotozPlotSource:
     name: TXPhotozPlotSource
-    aliases:
-        photoz_stack: shear_photoz_stack
-        nz_plot: lens_nz
-
-
 TXTruePhotozStackSource:
     name: TXTruePhotozStackSource
-    aliases:
-        tomography_catalog: shear_tomography_catalog
-        catalog: shear_catalog
-        weights_catalog: shear_catalog
-        photoz_stack: shear_photoz_stack
     weight_col: metacal/weight
     redshift_group: metacal
     zmax: 2.0
     nz: 201
 
 TXTruePhotozStackLens:
     name: TXTruePhotozStackLens
-    aliases:
-        tomography_catalog: lens_tomography_catalog
-        catalog: photometry_catalog
-        weights_catalog: dummy
-        photoz_stack: lens_photoz_stack
     redshift_group: photometry
     zmax: 2.0
     nz: 201
@@ -73,7 +54,7 @@ TXSourceTrueNumberDensity:
     chunk_rows: 100000
 
 TXSourceSelector:
-    input_pz: False
+    input_pz: false
     bands: riz #used for selection
     T_cut: 0.5
     s2n_cut: 10.0
@@ -98,36 +79,36 @@ TXRandomCat:
 TXTwoPoint:
     binslop: 0.1
     delta_gamma: 0.02
-    do_pos_pos: True
-    do_shear_shear: True
-    do_shear_pos: True
-    flip_g2: False  # False now seems to be right for metacal
+    do_pos_pos: true
+    do_shear_shear: true
+    do_shear_pos: true
+    flip_g2: false  # False now seems to be right for metacal
     min_sep: 2.5
     max_sep: 250
     nbins: 20
     verbose: 0
-    subtract_mean_shear: True
+    subtract_mean_shear: true
 
 TXSourceMaps:
     pixelization: healpix
     nside: 1024
-    sparse: True
+    sparse: true
 
 TXLensMaps:
     pixelization: healpix
     nside: 1024
-    sparse: True
+    sparse: true
 
 
 TXAuxiliarySourceMaps:
-    chunk_rows:  100000
-    sparse:  True
-    psf_prefix:  psf_
+    chunk_rows: 100000
+    sparse: true
+    psf_prefix: psf_
 
 TXAuxiliaryLensMaps:
-    chunk_rows:  100000
-    sparse:  True
-    bright_obj_threshold:  22.0
+    chunk_rows: 100000
+    sparse: true
+    bright_obj_threshold: 22.0
 
 TXSimpleMask:
     depth_cut: 23.0
@@ -140,8 +121,8 @@ TXLensDiagnosticPlots: {}
 
 TXTwoPointFourier:
     chunk_rows: 100000
-    flip_g2: True
-    flip_g1: True
+    flip_g2: true
+    flip_g1: true
     bandwidth: 200
     apodization_size: 0.0
     cache_dir: ./cache
@@ -156,7 +137,7 @@ TXLensingNoiseMaps:
 TXMeanLensSelector:
     # Mag cuts
     chunk_rows: 100000
-    lens_zbin_edges: [0.0,0.2,0.4]
+    lens_zbin_edges: [0.0, 0.2, 0.4]
     cperp_cut: 0.2
     r_cpar_cut: 13.5
     r_lo_cut: 16.0
@@ -168,7 +149,7 @@ TXMeanLensSelector:
 TXTruthLensSelector:
     # Mag cuts
     chunk_rows: 100000
-    lens_zbin_edges: [0.0,0.2,0.4]
+    lens_zbin_edges: [0.0, 0.2, 0.4]
     cperp_cut: 0.2
     r_cpar_cut: 13.5
     r_lo_cut: 16.0