Merge branch 'spacetelescope:main' into safe-extract2d

changes/8952.firstframe.rst

@@ -0,0 +1 @@
+Update the firstframe step to optionally not flag when a ramp saturates in group 3

changes/8975.resample.rst

@@ -0,0 +1 @@
+Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment variable

docs/jwst/firstframe/arguments.rst

@@ -1,4 +1,13 @@
 Step Arguments
 ==============
 
-The first frame correction has no step-specific arguments.
+The ``firstframe`` step has the following step-specific arguments.
+
+``--bright_use_group1`` (boolean, default=False)
+    If True, setting the group 1 groupdq to DO_NOT_USE will not be done 
+    for pixels that have the saturation flag set for group 3.  
+    This will allow a slope to be determined for pixels that saturate in group 3.
+    This change in flagging will only impact pixels that saturate in group 3, the behavior
+    for all other pixels will be unchanged.
+    The `bright_use_group1` flag can be set for all data, only data/pixels that saturate 
+    in group 3 will see a difference in behavior.

docs/jwst/outlier_detection/arguments.rst

@@ -79,15 +79,6 @@ Step Arguments for Imaging and Slit-like Spectroscopic data
   Specifies whether or not to resample the input images when
   performing outlier detection.
 
-``--allowed_memory``
-  Specifies the fractional amount of
-  free memory to allow when creating the resampled image. If ``None``, the
-  environment variable ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no
-  check is made. If the resampled image would be larger than specified, an
-  ``OutputTooLargeError`` exception will be generated.
-  For example, if set to ``0.5``, only resampled images that use less than half
-  the available memory can be created.
-
 ``--in_memory``
   Specifies whether or not to load and create all images that are used during
   processing into memory. If ``False``, input files are loaded from disk when

docs/jwst/resample/arguments.rst

@@ -108,15 +108,6 @@ image.
 ``--blendheaders`` (bool, default=True)
     Blend metadata from all input images into the resampled output image.
 
-``--allowed_memory`` (float, default=None)
-    Specifies the fractional amount of free memory to allow when creating the
-    resampled image. If ``None``, the environment variable
-    ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no check is made. If the
-    resampled image would be larger than specified, an ``OutputTooLargeError``
-    exception will be generated.
-
-    For example, if set to ``0.5``, only resampled images that use less than
-    half the available memory can be created.
 
 ``--in_memory`` (boolean, default=True)
   Specifies whether or not to load and create all images that are used during

jwst/firstframe/firstframe_step.py

@@ -16,6 +16,7 @@ class FirstFrameStep(Step):
     class_alias = "firstframe"
 
     spec = """
+        bright_use_group1 = boolean(default=False) # do not flag group1 if group3 is saturated   
     """
 
     def process(self, step_input):
@@ -25,16 +26,18 @@ def process(self, step_input):
 
             # check the data is MIRI data
             detector = input_model.meta.instrument.detector.upper()
-            if detector[:3] != 'MIR':
-                self.log.warning('First Frame Correction is only for MIRI data')
-                self.log.warning('First frame step will be skipped')
-                input_model.meta.cal_step.firstframe = 'SKIPPED'
+            if detector[:3] != "MIR":
+                self.log.warning("First Frame Correction is only for MIRI data")
+                self.log.warning("First frame step will be skipped")
+                input_model.meta.cal_step.firstframe = "SKIPPED"
                 return input_model
 
             # Cork on a copy
             result = input_model.copy()
 
             # Do the firstframe correction subtraction
-            result = firstframe_sub.do_correction(result)
+            result = firstframe_sub.do_correction(
+                result, bright_use_group1=self.bright_use_group1
+            )
 
         return result

jwst/firstframe/firstframe_sub.py

@@ -10,7 +10,7 @@
 log.setLevel(logging.DEBUG)
 
 
-def do_correction(output):
+def do_correction(output, bright_use_group1=False):
     """
     Short Summary
     -------------
@@ -22,6 +22,8 @@ def do_correction(output):
     ----------
     output: data model object
         science data to be corrected
+    bright_use_group1: boolean
+        do not flag group1 for bright pixels = group 3 saturated
 
     Returns
     -------
@@ -36,13 +38,29 @@ def do_correction(output):
     # Update the step status, and if ngroups > 3, set all GROUPDQ in
     # the first group to 'DO_NOT_USE'
     if sci_ngroups > 3:
-        output.groupdq[:, 0, :, :] = \
-            np.bitwise_or(output.groupdq[:, 0, :, :], dqflags.group['DO_NOT_USE'])
+        if bright_use_group1:
+            # do not set DO_NOT_USE in the case where saturation happens in
+            # group3 as in this case the first frame effect is small compared to the
+            # signal in group2-group1
+            svals = (output.groupdq[:, 2, :, :] & dqflags.group["SATURATED"]) > 0
+            tvals = output.groupdq[:, 0, :, :]
+            tvals[~svals] = np.bitwise_or(
+                (output.groupdq[:, 0, :, :])[~svals], dqflags.group["DO_NOT_USE"]
+            )
+            output.groupdq[:, 0, :, :] = tvals
+            log.info(
+                f"FirstFrame Sub: bright_first_frame set, #{np.sum(svals)} bright pixels group1 not set to DO_NOT_USE"
+            )
+        else:
+            output.groupdq[:, 0, :, :] = np.bitwise_or(
+                output.groupdq[:, 0, :, :], dqflags.group["DO_NOT_USE"]
+            )
+
         log.debug("FirstFrame Sub: resetting GROUPDQ in first frame to DO_NOT_USE")
-        output.meta.cal_step.firstframe = 'COMPLETE'
-    else:   # too few groups
+        output.meta.cal_step.firstframe = "COMPLETE"
+    else:  # too few groups
         log.warning("Too few groups to apply correction")
         log.warning("Step will be skipped")
-        output.meta.cal_step.firstframe = 'SKIPPED'
+        output.meta.cal_step.firstframe = "SKIPPED"
 
     return output

jwst/firstframe/tests/test_firstframe.py

@@ -9,7 +9,7 @@
 def test_firstframe_set_groupdq():
     """
     Test if the firstframe code set the groupdq flag on the first
-    group to 'do_not_use' for 5 integrations
+    group to 'do_not_use' for 5 groups
     """
 
     # size of integration
@@ -216,3 +216,60 @@ def test_miri():
                                   dq_diff,
                                   err_msg='Diff in groupdq flags is not '
                                           + 'equal to DO_NOT_USE')
+
+
+def test_firstframe_bright_use_group1():
+    """
+    Test if the firstframe code when bright_use_group1 is set to True.
+    
+    The groupdq flag for group 1 should not be set to DO_NOT_USE for the pixels that saturate 
+    the 3rd group.  Otherwise, all other pixels should have their group1 groupdq 
+    flags set to DO_NOT_USE.
+    """
+
+    # size of integration
+    ngroups = 5
+    xsize = 1032
+    ysize = 1024
+
+    # create the data and groupdq arrays
+    csize = (1, ngroups, ysize, xsize)
+    data = np.full(csize, 1.0)
+    groupdq = np.zeros(csize, dtype=int)
+
+    # set a fraction of the pixels to saturate in between the 2nd and 3rd groups
+    groupdq[0, 2, 0:100, :] = dqflags.group['SATURATED']
+
+    # set a fraction of the pixels to saturate in between the 1st and 2nd groups
+    groupdq[0, 2, 200:300, :] = dqflags.group['SATURATED']
+    groupdq[0, 1, 200:300, :] = dqflags.group['SATURATED']
+
+    # create a JWST datamodel for MIRI data
+    dm_ramp = RampModel(data=data, groupdq=groupdq)
+
+    # run the first frame correction step on a copy (the detection to make the copy or
+    # not would have happened at _step.py)
+    dm_ramp_firstframe = do_correction(dm_ramp.copy(), bright_use_group1=True)
+
+    # check that the difference in the groupdq flags is equal to
+    #   the 'do_not_use' flag
+    dq_diff = dm_ramp_firstframe.groupdq[0, 0, :, :] - dm_ramp.groupdq[0, 0, :, :]
+
+    expected_diff = np.full((ysize, xsize), dqflags.group['DO_NOT_USE'], dtype=int)
+    expected_diff[0:100, :] = 0
+    expected_diff[200:300, :] = 0
+
+    np.testing.assert_array_equal(expected_diff,
+                                  dq_diff,
+                                  err_msg='Diff in groupdq flags is not '
+                                  + 'equal to the DO_NOT_USE flag')
+
+    # test that the groupdq flags are not changed for the rest of the groups
+    dq_diff = (dm_ramp_firstframe.groupdq[0, 1:ngroups, :, :]
+               - dm_ramp.groupdq[0, 1:ngroups, :, :])
+    np.testing.assert_array_equal(np.full((ngroups - 1, ysize, xsize),
+                                          0,
+                                          dtype=int),
+                                  dq_diff,
+                                  err_msg='n >= 2 groupdq flags changes '
+                                  + 'and they should not be')

jwst/outlier_detection/imaging.py

@@ -35,7 +35,6 @@ def detect_outliers(
     pixfrac,
     kernel,
     fillval,
-    allowed_memory,
     in_memory,
     make_output_path,
 ):
@@ -65,7 +64,6 @@ def detect_outliers(
             kernel=kernel,
             fillval=fillval,
             good_bits=good_bits,
-            allowed_memory=allowed_memory,
         )
         median_data, median_wcs = median_with_resampling(input_models,
                                                     resamp,

jwst/outlier_detection/outlier_detection_step.py

@@ -59,7 +59,6 @@ class OutlierDetectionStep(Step):
         resample_data = boolean(default=True)
         good_bits = string(default="~DO_NOT_USE")  # DQ flags to allow
         search_output_file = boolean(default=False)
-        allowed_memory = float(default=None)  # Fraction of memory to use for the combined image
         in_memory = boolean(default=False) # ignored if run within the pipeline; set at pipeline level instead
     """
 
@@ -113,7 +112,6 @@ def process(self, input_data):
                 self.pixfrac,
                 self.kernel,
                 self.fillval,
-                self.allowed_memory,
                 self.in_memory,
                 self.make_output_path,
             )

jwst/outlier_detection/tests/test_outlier_detection.py

@@ -677,6 +677,5 @@ def make_resamp(input_models):
         good_bits="~DO_NOT_USE",
         in_memory=in_memory,
         asn_id="test",
-        allowed_memory=None,
     )
     return resamp

jwst/ramp_fitting/tests/test_ramp_fit.py

@@ -413,7 +413,6 @@ def test_photon_noise_only_bad_last_frame_two_groups(self, method):
         data = slopes.data
         np.testing.assert_allclose(data[50, 50], cds_slope, 1e-6)
 
-    @pytest.mark.skip(reason="Unweighted fit not implemented.")
     def test_photon_noise_with_unweighted_fit(self, method):
         model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=5, gain=1000, readnoise=1)
         model1.data[0, 0, 50, 50] = 10.0

jwst/regtest/test_fgs_image3.py

@@ -1,7 +1,6 @@
 import pytest
 from astropy.io.fits.diff import FITSDiff
 
-from jwst.resample.resample import OutputTooLargeError
 from jwst.stpipe import Step
 
 
@@ -34,23 +33,3 @@ def test_fgs_image3_catalog(run_fgs_image3, rtdata_module, diff_astropy_tables):
     rtdata.get_truth("truth/test_fgs_image3/jw01029-o001_t009_fgs_clear_cat.ecsv")
 
     assert diff_astropy_tables(rtdata.output, rtdata.truth, rtol=1e-3, atol=1e-4)
-
-
-@pytest.mark.bigdata
-def test_fgs_toobig(rtdata, fitsdiff_default_kwargs, caplog, monkeypatch):
-    """Test for the situation where the combined mosaic is too large"""
-
-    # Set the environment to not allow the resultant too-large image.
-    # Note: this test was originally run on two pre-flight images
-    # with WCSs from very different parts of the sky.
-    # This condition should hopefully never be encountered in reductions
-    # of in-flight data.  To test the software failsafe, we now use real data
-    # that makes a reasonable sized mosaic, but set the allowed memory to a
-    # small value.
-    monkeypatch.setenv('DMODEL_ALLOWED_MEMORY', "0.0001")
-
-    rtdata.get_asn('fgs/image3/jw01029-o001_20240716t172128_image3_00001_asn.json')
-
-    args = ['jwst.resample.ResampleStep', rtdata.input]
-    with pytest.raises(OutputTooLargeError):
-        Step.from_cmdline(args)

jwst/regtest/test_miri_image.py

@@ -32,6 +32,52 @@ def run_detector1(rtdata_module):
     Step.from_cmdline(args)
 
 
+@pytest.fixture(scope="module")
+def run_detector1_multiprocess_rate(rtdata_module):
+    """Run detector1 pipeline on MIRI imaging data."""
+    rtdata = rtdata_module
+    rtdata.get_data("miri/image/jw01024001001_04101_00001_mirimage_uncal.fits")
+
+    # Run detector1 pipeline only on one of the _uncal files
+    args = ["jwst.pipeline.Detector1Pipeline", rtdata.input,
+            "--save_calibrated_ramp=True",
+            "--steps.dq_init.save_results=True",
+            "--steps.saturation.save_results=True",
+            "--steps.firstframe.save_results=True",
+            "--steps.lastframe.save_results=True",
+            "--steps.reset.save_results=True",
+            "--steps.linearity.save_results=True",
+            "--steps.rscd.save_results=True",
+            "--steps.dark_current.save_results=True",
+            "--steps.refpix.save_results=True",
+            "--steps.ramp_fit.maximum_cores=2", # Multiprocessing
+            ]
+    Step.from_cmdline(args)
+
+
+@pytest.fixture(scope="module")
+def run_detector1_multiprocess_jump(rtdata_module):
+    """Run detector1 pipeline on MIRI imaging data."""
+    rtdata = rtdata_module
+    rtdata.get_data("miri/image/jw01024001001_04101_00001_mirimage_uncal.fits")
+
+    # Run detector1 pipeline only on one of the _uncal files
+    args = ["jwst.pipeline.Detector1Pipeline", rtdata.input,
+            "--save_calibrated_ramp=True",
+            "--steps.dq_init.save_results=True",
+            "--steps.saturation.save_results=True",
+            "--steps.firstframe.save_results=True",
+            "--steps.lastframe.save_results=True",
+            "--steps.reset.save_results=True",
+            "--steps.linearity.save_results=True",
+            "--steps.rscd.save_results=True",
+            "--steps.dark_current.save_results=True",
+            "--steps.refpix.save_results=True",
+            "--steps.jump.maximum_cores=2", # Multiprocessing
+            ]
+    Step.from_cmdline(args)
+
+
 @pytest.fixture(scope="module")
 def run_detector1_with_average_dark_current(rtdata_module):
     """Run detector1 pipeline on MIRI imaging data, providing an
@@ -115,6 +161,18 @@ def test_miri_image_detector1(run_detector1, rtdata_module, fitsdiff_default_kwa
     _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix)
 
 
+@pytest.mark.bigdata
+def test_miri_image_detector1_multiprocess_rate(run_detector1_multiprocess_rate, rtdata_module, fitsdiff_default_kwargs):
+    """Regression test of detector1 pipeline performed on MIRI imaging data."""
+    _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate")
+
+
+@pytest.mark.bigdata
+def test_miri_image_detector1_multiprocess_jump(run_detector1_multiprocess_jump, rtdata_module, fitsdiff_default_kwargs):
+    """Regression test of detector1 pipeline performed on MIRI imaging data."""
+    _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate")
+
+
 @pytest.mark.bigdata
 def test_detector1_mem_usage(rtdata_module):
     """Determine the memory usage for Detector 1"""

jwst/regtest/test_miri_mrs_badpix_selfcal.py

@@ -59,22 +59,19 @@ def test_miri_mrs_badpix_selfcal(run_pipeline_selfcal, fitsdiff_default_kwargs):
         assert not os.path.isfile(fname)
 
 
+@pytest.mark.parametrize("basename", (
+    [f"{OUTSTEM_BKG}_badpix_selfcal.fits",] + 
+    [f"{OUTSTEM_BKG}_badpix_selfcal_bkg_{idx}.fits" for idx in range(4)]))
 @pytest.mark.bigdata
-def test_miri_mrs_badpix_selfcal_bkg(run_pipeline_background, fitsdiff_default_kwargs):
+def test_miri_mrs_badpix_selfcal_bkg(basename, run_pipeline_background, fitsdiff_default_kwargs):
     """Run a test for MIRI MRS data with dedicated background exposures."""
 
     rtdata = run_pipeline_background
 
     # Get the truth file
-    rtdata.get_truth(f"truth/test_miri_mrs_badpix_selfcal/{OUTSTEM_BKG}_badpix_selfcal.fits")
+    rtdata.output = basename
+    rtdata.get_truth(f"truth/test_miri_mrs_badpix_selfcal/{basename}")
 
-    # Compare the results
+    # Compare the results and check the bkg files in the background case, but not in the selfcal case
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
     assert diff.identical, diff.report()
-
-    # check the bkg files in the background case, but not in the selfcal case
-    for idx in range(4):
-        fname = f"{OUTSTEM_BKG}_badpix_selfcal_bkg_{idx}.fits"
-        truth = rtdata.get_truth(f"truth/test_miri_mrs_badpix_selfcal/{fname}")
-        diff = FITSDiff(fname, truth, **fitsdiff_default_kwargs)
-        assert diff.identical, diff.report()