Merge branch 'main' into first_frame_bright

CHANGES.rst

@@ -1,3 +1,12 @@
+1.16.1 (2024-10-30)
+===================
+
+resample_spec
+-------------
+
+- Update NIRSpec spectral resampling to add a missing correction factor in resampled
+  WCS tangent plane transformation. [#8908]
+
 1.16.0 (2024-09-20)
 ===================
 

README.md

@@ -216,12 +216,13 @@ the specified context and less than the context for the next release.
 
 | jwst tag            | DMS build | SDP_VER  | CRDS_CONTEXT | Released   | Ops Install | Notes                                         |
 |---------------------|-----------|----------|--------------|------------|-------------|-----------------------------------------------|
-| 1.16.0              | B11.1rc1  | TBD      | 1281         | 2024-09-20 | TBD         | First release candidate for B11.1             |
-| 1.15.1              | B11.0     | 2024.2.2 | 1242         | 2024-07-08 | 2024-09-12  | Final release candidate for B11.0             |
-| 1.15.0              | B11.0rc1  |          | 1241         | 2024-06-26 |             | First release candidate for B11.0             |
-| 1.14.1              |           |          | 1238         | 2024-06-27 |             | PyPI-only release for external users          |
-| 1.14.0              | B10.2.1   | 2024.1.1 | 1238         | 2024-03-29 | 2024-06-12  | Final release candidate for B10.2.1           |
-| 1.13.4              |           |          | 1185         | 2024-01-25 |             | PyPI-only release for external users          |
+| 1.16.1              | B11.1.1   | 2024.3.1 | 1298         | 2024-11-13 | TBD         | Final release candidate for B11.1             |
+| 1.16.0              | B11.1     | 2024.3.0 | 1298         | 2024-09-20 | TBD         | First release candidate for B11.1             |
+| 1.15.1              | B11.0     | 2024.2.2 | 1293         | 2024-07-08 | 2024-09-12  | Final release candidate for B11.0             |
+| 1.15.0              | B11.0rc1  |          | 1274         | 2024-06-26 |             | First release candidate for B11.0             |
+| 1.14.1              |           |          | 1240         | 2024-06-27 |             | PyPI-only release for external users          |
+| 1.14.0              | B10.2.1   | 2024.1.1 | 1240         | 2024-03-29 | 2024-06-12  | Final release candidate for B10.2.1           |
+| 1.13.4              |           |          | 1210         | 2024-01-25 |             | PyPI-only release for external users          |
 | 1.13.3              | B10.1     | 2023.4.0 | 1181         | 2024-01-05 |             | Final release candidate for B10.1             |
 | 1.13.2              | B10.1rc3  | 2023.4.0 | 1181         | 2023-12-21 |             | Third release candidate for B10.1             |
 | 1.13.1              | B10.1rc2  | 2023.4.0 | 1181         | 2023-12-19 |             | Second release candidate for B10.1            |

changes/8893.resample.rst

@@ -0,0 +1 @@
+Use s_region list to calculate output footprint instead of re-computing via WCS transforms

changes/8909.scripts.rst

@@ -0,0 +1 @@
+Remove the outdated schema_editor script.

changes/8916.background.rst

@@ -0,0 +1 @@
+Apply bitwise operations in correct order when counting good pixels in the background mask during WFSS background subtraction.

changes/8927.combine_1d.rst

@@ -0,0 +1 @@
+Fix wavelength sort order for single input spectrum.

changes/8927.master_background.rst

@@ -0,0 +1 @@
+Fix ModelContainer handling for user background input.

changes/8927.pipeline.rst

@@ -0,0 +1 @@
+Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``.

changes/8935.general.rst

@@ -0,0 +1 @@
+Include xml and db test data as package data for lib tests.

changes/8945.stpipe.rst

@@ -0,0 +1 @@
+Remove all uses of Step.__call__ to allow it's deprecation.

changes/8950.general.rst

@@ -0,0 +1 @@
+Update minimum required version of crds to allow for "data release style" contexts.

changes/8958.general.rst

@@ -0,0 +1 @@
+When blending metadata don't store columns containing all missing value (nans).

docs/jwst/dark_current/description.rst

@@ -6,8 +6,7 @@ Description
 
 Assumptions
 -----------
-It is assumed that the input science data have *NOT* had the zero group (or
-bias) subtracted. We also do not want the dark subtraction process to remove
+We do not want the dark subtraction process to remove
 the bias signal from the science exposure, therefore the dark reference data
 should have their own group zero subtracted from all groups. This means that
 group zero of the dark reference data will effectively be zero-valued.

jwst/assign_mtwcs/moving_target_wcs.py

@@ -18,6 +18,8 @@
 
 from jwst.datamodels import ModelLibrary
 from jwst.stpipe.utilities import record_step_status
+from jwst.assign_wcs.util import update_s_region_imaging
+from jwst.lib.exposure_types import IMAGING_TYPES
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
@@ -69,6 +71,8 @@ def assign_moving_target_wcs(input_models):
                                     model.meta.wcsinfo.mt_ra, model.meta.wcsinfo.mt_dec)
                 del model.meta.wcs
                 model.meta.wcs = new_wcs
+            if model.meta.exposure.type.lower() in IMAGING_TYPES:
+                update_s_region_imaging(model)
             record_step_status(model, "assign_mtwcs", True)
             input_models.shelve(model, i, modify=True)
 

jwst/background/background_sub.py

@@ -272,6 +272,20 @@ def average_background(input_model, bkg_list, sigma, maxiters):
     return avg_bkg
 
 
+def sufficient_background_pixels(dq_array, bkg_mask, min_pixels=100):
+    """Count number of good pixels for background use.
+
+    Check DQ flags of pixels selected for bkg use - XOR the DQ values with
+    the DO_NOT_USE flag to flip the DO_NOT_USE bit. Then count the number
+    of pixels that AND with the DO_NOT_USE flag, i.e. initially did not have
+    the DO_NOT_USE bit set.
+    """
+    return np.count_nonzero((dq_array[bkg_mask]
+                            ^ pixel['DO_NOT_USE'])
+                            & pixel['DO_NOT_USE']
+                            ) > min_pixels
+
+
 def subtract_wfss_bkg(input_model, bkg_filename, wl_range_name, mmag_extract=None):
     """Scale and subtract a background reference image from WFSS/GRISM data.
 
@@ -310,18 +324,12 @@ def subtract_wfss_bkg(input_model, bkg_filename, wl_range_name, mmag_extract=Non
     # i.e. in regions we can use as background.
     if got_catalog:
         bkg_mask = mask_from_source_cat(input_model, wl_range_name, mmag_extract)
-        # Ensure mask has 100 pixels and that those pixels correspond to valid
-        # pixels using model DQ array
-        if np.count_nonzero(input_model.dq[bkg_mask]
-                            ^ pixel['DO_NOT_USE']
-                            & pixel['DO_NOT_USE']
-                            ) < 100:
+        if not sufficient_background_pixels(input_model.dq, bkg_mask):
             log.warning("Not enough background pixels to work with.")
             log.warning("Step will be SKIPPED.")
             return None
     else:
         bkg_mask = np.ones(input_model.data.shape, dtype=bool)
-
     # Compute the mean values of science image and background reference
     # image, including only regions where there are no identified sources.
     # Exclude pixel values in the lower and upper 25% of the histogram.

jwst/background/tests/test_background.py

@@ -9,11 +9,13 @@
 from numpy.testing import assert_allclose
 
 from stdatamodels.jwst import datamodels
+from stdatamodels.jwst.datamodels.dqflags import pixel
 
 from jwst.assign_wcs import AssignWcsStep
 from jwst.background import BackgroundStep
 from jwst.stpipe import Step
-from jwst.background.background_sub import robust_mean, mask_from_source_cat, no_NaN
+from jwst.background.background_sub import (robust_mean, mask_from_source_cat,
+                                            no_NaN, sufficient_background_pixels)
 
 
 @pytest.fixture(scope="module")
@@ -403,3 +405,23 @@ def test_no_nan():
 
     # Make sure arrays are equal.
     assert np.array_equal(model.data, result.data)
+
+
+def test_sufficient_background_pixels():
+    model = datamodels.ImageModel(data=np.zeros((2048, 2048)),
+                                  dq=np.zeros((2048, 2048)))
+    refpix_flags = pixel['DO_NOT_USE'] | pixel['REFERENCE_PIXEL']
+    model.dq[:4, :] = refpix_flags
+    model.dq[-4:, :] = refpix_flags
+    model.dq[:, :4] = refpix_flags
+    model.dq[:, -4:] = refpix_flags
+
+    bkg_mask = np.ones((2048, 2048), dtype=bool)
+    # With full array minux refpix available for bkg, should be sufficient
+    assert sufficient_background_pixels(model.dq, bkg_mask)
+
+    bkg_mask[4: -4, :] = 0
+    bkg_mask[:, 4: -4] = 0
+    # Now mask out entire array, mocking full source coverage of detector -
+    # no pixels should be available for bkg
+    assert not sufficient_background_pixels(model.dq, bkg_mask)

jwst/combine_1d/combine1d.py

@@ -355,7 +355,7 @@ def count_input(input_spectra):
         # only include spectra that have more than 1 data point
         if len(input_wl) > 1:
             if wl is None:
-                wl = input_wl
+                wl = input_wl.copy()
             else:
                 wl = np.hstack((input_wl, wl))
     wl.sort()

jwst/datamodels/__init__.py

@@ -31,7 +31,7 @@
 _jwst_models = ["ModelContainer", "SourceModelContainer", "ModelLibrary"]
 
 # Deprecated modules in stdatamodels
-_deprecated_modules = ['schema']
+_deprecated_modules = ['schema', 'schema_editor']
 
 # Deprecated models in stdatamodels
 _deprecated_models: list[str] = []
@@ -52,6 +52,6 @@
         sys.modules[f"jwst.datamodels.{attr}"] = obj
 
 # Add a few submodules to sys.modules without exposing them locally
-for _submodule_name in ['schema_editor', 'validate']:
+for _submodule_name in ['validate']:
     _submodule = importlib.import_module(f"stdatamodels.jwst.datamodels.{_submodule_name}")
     sys.modules[f"jwst.datamodels.{_submodule_name}"] = _submodule

jwst/engdblog/engdblog.py

@@ -1,4 +1,5 @@
 from ..lib.engdb_tools import ENGDB_Service
+from ..lib.engdb_mast import EngdbMast
 from ..stpipe import Step
 
 
@@ -21,10 +22,10 @@ class EngDBLogStep(Step):
     """
 
     spec = """
-    stime = string(default='2021-01-25')  # Start time
-    etime = string(default='2021-01-27')  # End time
+    stime = string(default='2022-01-25 02:00:00')  # Start time
+    etime = string(default='2022-01-26 02:10:00')  # End time
     verbosity = option('initial', 'all', default='initial')  # How much to report.
-    engdb_url = string(default='http://localhost')  # Mock url
+    engdb_url = string(default=None)  # Mock url
     """
 
     def process(self, mnemonics):
@@ -46,7 +47,10 @@ def process(self, mnemonics):
         stime = self.stime
         etime = self.etime
         verbosity = self.verbosity
-        edb = ENGDB_Service(base_url=self.engdb_url)
+        if self.engdb_url is not None:
+            edb = ENGDB_Service(base_url=self.engdb_url)
+        else:
+            edb = EngdbMast()
 
         if isinstance(mnemonics, str):
             mnemonics = [mnemonics]

jwst/engdblog/tests/test_engdblog.py

@@ -4,9 +4,8 @@
 import os
 import pytest
 from tempfile import TemporaryDirectory
-
+from jwst.lib import engdb_mast
 from jwst.engdblog import EngDBLogStep
-from jwst.lib.tests.engdb_mock import EngDB_Mocker
 
 
 def test_engdblogstep(caplog, engdb):
@@ -17,7 +16,7 @@ def test_engdblogstep(caplog, engdb):
     assert 'EngDBLogStep instance created' in caplog.text
     assert mnemonic in caplog.text
     assert "Step EngDBLogStep running with args (['{}'],)".format(mnemonic) in caplog.text
-    assert '{}[2021-01-25:2021-01-27] = '.format(mnemonic) in caplog.text
+    assert '{}[2022-01-25 02:00:00:2022-01-26 02:10:00] = '.format(mnemonic) in caplog.text
     assert 'Step EngDBLogStep done' in caplog.text
 
 
@@ -29,7 +28,7 @@ def test_barestring(caplog, engdb):
     assert 'EngDBLogStep instance created' in caplog.text
     assert mnemonic in caplog.text
     assert f"Step EngDBLogStep running with args ('{mnemonic}',)." in caplog.text
-    assert '{}[2021-01-25:2021-01-27] = '.format(mnemonic) in caplog.text
+    assert '{}[2022-01-25 02:00:00:2022-01-26 02:10:00] = '.format(mnemonic) in caplog.text
     assert 'Step EngDBLogStep done' in caplog.text
 
 
@@ -38,7 +37,7 @@ def test_badmnemonic(caplog, engdb):
     result = EngDBLogStep.call([mnemonic])
     assert isinstance(result, dict)
     assert len(result) == 0
-    assert 'Cannot retrieve info for {}'.format(mnemonic) in caplog.text
+    assert '{} has no entries in time range'.format(mnemonic) in caplog.text
 
 
 def test_novalues(caplog, engdb):
@@ -65,13 +64,12 @@ def test_multi_mnemonics(caplog, engdb):
     result = EngDBLogStep.call(mnemonics)
     assert len(result) == 2
     for mnemonic in mnemonics:
-        assert '{}[2021-01-25:2021-01-27] = '.format(mnemonic) in caplog.text
+        assert '{}[2022-01-25 02:00:00:2022-01-26 02:10:00] = '.format(mnemonic) in caplog.text
 
 
 # #####################
 # Utilities for testing
 # #####################
 @pytest.fixture
 def engdb():
-    with EngDB_Mocker() as mocker:  # noqa: F841
-        yield
+    yield engdb_mast.EngdbMast()

jwst/lib/engdb_mast.py

@@ -187,8 +187,6 @@ def configure(self, base_url=None, token=None):
         # Get the token
         if token is None:
             token = getenv('MAST_API_TOKEN', None)
-        if token is None:
-            raise RuntimeError('No MAST token provided but is required. See https://auth.mast.stsci.edu/ for more information.')
         self.token = token
 
         # Get various timeout parameters

jwst/lib/suffix.py

@@ -41,7 +41,7 @@
     'c1d', 'cal', 'calints', 'cat', 'crf', 'crfints',
     'dark',
     'i2d',
-    'median',
+    'mbsub', 'median',
     'phot', 'psf-amiavg', 'psfalign', 'psfstack', 'psfsub',
     'ramp', 'rate', 'rateints', 'residual_fringe',
     's2d', 's3d', 'snr',