From e98fc3debe4b6321a999331fcde8e520d5a64068 Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Mon, 23 Sep 2024 15:04:52 -0400 Subject: [PATCH 1/3] TRT-558 - Update service to use earthdata-varinfo==3.0.0. --- CHANGELOG.md | 14 ++++- ...egridding_Service_User_Documentation.ipynb | 4 +- .../config/HRS_varinfo_config.json | 58 +++++++------------ pip_requirements.txt | 2 +- 4 files changed, 36 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21356d0..cced31b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The Harmony Regridding Service follows semantic versioning. All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [v1.0.2] - 2024-09-23 + +### Changed +This version of the Harmony Regridding Service updates to use +`earthdata-varinfo==3.0.0`, which primarily requires updates to the +configuration file the Harmony Regridding Service uses to specify metadata +overrides and excluded science variables to `earthdata-varinfo`. + ## [v1.0.1] - 2024-06-20 ### Changed @@ -22,6 +30,6 @@ include updated documentation and files outlined by the For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. -[unreleased]: https://github.com/nasa/harmony-regridding-service/compare/1.0.1..HEAD -[v1.0.1]: https://github.com/nasa/harmony-regridding-service/compare/1.0.0..1.0.1 -[v1.0.0]: https://github.com/nasa/harmony-regridding-service/compare/0.0.4-legacy..1.0.0 +[v1.0.2]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.0.2 +[v1.0.1]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.0.1 +[v1.0.0]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.0.0 diff --git a/docs/Harmony_Regridding_Service_User_Documentation.ipynb b/docs/Harmony_Regridding_Service_User_Documentation.ipynb index 8821f6e..bfa12fc 100644 --- a/docs/Harmony_Regridding_Service_User_Documentation.ipynb +++ b/docs/Harmony_Regridding_Service_User_Documentation.ipynb @@ -16,7 +16,7 @@ "\n", "The Harmony Regridding Services is a [Harmony](https://harmony.earthdata.nasa.gov) backend service that transforms the data within a specified input granule to a single geographic grid. The initial version of this service is only compatible with geographic source and target grids. The transformed output is returned to the user in the form of a standard Harmony STAC object, which contains links to the output granule in an AWS S3 bucket.\n", "\n", - "The Harmony Regridding Service leverages the `sds-varinfo` Python package (see: [here](https://git.earthdata.nasa.gov/projects/SITC/repos/sds-varinfo/browse)) to map variable dependencies and group variables by horizontal spatial dimensions. `pyresample` (see [here](https://pyresample.readthedocs.io/en/latest/)) is used to perform the resampling in a horizontal spatial plane.\n", + "The Harmony Regridding Service leverages the `earthdata-varinfo` Python package (see: [here](https://github.com/nasa/earthdata-varinfo)) to map variable dependencies and group variables by horizontal spatial dimensions. `pyresample` (see [here](https://pyresample.readthedocs.io/en/latest/)) is used to perform the resampling in a horizontal spatial plane.\n", "\n", "\n", "## Capabilities\n", @@ -292,7 +292,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/harmony_regridding_service/config/HRS_varinfo_config.json b/harmony_regridding_service/config/HRS_varinfo_config.json index 494275d..2deba03 100644 --- a/harmony_regridding_service/config/HRS_varinfo_config.json +++ b/harmony_regridding_service/config/HRS_varinfo_config.json @@ -1,7 +1,7 @@ { - "Identification": "trajectorysubsetter_varinfo_config", - "Version": 2, - "Collection_ShortName_Path": [ + "Identification": "harmony_regridding_service_varinfo_config", + "Version": 3, + "CollectionShortNamePath": [ "/HDF5_GLOBAL/short_name", "/NC_GLOBAL/short_name", "/Metadata/DatasetIdentification/shortName", @@ -16,58 +16,44 @@ "Mission": { "ATL\\d{2}": "ICESat2" }, - "Excluded_Science_Variables": [ + "ExcludedScienceVariables": [ { "Applicability": { "Mission": "ICESat2" }, - "Variable_Pattern": [ + "VariablePattern": [ "/quality_assessment/.*", "/orbit_info/.*", "/atlas_impulse_response/.*" ] } ], - "ProductEpochs": [ + "MetadataOverrides": [ { "Applicability": { - "Mission": "ICESat2" + "Mission": "ICESat2", + "ShortNamePath": "ATL14", + "VariablePattern": "/tile_stats/x" }, - "Epoch": "2005-01-01T00:00:00.000000" - } - ], - "CF_Overrides": [ + "Attributes": [ + { + "Name": "standard_name", + "Value": "projection_x_coordinate" + } + ] + }, { "Applicability": { "Mission": "ICESat2", - "ShortNamePath": "ATL14" + "ShortNamePath": "ATL14", + "VariablePattern": "/tile_stats/y" }, - "Applicability_Group": [ + "Attributes": [ { - "Applicability": { - "Variable_Pattern": "/tile_stats/x" - }, - "Attributes": [ - { - "Name": "standard_name", - "Value": "projection_x_coordinate" - } - ] - }, - { - "Applicability": { - "ShortNamePath": "ATL14", - "Variable_Pattern": "/tile_stats/y" - }, - "Attributes": [ - { - "Name": "standard_name", - "Value": "projection_y_coordinate" - } - ] + "Name": "standard_name", + "Value": "projection_y_coordinate" } ] } - ], - "CF_Supplements": [] + ] } diff --git a/pip_requirements.txt b/pip_requirements.txt index 6607197..42c0fd9 100644 --- a/pip_requirements.txt +++ b/pip_requirements.txt @@ -1,5 +1,5 @@ # Open source packages available from PyPI -earthdata-varinfo ~= 2.0.0 +earthdata-varinfo ~= 3.0.0 harmony-service-lib ~= 1.0.25 numpy ~= 1.24.2 netCDF4 ~= 1.6.3 From 79887504d16e089a1a1cd4dfe194d05228394990 Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Mon, 23 Sep 2024 16:21:52 -0400 Subject: [PATCH 2/3] TRT-558 - Update local Docker image names to use harmony-regridding-service. --- CHANGELOG.md | 5 +++++ bin/build-image | 3 ++- bin/build-test | 9 +++++---- bin/clean-images | 10 ++++++---- bin/run-test | 7 ++++--- docker/service.Dockerfile | 4 ++-- docker/tests.Dockerfile | 11 ++++++----- harmony_regridding_service/exceptions.py | 2 +- 8 files changed, 31 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cced31b..194c6da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ This version of the Harmony Regridding Service updates to use configuration file the Harmony Regridding Service uses to specify metadata overrides and excluded science variables to `earthdata-varinfo`. +This update also ensures that the Docker images used for local testing are +named `ghcr.io/nasa/harmony-regridding-service` and +`ghcr.io/nasa/harmony-regridding-service-test`, to comply with the released +image name for the service, and the name that Harmony-in-a-Box is looking for. + ## [v1.0.1] - 2024-06-20 ### Changed diff --git a/bin/build-image b/bin/build-image index 150a194..17e487c 100755 --- a/bin/build-image +++ b/bin/build-image @@ -7,10 +7,11 @@ # 2022-01-03: Updated Dockerfile path and added comments for tags. # 2023-01-26: Updated for the Harmony Regridding service. # 2024-04-12: Updated image name to: ghcr.io/nasa/harmony-regridder. +# 2024-09-23: Updated image name to: ghcr.io/nasa/harmony-regridding-service. # ############################################################################### -image="ghcr.io/nasa/harmony-regridder" +image="ghcr.io/nasa/harmony-regridding-service" # Retrieve the tag from the script arguments, or default to "latest". Bamboo # will pass the contents of `docker/service_version.txt` as this argument, diff --git a/bin/build-test b/bin/build-test index d547019..e083305 100755 --- a/bin/build-test +++ b/bin/build-test @@ -4,18 +4,19 @@ # Build a Docker container to run the test suite for the Harmony Regridding # service. # -# To build the ghcr.io/nasa/harmony-regridder-test image, the -# ghcr.io/nasa/harmony-regridder image must also be present locally, as it is -# used as the base image in `docker/tests.Dockerfile`. +# To build the ghcr.io/nasa/harmony-regridding-service-test image, the +# ghcr.io/nasa/harmony-regridding-service image must also be present locally, +# as it is used as the base image in `docker/tests.Dockerfile`. # # 2020-05-07: Adapted from SwotRepr. # 2022-01-03: Updated Dockerfile path. # 2023-01-26: Updated for the Harmony Regridding service. # 2024-04-12: Updated image name to: ghcr.io/nasa/harmony-regridder-test. +# 2024-09-23: Updated image name to: ghcr.io/nasa/harmony-regridding-service-test. # ############################################################################### -image="ghcr.io/nasa/harmony-regridder-test" +image="ghcr.io/nasa/harmony-regridding-service-test" tag=${1:-latest} diff --git a/bin/clean-images b/bin/clean-images index 2571549..ae9a180 100755 --- a/bin/clean-images +++ b/bin/clean-images @@ -1,12 +1,14 @@ #!/bin/bash ############################################################################### # -# Look for all versions of images containing the string "ghcr.io/nasa/harmony-regridder" -# and remove them. This is primarily for clean-up after development. +# Look for all versions of images containing the string +# "ghcr.io/nasa/harmony-regridding-service" and remove them. This is primarily +# for clean-up after development. # # 2020-05-07: Adapted from SwotRepr. # 2023-01-26: Updated for the Harmony Regridding Service. -# 2024-04-12: Update to account for new open-source Docker image names. +# 2024-04-12: Updated to account for new open-source Docker image names. +# 2024-09-23: Updated Docker image name to "ghcr.io/nasa/harmony-regridding-service". # ############################################################################### remove_image_by_name() { @@ -18,7 +20,7 @@ remove_image_by_name() { fi } -image_base_name="ghcr.io/nasa/harmony-regridder" +image_base_name="ghcr.io/nasa/harmony-regridding-service" # First remove test images: remove_image_by_name "${image_base_name}-test" diff --git a/bin/run-test b/bin/run-test index e1da17f..4be34b5 100755 --- a/bin/run-test +++ b/bin/run-test @@ -1,13 +1,14 @@ #!/bin/bash ############################################################################### # -# Execute the ghcr.io/nasa/harmony-regridder-test Docker image as a container, -# either locally or within a GitHub Action. +# Execute the ghcr.io/nasa/harmony-regridding-service-test Docker image as a +# container, either locally or within a GitHub Action. # # 2020-05-07: Adapted from SwotRepr. # 2021-01-05: Updated for JUnit output. # 2023-01-26: Updated for the Harmony Regridding service. # 2024-04-12: Updated to use new open-source Docker image names. +# 2024-09-23: Updated to use renamed test Docker image. # ############################################################################### @@ -27,4 +28,4 @@ mkdir -p coverage docker run --rm \ -v $(pwd)/test-reports:/home/tests/reports \ -v $(pwd)/coverage:/home/tests/coverage \ - ghcr.io/nasa/harmony-regridder-test "$@" + ghcr.io/nasa/harmony-regridding-service-test "$@" diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index 325476f..2fbda1a 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -1,7 +1,7 @@ ############################################################################### # -# Service image for ghcr.io/nasa/harmony-regridder, a Harmony backend service -# that transforms Level 3 or Level 4 data to another grid. +# Service image for ghcr.io/nasa/harmony-regridding-service, a Harmony backend +# service that transforms Level 3 or Level 4 data to another grid. # # This image installs dependencies via Pip. The service code is then copied # into the Docker image. diff --git a/docker/tests.Dockerfile b/docker/tests.Dockerfile index f0cd4ad..64304fd 100644 --- a/docker/tests.Dockerfile +++ b/docker/tests.Dockerfile @@ -1,15 +1,16 @@ ############################################################################### # # Test image for the Harmony Regridding service. This test image uses the main -# service image, ghcr.io/nasa/harmony-regridder, as a base layer for the tests. -# This ensures that the contents of the service image are tested, preventing -# discrepancies between the service and test environments. +# service image, ghcr.io/nasa/harmony-regridding-service, as a base layer for +# the tests. This ensures that the contents of the service image are tested, +# preventing discrepancies between the service and test environments. # # 2023-01-26: Added to repository. -# 2024-04-12: Update base image to ghcr.io/nasa/harmony-regridder. +# 2024-04-12: Updated base image to ghcr.io/nasa/harmony-regridder. +# 2024-09-23: Updated base image name to ghcr.io/nasa/harmony-regridding-service. # ############################################################################### -FROM ghcr.io/nasa/harmony-regridder +FROM ghcr.io/nasa/harmony-regridding-service ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/harmony_regridding_service/exceptions.py b/harmony_regridding_service/exceptions.py index 618171f..0de26b7 100644 --- a/harmony_regridding_service/exceptions.py +++ b/harmony_regridding_service/exceptions.py @@ -11,7 +11,7 @@ class RegridderException(HarmonyException): """Base service exception.""" def __init__(self, message=None): - super().__init__(message, 'sds/harmony-regridder') + super().__init__(message, 'ghcr.io/nasa/harmony-regridding-service') class InvalidTargetCRS(RegridderException): From c1ca806ac361f8ec46069677ce45380b00364b77 Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Mon, 23 Sep 2024 16:30:54 -0400 Subject: [PATCH 3/3] TRT-558 - Simplify service name in base service exception. --- harmony_regridding_service/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harmony_regridding_service/exceptions.py b/harmony_regridding_service/exceptions.py index 0de26b7..e4ed1e5 100644 --- a/harmony_regridding_service/exceptions.py +++ b/harmony_regridding_service/exceptions.py @@ -11,7 +11,7 @@ class RegridderException(HarmonyException): """Base service exception.""" def __init__(self, message=None): - super().__init__(message, 'ghcr.io/nasa/harmony-regridding-service') + super().__init__(message, 'harmony-regridding-service') class InvalidTargetCRS(RegridderException):