Merge pull request #90 from ecmwf-projects/qos_tags

Qos tags
ecmwf-projects · Aug 30, 2023 · b086357 · b086357
2 parents 9e48b85 + 6b94b7f
commit b086357
Show file tree

Hide file tree

Showing 8 changed files with 43 additions and 0 deletions.
diff --git a/alembic/versions/4947a1be081e_new_column_qos_tags.py b/alembic/versions/4947a1be081e_new_column_qos_tags.py
@@ -0,0 +1,27 @@
+"""new column qos_tags.
+
+Revision ID: 4947a1be081e
+Revises: 875f93f41d79
+Create Date: 2023-08-30 11:55:00.678617
+
+"""
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql as dialect_postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4947a1be081e"
+down_revision = "875f93f41d79"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "resources", sa.Column("qos_tags", dialect_postgresql.ARRAY(sa.String))
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("resources", "qos_tags")
diff --git a/cads_catalogue/database.py b/cads_catalogue/database.py
@@ -199,6 +199,7 @@ class Resource(BaseModel):
     responsible_organisation_role = sa.Column(sa.String)
     responsible_organisation_website = sa.Column(sa.String)
     portal = sa.Column(sa.String, index=True)
+    qos_tags = sa.Column(dialect_postgresql.ARRAY(sa.String))
     title = sa.Column(sa.String)
     topic = sa.Column(sa.String)
     type = sa.Column(sa.String, nullable=False)

diff --git a/cads_catalogue/manager.py b/cads_catalogue/manager.py
@@ -365,6 +365,7 @@ def load_resource_metadata_file(folder_path: str | pathlib.Path) -> dict[str, An
 
     metadata["lineage"] = data.get("lineage")
     metadata["publication_date"] = data.get("publication_date")
+    metadata["qos_tags"] = data.get("qos_tags", [])
     metadata["related_resources_keywords"] = data.get("related_resources_keywords", [])
 
     metadata["representative_fraction"] = data.get("representative_fraction")

diff --git a/tests/data/cads-forms-json/reanalysis-era5-land/metadata.json b/tests/data/cads-forms-json/reanalysis-era5-land/metadata.json
@@ -46,6 +46,7 @@
     "Variable domain: Land (biosphere)",
     "Provider: Copernicus C3S"
   ],
+  "qos_tags": ["tag1", "tag2", "tag3"],
   "title": "ERA5-Land hourly data from 1950 to present",
   "abstract": "ERA5-Land is a reanalysis dataset providing a consistent view of the evolution of land variables over several decades at an enhanced resolution compared to ERA5. ERA5-Land has been produced by replaying the land component of the ECMWF ERA5 climate reanalysis. Reanalysis combines model data with observations from across the world into a globally complete and consistent dataset using the laws of physics. Reanalysis produces data that goes several decades back in time, providing an accurate description of the climate of the past."
 }
diff --git a/tests/data/dumped_resources.txt b/tests/data/dumped_resources.txt
@@ -1632,6 +1632,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "CAMS global reanalysis (EAC4)",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -4265,6 +4266,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "CAMS global reanalysis (EAC4) monthly averaged fields",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -5314,6 +5316,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "Near surface meteorological variables from 1979 to 2019 derived from bias-corrected reanalysis",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -7588,6 +7591,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": ["tag1", "tag2", "tag3"],
         "title": "ERA5-Land hourly data from 1950 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -8807,6 +8811,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "ERA5-Land monthly averaged data from 1950 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -12801,6 +12806,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "ERA5 hourly data on pressure levels from 1959 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -36513,6 +36519,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "ERA5 hourly data on single levels from 1959 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -38445,6 +38452,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "Surface radiation budget from 1982 to present derived from satellite observations",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",

diff --git a/tests/data/dumped_resources2.txt b/tests/data/dumped_resources2.txt
@@ -1627,6 +1627,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": ["tag1", "tag2", "tag3"],
         "title": "ERA5-Land hourly data from 1950 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -2262,6 +2263,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "ERA5-Land monthly averaged data from 1950 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",

diff --git a/tests/data/dumped_resources3.txt b/tests/data/dumped_resources3.txt
@@ -1627,6 +1627,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": ["tag1", "tag2", "tag3"],
         "title": "ERA5-Land hourly data from 1950 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",
@@ -2262,6 +2263,7 @@
         "responsible_organisation_role": "pointOfContact",
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "portal": "c3s",
+        "qos_tags": [],
         "title": "ERA5-Land monthly averaged data from 1950 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",

diff --git a/tests/test_40_manager.py b/tests/test_40_manager.py
@@ -1648,6 +1648,7 @@ def test_load_resource_from_folder() -> None:
         "responsible_organisation_website": "https://www.ecmwf.int/",
         "resource_update": "2023-02-17",
         "portal": "c3s",
+        "qos_tags": ["tag1", "tag2", "tag3"],
         "title": "ERA5-Land hourly data from 1950 to present",
         "topic": "climatologyMeteorologyAtmosphere",
         "type": "dataset",