tests fixed

cdsobs/cdm/api.py

@@ -349,7 +349,10 @@ def processing_level_fields(self) -> list[str]:
 
     def get_var_processing_level_field_name(self, var: str) -> str:
         return [
-            auxf["auxvar"] for auxf in self[var] if "processing_level" in auxf["auxvar"]
+            auxf["auxvar"]
+            for auxf in self[var]
+            if "processing_level" in auxf["auxvar"]
+            and "quality_flag" not in auxf["auxvar"]
         ][0]
 
 

cdsobs/ingestion/api.py

@@ -326,11 +326,10 @@ def _melt_variables(
             var_quality_flag = homogenised_data_melted.loc[var_mask, flag_name]
             homogenised_data_melted.loc[var_mask, "quality_flag"] = var_quality_flag
             homogenised_data_melted = homogenised_data_melted.drop(flag_name, axis=1)
-
-        # Ensure is int and fill nans with 3 (missing according to the CDM)
-        homogenised_data_melted["quality_flag"] = (
-            homogenised_data_melted["quality_flag"].fillna(3).astype("int")
-        )
+            # Ensure is int and fill nans with 3 (missing according to the CDM)
+            homogenised_data_melted["quality_flag"] = (
+                homogenised_data_melted["quality_flag"].fillna(3).astype("int")
+            )
         # Add processing level
         vars_with_pl = aux_fields.vars_with_processing_level()
         if len(vars_with_pl) > 0:
@@ -344,7 +343,6 @@ def _melt_variables(
             ] = var_processing_level
             homogenised_data_melted = homogenised_data_melted.drop(pl_name, axis=1)
 
-        homogenised_data_melted["processing_level"] = 6
     # Encode observed_variables
     logger.info("Encoding observed variables using the CDM variable codes.")
     code_table = read_cdm_code_table(cdm_tables_location, "observed_variable").table

cdsobs/utils/utils.py

@@ -62,10 +62,19 @@ def get_code_mapping(
         attrs = incobj["observed_variable"].attrs
     else:
         raise RuntimeError("Unsupported input type")
+    labels = attrs["labels"]
+    codes = attrs["codes"]
+    if not isinstance(attrs["labels"], list):
+        labels = [
+            labels,
+        ]
+        codes = [
+            codes,
+        ]
     if inverse:
-        mapping = {c: v for v, c in zip(attrs["labels"], attrs["codes"])}
+        mapping = {c: v for v, c in zip(labels, codes)}
     else:
-        mapping = {v: c for v, c in zip(attrs["labels"], attrs["codes"])}
+        mapping = {v: c for v, c in zip(labels, codes)}
     return mapping
 
 

tests/test_api.py

@@ -20,20 +20,20 @@
 @pytest.mark.parametrize(
     "dataset_name,source,test_update",
     [
-        # (
-        #     "insitu-observations-woudc-ozone-total-column-and-profiles",
-        #     "OzoneSonde",
-        #     False,
-        # ),
-        # (
-        #     "insitu-observations-woudc-ozone-total-column-and-profiles",
-        #     "TotalOzone",
-        #     False,
-        # ),
-        # ("insitu-observations-igra-baseline-network", "IGRA", False),
-        # ("insitu-observations-igra-baseline-network", "IGRA_H", False),
-        # ("insitu-comprehensive-upper-air-observation-network", "CUON", True),
-        # ("insitu-observations-gruan-reference-network", "GRUAN", False),
+        (
+            "insitu-observations-woudc-ozone-total-column-and-profiles",
+            "OzoneSonde",
+            False,
+        ),
+        (
+            "insitu-observations-woudc-ozone-total-column-and-profiles",
+            "TotalOzone",
+            False,
+        ),
+        ("insitu-observations-igra-baseline-network", "IGRA", False),
+        ("insitu-observations-igra-baseline-network", "IGRA_H", False),
+        ("insitu-comprehensive-upper-air-observation-network", "CUON", False),
+        ("insitu-observations-gruan-reference-network", "GRUAN", False),
         (
             "insitu-observations-near-surface-temperature-us-climate-reference-network",
             "uscrn_subhourly",
@@ -54,21 +54,21 @@
             "uscrn_monthly",
             False,
         ),
-        # (
-        #     "insitu-observations-gnss",
-        #     "IGS",
-        #     False,
-        # ),
-        # (
-        #     "insitu-observations-gnss",
-        #     "EPN",
-        #     False,
-        # ),
-        # (
-        #     "insitu-observations-gnss",
-        #     "IGS_R3",
-        #     False,
-        # ),
+        (
+            "insitu-observations-gnss",
+            "IGS",
+            False,
+        ),
+        (
+            "insitu-observations-gnss",
+            "EPN",
+            False,
+        ),
+        (
+            "insitu-observations-gnss",
+            "IGS_R3",
+            False,
+        ),
     ],
 )
 def test_run_ingestion_pipeline(
@@ -88,10 +88,16 @@ def test_run_ingestion_pipeline(
         update=False,
     )
     # assert insertions have been made
-    counter = test_session.scalar(sa.select(sa.func.count()).select_from(Catalogue))
+    counter = test_session.scalar(
+        sa.select(sa.func.count())
+        .select_from(Catalogue)
+        .where(Catalogue.dataset == dataset_name)
+    )
     assert counter > 0
     # Check variables
-    asset = test_session.scalar(sa.select(Catalogue.asset))
+    asset = test_session.scalar(
+        sa.select(Catalogue.asset).where(Catalogue.dataset == dataset_name)
+    )
     s3client = S3Client.from_config(test_config.s3config)
     asset_filename = asset.split("/")[1]
     asset_local_path = Path(tmp_path, asset_filename)

tests/test_cdm_api.py

@@ -115,7 +115,14 @@ def test_get_aux_fields_mapping_from_service_definition():
         "soil_moisture_50cm_from_earth_surface": [],
         "soil_moisture_5cm_from_earth_surface": [],
         "soil_temperature": [
-            {"auxvar": "soil_temperature_quality_flag", "metadata_name": "quality_flag"}
+            {
+                "auxvar": "soil_temperature_quality_flag",
+                "metadata_name": "quality_flag",
+            },
+            {
+                "auxvar": "soil_temperature_processing_level",
+                "metadata_name": "processing_level",
+            },
         ],
         "soil_temperature_100cm_from_earth_surface": [],
         "soil_temperature_10cm_from_earth_surface": [],
@@ -147,8 +154,10 @@ def test_get_aux_fields_mapping_from_service_definition():
         "minimum_solar_irradiance_quality_flag",
         "relative_humidity_quality_flag",
         "soil_temperature_quality_flag",
+        "soil_temperature_processing_level",
         "solar_irradiance_quality_flag",
     ]
+
     assert not actual.var_has_uncertainty_field("accumulated_precipitation")
     assert actual.var_has_uncertainty_field("air_temperature")
     assert actual.vars_with_uncertainty_field == ["air_temperature"]

tests/test_http_api.py

@@ -155,7 +155,12 @@ def test_get_dataset_auxiliary_variables_mapping():
         "soil_moisture_20cm_from_earth_surface": [],
         "soil_moisture_50cm_from_earth_surface": [],
         "soil_moisture_5cm_from_earth_surface": [],
-        "soil_temperature": [],
+        "soil_temperature": [
+            {
+                "auxvar": "soil_temperature_processing_level",
+                "metadata_name": "processing_level",
+            }
+        ],
         "soil_temperature_100cm_from_earth_surface": [],
         "soil_temperature_10cm_from_earth_surface": [],
         "soil_temperature_20cm_from_earth_surface": [],