diff --git a/cdsobs/cdm/api.py b/cdsobs/cdm/api.py index ea59361..d0dbb84 100644 --- a/cdsobs/cdm/api.py +++ b/cdsobs/cdm/api.py @@ -349,7 +349,10 @@ def processing_level_fields(self) -> list[str]: def get_var_processing_level_field_name(self, var: str) -> str: return [ - auxf["auxvar"] for auxf in self[var] if "processing_level" in auxf["auxvar"] + auxf["auxvar"] + for auxf in self[var] + if "processing_level" in auxf["auxvar"] + and "quality_flag" not in auxf["auxvar"] ][0] diff --git a/cdsobs/ingestion/api.py b/cdsobs/ingestion/api.py index 46c0e7e..b49655a 100644 --- a/cdsobs/ingestion/api.py +++ b/cdsobs/ingestion/api.py @@ -326,11 +326,10 @@ def _melt_variables( var_quality_flag = homogenised_data_melted.loc[var_mask, flag_name] homogenised_data_melted.loc[var_mask, "quality_flag"] = var_quality_flag homogenised_data_melted = homogenised_data_melted.drop(flag_name, axis=1) - - # Ensure is int and fill nans with 3 (missing according to the CDM) - homogenised_data_melted["quality_flag"] = ( - homogenised_data_melted["quality_flag"].fillna(3).astype("int") - ) + # Ensure is int and fill nans with 3 (missing according to the CDM) + homogenised_data_melted["quality_flag"] = ( + homogenised_data_melted["quality_flag"].fillna(3).astype("int") + ) # Add processing level vars_with_pl = aux_fields.vars_with_processing_level() if len(vars_with_pl) > 0: @@ -344,7 +343,6 @@ def _melt_variables( ] = var_processing_level homogenised_data_melted = homogenised_data_melted.drop(pl_name, axis=1) - homogenised_data_melted["processing_level"] = 6 # Encode observed_variables logger.info("Encoding observed variables using the CDM variable codes.") code_table = read_cdm_code_table(cdm_tables_location, "observed_variable").table diff --git a/cdsobs/utils/utils.py b/cdsobs/utils/utils.py index cdaf7af..b201c2e 100644 --- a/cdsobs/utils/utils.py +++ b/cdsobs/utils/utils.py @@ -62,10 +62,19 @@ def get_code_mapping( attrs = incobj["observed_variable"].attrs else: raise RuntimeError("Unsupported input type") + labels = attrs["labels"] + codes = attrs["codes"] + if not isinstance(attrs["labels"], list): + labels = [ + labels, + ] + codes = [ + codes, + ] if inverse: - mapping = {c: v for v, c in zip(attrs["labels"], attrs["codes"])} + mapping = {c: v for v, c in zip(labels, codes)} else: - mapping = {v: c for v, c in zip(attrs["labels"], attrs["codes"])} + mapping = {v: c for v, c in zip(labels, codes)} return mapping diff --git a/tests/test_api.py b/tests/test_api.py index 2af9e61..dde9daf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,20 +20,20 @@ @pytest.mark.parametrize( "dataset_name,source,test_update", [ - # ( - # "insitu-observations-woudc-ozone-total-column-and-profiles", - # "OzoneSonde", - # False, - # ), - # ( - # "insitu-observations-woudc-ozone-total-column-and-profiles", - # "TotalOzone", - # False, - # ), - # ("insitu-observations-igra-baseline-network", "IGRA", False), - # ("insitu-observations-igra-baseline-network", "IGRA_H", False), - # ("insitu-comprehensive-upper-air-observation-network", "CUON", True), - # ("insitu-observations-gruan-reference-network", "GRUAN", False), + ( + "insitu-observations-woudc-ozone-total-column-and-profiles", + "OzoneSonde", + False, + ), + ( + "insitu-observations-woudc-ozone-total-column-and-profiles", + "TotalOzone", + False, + ), + ("insitu-observations-igra-baseline-network", "IGRA", False), + ("insitu-observations-igra-baseline-network", "IGRA_H", False), + ("insitu-comprehensive-upper-air-observation-network", "CUON", False), + ("insitu-observations-gruan-reference-network", "GRUAN", False), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", "uscrn_subhourly", @@ -54,21 +54,21 @@ "uscrn_monthly", False, ), - # ( - # "insitu-observations-gnss", - # "IGS", - # False, - # ), - # ( - # "insitu-observations-gnss", - # "EPN", - # False, - # ), - # ( - # "insitu-observations-gnss", - # "IGS_R3", - # False, - # ), + ( + "insitu-observations-gnss", + "IGS", + False, + ), + ( + "insitu-observations-gnss", + "EPN", + False, + ), + ( + "insitu-observations-gnss", + "IGS_R3", + False, + ), ], ) def test_run_ingestion_pipeline( @@ -88,10 +88,16 @@ def test_run_ingestion_pipeline( update=False, ) # assert insertions have been made - counter = test_session.scalar(sa.select(sa.func.count()).select_from(Catalogue)) + counter = test_session.scalar( + sa.select(sa.func.count()) + .select_from(Catalogue) + .where(Catalogue.dataset == dataset_name) + ) assert counter > 0 # Check variables - asset = test_session.scalar(sa.select(Catalogue.asset)) + asset = test_session.scalar( + sa.select(Catalogue.asset).where(Catalogue.dataset == dataset_name) + ) s3client = S3Client.from_config(test_config.s3config) asset_filename = asset.split("/")[1] asset_local_path = Path(tmp_path, asset_filename) diff --git a/tests/test_cdm_api.py b/tests/test_cdm_api.py index c2b0181..4e316bd 100644 --- a/tests/test_cdm_api.py +++ b/tests/test_cdm_api.py @@ -115,7 +115,14 @@ def test_get_aux_fields_mapping_from_service_definition(): "soil_moisture_50cm_from_earth_surface": [], "soil_moisture_5cm_from_earth_surface": [], "soil_temperature": [ - {"auxvar": "soil_temperature_quality_flag", "metadata_name": "quality_flag"} + { + "auxvar": "soil_temperature_quality_flag", + "metadata_name": "quality_flag", + }, + { + "auxvar": "soil_temperature_processing_level", + "metadata_name": "processing_level", + }, ], "soil_temperature_100cm_from_earth_surface": [], "soil_temperature_10cm_from_earth_surface": [], @@ -147,8 +154,10 @@ def test_get_aux_fields_mapping_from_service_definition(): "minimum_solar_irradiance_quality_flag", "relative_humidity_quality_flag", "soil_temperature_quality_flag", + "soil_temperature_processing_level", "solar_irradiance_quality_flag", ] + assert not actual.var_has_uncertainty_field("accumulated_precipitation") assert actual.var_has_uncertainty_field("air_temperature") assert actual.vars_with_uncertainty_field == ["air_temperature"] diff --git a/tests/test_http_api.py b/tests/test_http_api.py index 64e0235..f846dbf 100644 --- a/tests/test_http_api.py +++ b/tests/test_http_api.py @@ -155,7 +155,12 @@ def test_get_dataset_auxiliary_variables_mapping(): "soil_moisture_20cm_from_earth_surface": [], "soil_moisture_50cm_from_earth_surface": [], "soil_moisture_5cm_from_earth_surface": [], - "soil_temperature": [], + "soil_temperature": [ + { + "auxvar": "soil_temperature_processing_level", + "metadata_name": "processing_level", + } + ], "soil_temperature_100cm_from_earth_surface": [], "soil_temperature_10cm_from_earth_surface": [], "soil_temperature_20cm_from_earth_surface": [],