From c6f1123e9ab4c1b5e3c31008c5cbdee0d00d6779 Mon Sep 17 00:00:00 2001 From: Lars Buntemeyer Date: Fri, 6 Oct 2023 11:14:00 +0200 Subject: [PATCH 1/4] added CORDEX-Reklies facets --- esmvalcore/esgf/facets.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/esmvalcore/esgf/facets.py b/esmvalcore/esgf/facets.py index 3947ba2a6c..0b65aba21e 100644 --- a/esmvalcore/esgf/facets.py +++ b/esmvalcore/esgf/facets.py @@ -43,6 +43,17 @@ 'product': 'product', 'short_name': 'variable', }, + 'CORDEX-Reklies': { + 'dataset': 'rcm_name', + 'driver': 'driving_model', + 'domain': 'domain', + 'ensemble': 'ensemble', + 'exp': 'experiment', + 'frequency': 'time_frequency', + 'institute': 'institute', + 'product': 'product', + 'short_name': 'variable', + }, 'obs4MIPs': { 'dataset': 'source_id', 'frequency': 'time_frequency', @@ -73,6 +84,7 @@ }, 'CMIP6': {}, 'CORDEX': {}, + 'CORDEX-Reklies': {}, 'obs4MIPs': {}, } """Cache for the mapping between recipe/filesystem and ESGF dataset names.""" From 7524501d726b53eed6fe73719f69bc52e7bdaa45 Mon Sep 17 00:00:00 2001 From: Lars Buntemeyer Date: Fri, 6 Oct 2023 11:52:29 +0200 Subject: [PATCH 2/4] added adjust and ESD facets --- esmvalcore/esgf/facets.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/esmvalcore/esgf/facets.py b/esmvalcore/esgf/facets.py index 0b65aba21e..81aba0b4c4 100644 --- a/esmvalcore/esgf/facets.py +++ b/esmvalcore/esgf/facets.py @@ -54,6 +54,28 @@ 'product': 'product', 'short_name': 'variable', }, + 'CORDEX-Adjust': { + 'dataset': 'rcm_name', + 'driver': 'driving_model', + 'domain': 'domain', + 'ensemble': 'ensemble', + 'exp': 'experiment', + 'frequency': 'time_frequency', + 'institute': 'institute', + 'product': 'product', + 'short_name': 'variable', + }, + 'CORDEX-ESD': { + 'dataset': 'rcm_name', + 'driver': 'driving_model', + 'domain': 'domain', + 'ensemble': 'ensemble', + 'exp': 'experiment', + 'frequency': 'time_frequency', + 'institute': 'institute', + 'product': 'product', + 'short_name': 'variable', + }, 'obs4MIPs': { 'dataset': 'source_id', 'frequency': 'time_frequency', @@ -85,6 +107,8 @@ 'CMIP6': {}, 'CORDEX': {}, 'CORDEX-Reklies': {}, + 'CORDEX-Adjust': {}, + 'CORDEX-ESD': {}, 'obs4MIPs': {}, } """Cache for the mapping between recipe/filesystem and ESGF dataset names.""" From 7f47e17232c7f8a603ca579949f58b97b90e774a Mon Sep 17 00:00:00 2001 From: Lars Buntemeyer Date: Sat, 28 Oct 2023 23:18:19 +0200 Subject: [PATCH 3/4] added integation test for CORDEX-Adjust --- .../integration/esgf/test_search_download.py | 216 ++++++++++-------- 1 file changed, 123 insertions(+), 93 deletions(-) diff --git a/tests/integration/esgf/test_search_download.py b/tests/integration/esgf/test_search_download.py index 29e215be1d..5785657062 100644 --- a/tests/integration/esgf/test_search_download.py +++ b/tests/integration/esgf/test_search_download.py @@ -8,96 +8,122 @@ from esmvalcore.esgf import _search, download, find_files -VARIABLES = [{ - 'dataset': 'cccma_cgcm3_1', - 'ensemble': 'run1', - 'exp': 'historical', - 'frequency': 'mon', - 'project': 'CMIP3', - 'short_name': 'tas', - 'version': 'v1', -}, { - 'dataset': 'inmcm4', - 'ensemble': 'r1i1p1', - 'exp': ['historical', 'rcp85'], - 'mip': 'Amon', - 'project': 'CMIP5', - 'short_name': 'tas', - 'version': 'v20130207', -}, { - 'dataset': 'FIO-ESM', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'mip': 'Amon', - 'project': 'CMIP5', - 'short_name': 'tas', -}, { - 'dataset': 'HadGEM2-CC', - 'ensemble': 'r1i1p1', - 'exp': 'rcp85', - 'mip': 'Amon', - 'project': 'CMIP5', - 'short_name': 'tas', - 'timerange': '2080/2100', -}, { - 'dataset': 'EC-EARTH', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'mip': 'Amon', - 'project': 'CMIP5', - 'short_name': 'tas', - 'start_year': 1990, # test legacy way of specifying timerange - 'end_year': 1999, -}, { - 'dataset': 'AWI-ESM-1-1-LR', - 'ensemble': 'r1i1p1f1', - 'exp': 'historical', - 'grid': 'gn', - 'mip': 'Amon', - 'project': 'CMIP6', - 'short_name': 'tas', - 'timerange': '2000/2001', - 'version': 'v20200212', -}, { - 'dataset': 'CESM2', - 'ensemble': 'r4i1p1f1', - 'exp': 'historical', - 'grid': 'gn', - 'mip': 'Amon', - 'project': 'CMIP6', - 'short_name': 'tas', - 'timerange': '2000/2001', -}, { - 'dataset': 'RACMO22E', - 'driver': 'MOHC-HadGEM2-ES', - 'domain': 'EUR-11', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'frequency': 'mon', - 'project': 'CORDEX', - 'short_name': 'tas', - 'timerange': '1950/1952', - 'version': 'v20160620', -}, { - 'dataset': 'CERES-EBAF', - 'frequency': 'mon', - 'project': 'obs4MIPs', - 'short_name': 'rsutcs', - 'version': 'v20160610', -}, { - 'dataset': 'GPCP-V2.3', - 'project': 'obs4MIPs', - 'short_name': 'pr', -}] +VARIABLES = [ + { + 'dataset': 'cccma_cgcm3_1', + 'ensemble': 'run1', + 'exp': 'historical', + 'frequency': 'mon', + 'project': 'CMIP3', + 'short_name': 'tas', + 'version': 'v1', + }, + { + 'dataset': 'inmcm4', + 'ensemble': 'r1i1p1', + 'exp': ['historical', 'rcp85'], + 'mip': 'Amon', + 'project': 'CMIP5', + 'short_name': 'tas', + 'version': 'v20130207', + }, + { + 'dataset': 'FIO-ESM', + 'ensemble': 'r1i1p1', + 'exp': 'historical', + 'mip': 'Amon', + 'project': 'CMIP5', + 'short_name': 'tas', + }, + { + 'dataset': 'HadGEM2-CC', + 'ensemble': 'r1i1p1', + 'exp': 'rcp85', + 'mip': 'Amon', + 'project': 'CMIP5', + 'short_name': 'tas', + 'timerange': '2080/2100', + }, + { + 'dataset': 'EC-EARTH', + 'ensemble': 'r1i1p1', + 'exp': 'historical', + 'mip': 'Amon', + 'project': 'CMIP5', + 'short_name': 'tas', + 'start_year': 1990, # test legacy way of specifying timerange + 'end_year': 1999, + }, + { + 'dataset': 'AWI-ESM-1-1-LR', + 'ensemble': 'r1i1p1f1', + 'exp': 'historical', + 'grid': 'gn', + 'mip': 'Amon', + 'project': 'CMIP6', + 'short_name': 'tas', + 'timerange': '2000/2001', + 'version': 'v20200212', + }, + { + 'dataset': 'CESM2', + 'ensemble': 'r4i1p1f1', + 'exp': 'historical', + 'grid': 'gn', + 'mip': 'Amon', + 'project': 'CMIP6', + 'short_name': 'tas', + 'timerange': '2000/2001', + }, + { + 'dataset': 'RACMO22E', + 'driver': 'MOHC-HadGEM2-ES', + 'domain': 'EUR-11', + 'ensemble': 'r1i1p1', + 'exp': 'historical', + 'frequency': 'mon', + 'project': 'CORDEX', + 'short_name': 'tas', + 'timerange': '1950/1952', + 'version': 'v20160620', + }, + { + 'dataset': 'REMO2009', + 'driver': 'MPI-M-MPI-ESM-LR', + 'domain': 'EUR-11', + 'ensemble': 'r1i1p1', + 'exp': 'rcp26', + 'frequency': 'mon', + 'project': 'CORDEX-Adjust', + 'short_name': 'tasAdjust', + 'timerange': '2005/2010', + 'version': 'v20160919', + }, + { + 'dataset': 'CERES-EBAF', + 'frequency': 'mon', + 'project': 'obs4MIPs', + 'short_name': 'rsutcs', + 'version': 'v20160610', + }, + { + 'dataset': 'GPCP-V2.3', + 'project': 'obs4MIPs', + 'short_name': 'pr', + } +] def get_mock_connection(facets, results): """Create a mock pyesgf.search.SearchConnection instance.""" + class MockFileSearchContext: + def search(self, **kwargs): return results class MockConnection: + def new_context(self, *args, **kwargs): assert kwargs == facets return MockFileSearchContext() @@ -138,18 +164,22 @@ def test_mock_search(variable, mocker): if json_file in expected_results: expected_files = expected_results[json_file] else: - expected_results[json_file] = [ - { - 'checksums': file._checksums, - 'dataset': file.dataset, - 'facets': file.facets, - 'local_file': str(file.local_file(Path())), - 'name': file.name, - 'size': file.size, - 'urls': file.urls, - } - for file in files - ] + expected_results[json_file] = [{ + 'checksums': + file._checksums, + 'dataset': + file.dataset, + 'facets': + file.facets, + 'local_file': + str(file.local_file(Path())), + 'name': + file.name, + 'size': + file.size, + 'urls': + file.urls, + } for file in files] with expected_results_file.open('w', encoding='utf-8') as file: yaml.safe_dump(expected_results, file) From 3aae9ffc1745187194d4ed6696a21dfb3212bf44 Mon Sep 17 00:00:00 2001 From: Lars Buntemeyer Date: Sat, 28 Oct 2023 23:40:59 +0200 Subject: [PATCH 4/4] added integation test for CORDEX-ESD --- tests/integration/esgf/test_search_download.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/integration/esgf/test_search_download.py b/tests/integration/esgf/test_search_download.py index 5785657062..4201f99643 100644 --- a/tests/integration/esgf/test_search_download.py +++ b/tests/integration/esgf/test_search_download.py @@ -99,6 +99,17 @@ 'timerange': '2005/2010', 'version': 'v20160919', }, + { + 'dataset': 'EPISODES2018', + 'driver': 'MPI-M-MPI-ESM-LR', + 'domain': 'EUR-11', + 'ensemble': 'r1i1p1', + 'exp': 'rcp26', + 'frequency': 'mon', + 'project': 'CORDEX-ESD', + 'short_name': 'tas', + 'version': 'v20180409', + }, { 'dataset': 'CERES-EBAF', 'frequency': 'mon',