added integration test for the new adaptor

In does really deploy the HTTP API instead of using a test client.

cdsobs/api_rest/endpoints.py

@@ -1,6 +1,7 @@
+import os
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Iterator
 
 import sqlalchemy.orm
 from fastapi import APIRouter, Depends, HTTPException
@@ -29,8 +30,11 @@ class HttpAPISession:
     catalogue_session: sqlalchemy.orm.Session
 
 
-def session_gen() -> HttpAPISession:
-    cdsobs_config_yml = Path.home().joinpath(".cdsobs/cdsobs_config.yml")
+def session_gen() -> Iterator[HttpAPISession]:
+    if "CDSOBS_CONFIG" in os.environ:
+        cdsobs_config_yml = Path(os.environ["CDSOBS_CONFIG"])
+    else:
+        cdsobs_config_yml = Path.home().joinpath(".cdsobs/cdsobs_config.yml")
     if not Path(cdsobs_config_yml).exists():
         raise ConfigNotFound()
     cdsobs_config = validate_config(cdsobs_config_yml)

cdsobs/observation_catalogue/repositories/base_repository.py

@@ -20,7 +20,7 @@ def get(self, record_id: Any):
             sa.select(self.model).filter(self.model.id == record_id).limit(1)
         ).first()
 
-    def get_all(self, skip: int = 0, limit: int = 100) -> Sequence[Base]:
+    def get_all(self, skip: int = 0, limit: int = 100) -> Sequence[Any]:
         return self.session.scalars(
             sa.select(self.model).offset(skip).limit(limit)
         ).all()

environment.yml

@@ -46,4 +46,4 @@ dependencies:
 - python=3.10
 - pip:
   - sqlalchemy_json
-  - git+https://github.com/ecmwf-projects/cads-adaptors
+  - git+https://github.com/ecmwf-projects/cads-adaptors@obs_adaptor

tests/conftest.py

@@ -1,11 +1,15 @@
 import copy
 import importlib
 import pickle
+import time
 from dataclasses import dataclass
 from datetime import datetime
+from multiprocessing import Process
 from pathlib import Path
 
 import pytest
+import requests
+import uvicorn
 
 from cdsobs.api import run_ingestion_pipeline
 from cdsobs.config import CDSObsConfig
@@ -161,3 +165,37 @@ def mock_retrieve_args():
         time_coverage=(datetime(1998, 1, 1), datetime(1998, 2, 1)),
     )
     return RetrieveArgs(dataset=DS_TEST_NAME, params=params)
+
+
+def run_server():
+    uvicorn.run(
+        "cdsobs.api_rest.app:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=False,
+        workers=1,
+    )
+
+
+def wait_until_api_is_ready(timeout: int, period=0.25):
+    mustend = time.time() + timeout
+    while time.time() < mustend:
+        try:
+            res = requests.get("http://localhost:8000/cdm/lite_variables")
+            res.raise_for_status()
+            return True
+        except requests.exceptions.HTTPError:
+            time.sleep(period)
+        except requests.exceptions.ConnectionError:
+            time.sleep(period)
+    return False
+
+
+@pytest.fixture()
+def test_api_server():
+    proc = Process(target=run_server, args=(), daemon=True)
+    proc.start()
+    wait_until_api_is_ready(5)
+    yield proc
+    # Cleanup after test
+    proc.terminate()

tests/retrieve/test_adaptor.py

@@ -1,19 +1,18 @@
 from pathlib import Path
 
-import pytest
 import xarray
 
 
-@pytest.mark.skip("Depends on cads_adaptors")
-def test_adaptor(test_config, test_repository, tmp_path):
+def test_adaptor(test_config, test_repository, tmp_path, test_api_server):
+    """Full test with a local instance of the HTTP API."""
     from cads_adaptors import ObservationsAdaptor
 
     test_request = {
-        "observation_type": ["vertical_profile"],
+        "observation_type": ["total_column"],
         "format": "netCDF",
-        "variable": ["air_temperature"],
-        "year": ["1969"],
-        "month": ["01"],
+        "variable": ["total_ozone_column"],
+        "year": ["2011"],
+        "month": ["02"],
         "day": [
             "01",
             "02",
@@ -26,11 +25,7 @@ def test_adaptor(test_config, test_repository, tmp_path):
     test_adaptor_config = {
         "entry_point": "cads_adaptors:ObservationsAdaptor",
         "collection_id": "insitu-observations-woudc-ozone-total-column-and-profiles",
-        "catalogue_url": test_config.catalogue_db.get_url(),
-        "storage_url": "http://"
-        + test_config.s3config.host
-        + ":"
-        + str(test_config.s3config.port),
+        "obs_api_url": "http://localhost:8000",
         "mapping": {
             "remap": {
                 "observation_type": {