Merge pull request #39 from ecmwf-projects/legacy-retry-options

implement legacy retry options

cads_api_client/legacy_api_client.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+import functools
 import warnings
-from typing import Any, overload
+from typing import Any, Callable, overload
 
 import cdsapi.api
 import requests
@@ -45,10 +46,21 @@ def __init__(
 
         self.url, self.key, _ = cdsapi.api.get_url_key_verify(url, key, None)
         self.session = kwargs.pop("session", requests.Session())
+        self.sleep_max = kwargs.pop("sleep_max", 120)
         self.client = api_client.ApiClient(
-            url=self.url, key=self.key, session=self.session
+            url=self.url,
+            key=self.key,
+            session=self.session,
+            sleep_max=self.sleep_max,
         )
 
+        self.timeout = kwargs.pop("timeout", 60)
+        self.retry_max = kwargs.pop("retry_max", 500)
+        self.retry_options = {
+            "maximum_tries": self.retry_max,
+            "retry_after": self.sleep_max,
+        }
+
         if kwargs:
             warnings.warn(
                 "This is a beta version."
@@ -68,15 +80,23 @@ def retrieve(self, name: str, request: dict[str, Any], target: str) -> str: ...
     @overload
     def retrieve(
         self, name: str, request: dict[str, Any], target: None = ...
-    ) -> processing.Remote: ...
+    ) -> processing.Results: ...
 
     def retrieve(
         self, name: str, request: dict[str, Any], target: str | None = None
-    ) -> str | processing.Remote:
-        if target is None:
-            collection = self.client.collection(name)
-            return collection.submit(**request)
-        return self.client.retrieve(name, target, **request)
+    ) -> str | processing.Results:
+        result = self.client.submit_and_wait_on_result(
+            collection_id=name,
+            retry_options=self.retry_options,
+            **request,
+        )
+        partial_download: Callable[..., str] = functools.partial(
+            result.download,
+            timeout=self.timeout,
+            retry_options=self.retry_options,
+        )
+        result.download = partial_download  # type: ignore[method-assign]
+        return result if target is None else result.download(target)
 
     def service(self, name, *args, **kwargs):  # type: ignore
         self.raise_not_implemented_error()

ci/environment-ci.yml

@@ -17,3 +17,5 @@ dependencies:
 - types-requests
 - pip:
   - responses
+# FIXME: install from B-Open fork until the PR gets merged
+  - git+https://github.com/bopen/cdsapi.git@COPDS-1531-beta-api

tests/integration_test_40_legacy_api_client.py

@@ -0,0 +1,21 @@
+import pathlib
+
+from cads_api_client import legacy_api_client
+
+
+def test_retrieve(tmp_path: pathlib.Path, api_root_url: str, api_key: str) -> None:
+    client = legacy_api_client.LegacyApiClient(url=api_root_url, key=api_key)
+
+    collection_id = "test-adaptor-dummy"
+    request = {"size": 1}
+
+    target = tmp_path / "test-retrieve-with-target.grib"
+    actual_target = client.retrieve(collection_id, request, str(target))
+    assert str(target) == actual_target
+    assert target.stat().st_size == 1
+
+    result = client.retrieve(collection_id, request)
+    target = tmp_path / "test-retrieve-no-target.grib"
+    actual_target = result.download(str(target))
+    assert str(target) == actual_target
+    assert target.stat().st_size == 1