Merge pull request #6 from ecmwf-projects/forms-fix

Forms fix

cdsobs/forms_jsons.py

@@ -86,17 +86,19 @@ def get_constraints_json(session, output_path: Path, dataset) -> Path:
     # This is probably slow, can it be improved?
     catalogue_entries = get_catalogue_entries_stream(session, dataset)
     merged_constraints = merged_constraints_table(catalogue_entries)
+    # Remove the stations here to avoid using too much memory, set to true the constraints
+    # if there is data for any of the stations
+    merged_constraints = merged_constraints.drop("stations", axis=1)
+    merged_constraints = merged_constraints.groupby(["time", "source"]).any()
     logger.info("Computing flat constraints.")
     # This returns a bool series where True are available combinations
-    flat_constraints = merged_constraints.set_index(
-        ["stations", "time", "source"]
-    ).stack()
+    flat_constraints = merged_constraints.stack()
     # Filter out the False rows
     flat_constraints = flat_constraints.loc[flat_constraints]
     # Turn into a dataframe and remove the bools column (named 0 by default)
     flat_constraints = (
         flat_constraints.reset_index()
-        .rename(dict(level_3="variable"), axis=1)
+        .rename(dict(level_2="variable"), axis=1)
         .drop(0, axis=1)
         .rename(dict(source="dataset_source"), axis=1)
     )
@@ -108,7 +110,7 @@ def get_constraints_json(session, output_path: Path, dataset) -> Path:
     flat_constraints = flat_constraints.drop("time", axis=1)
     logger.info("Computing compressed constraints")
     compressed_constraints = iterative_ordering(
-        flat_constraints, flat_constraints.columns.drop("stations")
+        flat_constraints, flat_constraints.columns
     )
     constraints_path = Path(output_path, "constraints.json")
     logger.info(f"Writing {constraints_path}")