Limit the use of EverestConfig in function signatures in everest csv …

…exporting

src/everest/__init__.py

@@ -15,21 +15,19 @@
 from everest import detached, docs, jobs, templates, util
 from everest.bin.utils import export_to_csv, export_with_progress
 from everest.config_keys import ConfigKeys
-from everest.export import MetaDataColumnNames, export, filter_data, validate_export
+from everest.export import MetaDataColumnNames, filter_data
 
 __author__ = "Equinor ASA and TNO"
 __all__ = [
     "ConfigKeys",
     "MetaDataColumnNames",
     "detached",
     "docs",
-    "export",
     "export_to_csv",
     "export_with_progress",
     "filter_data",
     "jobs",
     "load",
     "templates",
     "util",
-    "validate_export",
 ]

src/everest/bin/everexport_script.py

@@ -4,9 +4,10 @@
 import logging
 from functools import partial
 
-from everest import export_to_csv, validate_export
+from everest import export_to_csv, export_with_progress
 from everest.config import EverestConfig
 from everest.config.export_config import ExportConfig
+from everest.export import check_for_errors
 from everest.strings import EVEREST
 
 
@@ -30,10 +31,19 @@ def everexport_entry(args=None):
         batch_list = [int(item) for item in options.batches]
         config.export.batches = batch_list
 
-    err_msgs, export_ecl = validate_export(config)
+    err_msgs, export_ecl = check_for_errors(
+        config=config.export,
+        optimization_output_path=config.optimization_output_dir,
+        storage_path=config.storage_dir,
+        data_file_path=config.model.data_file,
+    )
     for msg in err_msgs:
         logger.warning(msg)
-    export_to_csv(config, export_ecl=export_ecl)
+
+    export_to_csv(
+        data_frame=export_with_progress(config, export_ecl),
+        export_path=config.export_path,
+    )
 
 
 def _build_args_parser():

src/everest/bin/everload_script.py

@@ -11,9 +11,9 @@
 from ert.config import ErtConfig
 from ert.storage import open_storage
 from everest import MetaDataColumnNames as MDCN
-from everest import export
 from everest.config import EverestConfig
 from everest.config.export_config import ExportConfig
+from everest.export import export_data
 from everest.simulator.everest_to_ert import _everest_to_ert_config_dict
 from everest.util import version_info
 
@@ -162,7 +162,12 @@ def reload_data(ever_config: EverestConfig, backup_path=None):
     ert_config = ErtConfig.with_plugins().from_dict(config_dict=ert_config_dict)
 
     # load information about batches from previous run
-    df = export(ever_config, export_ecl=False)
+    df = export_data(
+        export_config=ever_config.export,
+        output_dir=ever_config.output_dir,
+        data_file=ever_config.model.data_file if ever_config.model else None,
+        export_ecl=False,
+    )
     groups = df.groupby(by=MDCN.BATCH)
 
     # backup or delete the previous internal storage

src/everest/bin/utils.py

@@ -8,6 +8,7 @@
 
 import colorama
 from colorama import Fore
+from pandas import DataFrame
 
 from ert.resources import all_shell_script_fm_steps
 from ert.simulator.batch_simulator_context import Status
@@ -20,7 +21,7 @@
     get_opt_status,
     start_monitor,
 )
-from everest.export import export
+from everest.export import export_data
 from everest.simulator import JOB_FAILURE, JOB_RUNNING, JOB_SUCCESS
 from everest.strings import EVEREST
 
@@ -30,29 +31,28 @@
     ProgressBar = None  # type: ignore
 
 
-def export_with_progress(config, export_ecl=True):
+def export_with_progress(config: EverestConfig, export_ecl=True):
     logging.getLogger(EVEREST).info("Exporting results to csv ...")
     if ProgressBar is not None:
         widgets = [Percentage(), "  ", Bar(), "  ", Timer(), "  ", AdaptiveETA()]
         with ProgressBar(max_value=1, widgets=widgets) as bar:
-            export_data = export(
-                config=config, export_ecl=export_ecl, progress_callback=bar.update
+            return export_data(
+                export_config=config.export,
+                output_dir=config.output_dir,
+                data_file=config.model.data_file if config.model else None,
+                export_ecl=export_ecl,
+                progress_callback=bar.update,
             )
-    else:
-        export_data = export(config=config, export_ecl=export_ecl)
-
-    return export_data
-
-
-def export_to_csv(config: EverestConfig, data_frame=None, export_ecl=True):
-    if data_frame is None:
-        data_frame = export_with_progress(config, export_ecl)
+    return export_data(
+        export_config=config.export,
+        output_dir=config.output_dir,
+        data_file=config.model.data_file if config.model else None,
+        export_ecl=export_ecl,
+    )
 
-    export_path = config.export_path
-    output_folder = os.path.dirname(export_path)
-    if not os.path.exists(output_folder):
-        os.makedirs(output_folder)
 
+def export_to_csv(data_frame: DataFrame, export_path: str) -> None:
+    os.makedirs(os.path.dirname(export_path), exist_ok=True)
     data_frame.to_csv(export_path, sep=";", index=False)
     logging.getLogger(EVEREST).info("Data exported to {}".format(export_path))
 

src/everest/detached/__init__.py

@@ -86,7 +86,8 @@ def start_server(config: EverestConfig, ert_config: ErtConfig, storage):
         )
 
     try:
-        _save_running_config(config)
+        save_config_path = os.path.join(config.output_dir, config.config_file)
+        config.dump(save_config_path)
     except (OSError, LookupError) as e:
         logging.getLogger(EVEREST).error(
             "Failed to save optimization config: {}".format(e)
@@ -121,13 +122,6 @@ def start_server(config: EverestConfig, ert_config: ErtConfig, storage):
     return _context
 
 
-def _save_running_config(config: EverestConfig):
-    assert config.output_dir is not None
-    assert config.config_file is not None
-    save_config_path = os.path.join(config.output_dir, config.config_file)
-    config.dump(save_config_path)
-
-
 def context_stop_and_wait():
     global _context  # noqa: PLW0602
     if _context:

src/everest/detached/jobs/everserver.py

@@ -22,9 +22,10 @@
 from ert.config import QueueSystem
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.run_models.everest_run_model import EverestRunModel
-from everest import export_to_csv, validate_export
+from everest import export_to_csv, export_with_progress
 from everest.config import EverestConfig
 from everest.detached import ServerStatus, get_opt_status, update_everserver_status
+from everest.export import check_for_errors
 from everest.simulator import JOB_FAILURE
 from everest.strings import (
     EVEREST,
@@ -304,10 +305,23 @@ def main():
     try:
         # Exporting data
         update_everserver_status(config, ServerStatus.exporting_to_csv)
-        err_msgs, export_ecl = validate_export(config)
-        for msg in err_msgs:
-            logging.getLogger(EVEREST).warning(msg)
-        export_to_csv(config, export_ecl=export_ecl)
+
+        if config.export is not None:
+            err_msgs, export_ecl = check_for_errors(
+                config=config.export,
+                optimization_output_path=config.optimization_output_dir,
+                storage_path=config.storage_dir,
+                data_file_path=config.model.data_file,
+            )
+            for msg in err_msgs:
+                logging.getLogger(EVEREST).warning(msg)
+        else:
+            export_ecl = True
+
+        export_to_csv(
+            data_frame=export_with_progress(config, export_ecl),
+            export_path=config.export_path,
+        )
     except:
         update_everserver_status(
             config, ServerStatus.failed, message=traceback.format_exc()