Merge pull request #125 from agarny/simulation

Simulation-related changes required for M4.5.7

app/config.py

@@ -31,6 +31,7 @@ class Config(object):
     BIOLUCIDA_PASSWORD = os.environ.get("BIOLUCIDA_PASSWORD", "local-password")
     KNOWLEDGEBASE_KEY = os.environ.get("KNOWLEDGEBASE_KEY", "secret-key")
     DEPLOY_ENV = os.environ.get("DEPLOY_ENV", "development")
+    SPARC_API_DEBUGGING = os.environ.get("SPARC_API_DEBUGGING", "TRUE")
     SPARC_APP_HOST = os.environ.get("SPARC_APP_HOST", "https://sparc-app.herokuapp.com")
     SCI_CRUNCH_HOST = os.environ.get("SCICRUNCH_HOST", "https://scicrunch.org/api/1/elastic/SPARC_PortalDatasets_pr")
     MAPSTATE_TABLENAME = os.environ.get("MAPSTATE_TABLENAME", "mapstates")

app/main.py

@@ -37,7 +37,8 @@
     reform_related_terms
 from app.serializer import ContactRequestSchema
 from app.utilities import img_to_base64_str
-from app.osparc import run_simulation
+from app.osparc import start_simulation as do_start_simulation
+from app.osparc import check_simulation as do_check_simulation
 from app.biolucida_process_results import process_results as process_biolucida_results
 
 logging.basicConfig()
@@ -171,14 +172,14 @@ def get_metrics():
     if contentful:
         cf_response = get_funded_projects_count(contentful)
         usage_metrics['funded_projects_count'] = cf_response
-        
+
     ps_response = get_download_count()
     usage_metrics['1year_download_count'] = ps_response
 
     if not metrics_scheduler.running:
         logging.info('Starting scheduler for metrics acquisition')
         metrics_scheduler.start()
-    
+
 
 # Gets oSPARC viewers before the first request after startup and then once a day.
 viewers_scheduler.add_job(func=get_osparc_file_viewers, trigger="interval", days=1)
@@ -643,17 +644,20 @@ def inject_template_data(resp):
         )
     except ClientError:
         # If the file is not under folder 'files', check under folder 'packages'
-        logging.warning(
-            "Required file template.json was not found under /files folder, trying under /packages..."
-        )
+        debugging = Config.SPARC_API_DEBUGGING == "TRUE"
+        if debugging:
+            logging.warning(
+                "Required file template.json was not found under /files folder, trying under /packages..."
+            )
         try:
             response = s3.get_object(
                 Bucket="pennsieve-prod-discover-publish-use1",
                 Key="{}/{}/packages/template.json".format(id_, version),
                 RequestPayer="requester",
             )
         except ClientError as e:
-            logging.error(e)
+            if debugging:
+                logging.error(e)
             return
 
     template = response["Body"].read()
@@ -899,7 +903,7 @@ def create_wrike_task():
         hed = { 'Authorization': 'Bearer ' + Config.WRIKE_TOKEN }
         ## Updated Wrike Space info based off type of task. We default to drc_feedback folder if type is not present.
         url = 'https://www.wrike.com/api/v4/folders/' + Config.DRC_FEEDBACK_FOLDER_ID + '/tasks'
-        followers = [Config.CCB_HEAD_WRIKE_ID, Config.DAT_CORE_TECH_LEAD_WRIKE_ID, Config.MAP_CORE_TECH_LEAD_WRIKE_ID, Config.K_CORE_TECH_LEAD_WRIKE_ID, Config.SIM_CORE_TECH_LEAD_WRIKE_ID, Config.MODERATOR_WRIKE_ID]     
+        followers = [Config.CCB_HEAD_WRIKE_ID, Config.DAT_CORE_TECH_LEAD_WRIKE_ID, Config.MAP_CORE_TECH_LEAD_WRIKE_ID, Config.K_CORE_TECH_LEAD_WRIKE_ID, Config.SIM_CORE_TECH_LEAD_WRIKE_ID, Config.MODERATOR_WRIKE_ID]
         responsibles = [Config.CCB_HEAD_WRIKE_ID, Config.DAT_CORE_TECH_LEAD_WRIKE_ID, Config.MAP_CORE_TECH_LEAD_WRIKE_ID, Config.K_CORE_TECH_LEAD_WRIKE_ID, Config.SIM_CORE_TECH_LEAD_WRIKE_ID, Config.MODERATOR_WRIKE_ID]
         customStatus = Config.DRC_WRIKE_CUSTOM_STATUS_ID
         taskType = ""
@@ -1134,10 +1138,10 @@ def get_available_uberonids():
     return jsonify(result)
 
 
-# Get list of terms a level up/down from 
+# Get list of terms a level up/down from
 @app.route("/get-related-terms/<query>")
 def get_related_terms(query):
-    
+
     payload = {
         'direction': request.args.get('direction', default='OUTGOING'),
         'relationshipType': request.args.get('relationshipType', default='BFO:0000050'),
@@ -1174,14 +1178,24 @@ def simulation_ui_file(identifier):
         abort(404, description="no simulation UI file could be found")
 
 
-@app.route("/simulation", methods=["POST"])
-def simulation():
+@app.route("/start_simulation", methods=["POST"])
+def start_simulation():
+    data = request.get_json()
+
+    if data and "solver" in data and "name" in data["solver"] and "version" in data["solver"]:
+        return json.dumps(do_start_simulation(data))
+    else:
+        abort(400, description="Missing solver name and/or solver version")
+
+
+@app.route("/check_simulation", methods=["POST"])
+def check_simulation():
     data = request.get_json()
 
-    if data and "model_url" in data and "json_config" in data:
-        return json.dumps(run_simulation(data["model_url"], data["json_config"]))
+    if data and "job_id" in data and "solver" in data and "name" in data["solver"] and "version" in data["solver"]:
+        return json.dumps(do_check_simulation(data))
     else:
-        abort(400, description="Missing model URL and/or JSON configuration")
+        abort(400, description="Missing solver name, solver version and/or job id")
 
 
 @app.route("/pmr_latest_exposure", methods=["POST"])

app/osparc.py

@@ -1,71 +1,159 @@
-from app.config import Config
 import json
 import osparc
 import tempfile
+
+from app.config import Config
+from flask import abort
+from osparc.rest import ApiException
 from time import sleep
 
 
+OPENCOR_SOLVER = "simcore/services/comp/opencor"
+DATASET_4_SOLVER = "simcore/services/comp/rabbit-ss-0d-cardiac-model"
+DATASET_17_SOLVER = "simcore/services/comp/human-gb-0d-cardiac-model"
+DATASET_78_SOLVER = "simcore/services/comp/kember-cardiac-model"
+
+
 class SimulationException(Exception):
     pass
 
 
-def run_simulation(model_url, json_config):
-    with tempfile.NamedTemporaryFile(mode="w+") as temp_config_file:
-        json.dump(json_config, temp_config_file)
+def start_simulation(data):
+    # Determine the type of simulation.
 
-        temp_config_file.seek(0)
+    solver_name = data["solver"]["name"]
 
-        try:
-            api_client = osparc.ApiClient(osparc.Configuration(
-                host=Config.OSPARC_API_URL,
-                username=Config.OSPARC_API_KEY,
-                password=Config.OSPARC_API_SECRET
-            ))
+    if solver_name == OPENCOR_SOLVER:
+        if not "opencor" in data:
+            abort(400, description="Missing OpenCOR settings")
+    else:
+        if "osparc" in data:
+            if ((solver_name != DATASET_4_SOLVER)
+                and (solver_name != DATASET_17_SOLVER)
+                    and (solver_name != DATASET_78_SOLVER)):
+                abort(400, description="Unknown oSPARC solver")
+        else:
+            abort(400, description="Missing oSPARC settings")
 
-            # Upload the configuration file.
+    # Start the simulation.
 
-            files_api = osparc.FilesApi(api_client)
+    try:
+        api_client = osparc.ApiClient(osparc.Configuration(
+            host=Config.OSPARC_API_URL,
+            username=Config.OSPARC_API_KEY,
+            password=Config.OSPARC_API_SECRET
+        ))
 
-            try:
-                config_file = files_api.upload_file(temp_config_file.name)
-            except:
-                raise SimulationException(
-                    "the simulation configuration file could not be uploaded")
+        # Upload the configuration file, in the case of an OpenCOR simulation or
+        # in the case of an oSPARC simulation input file.
 
-            # Create the simulation.
+        has_solver_input = "input" in data["solver"]
 
-            solvers_api = osparc.SolversApi(api_client)
+        if solver_name == OPENCOR_SOLVER:
+            temp_config_file = tempfile.NamedTemporaryFile(mode="w+")
 
-            solver = solvers_api.get_solver_release(
-                "simcore/services/comp/opencor", "1.0.3")
+            json.dump(data["opencor"]["json_config"], temp_config_file)
+
+            temp_config_file.seek(0)
+
+            try:
+                files_api = osparc.FilesApi(api_client)
 
-            job = solvers_api.create_job(
-                solver.id,
-                solver.version,
-                osparc.JobInputs({
-                    "model_url": model_url,
-                    "config_file": config_file
-                })
-            )
+                config_file = files_api.upload_file(temp_config_file.name)
+            except ApiException as e:
+                raise SimulationException(
+                    f"the simulation configuration file could not be uploaded ({e})")
 
-            # Start the simulation job.
+            temp_config_file.close()
+        elif has_solver_input:
+            temp_input_file = tempfile.NamedTemporaryFile(mode="w+")
 
-            status = solvers_api.start_job(solver.id, solver.version, job.id)
+            temp_input_file.write(data["solver"]["input"]["value"])
+            temp_input_file.seek(0)
 
-            if status.state != "PUBLISHED":
-                raise SimulationException("the simulation job could not be submitted")
+            try:
+                files_api = osparc.FilesApi(api_client)
 
-            # Wait for the simulation job to be complete (or to fail).
+                input_file = files_api.upload_file(temp_input_file.name)
+            except ApiException as e:
+                raise SimulationException(
+                    f"the solver input file could not be uploaded ({e})")
 
-            while True:
-                status = solvers_api.inspect_job(solver.id, solver.version, job.id)
+            temp_input_file.close()
 
-                if status.progress == 100:
-                    break
+        # Create the simulation job with the job inputs that matches our
+        # simulation type.
 
-                sleep(1)
+        solvers_api = osparc.SolversApi(api_client)
 
-            status = solvers_api.inspect_job(solver.id, solver.version, job.id)
+        try:
+            solver = solvers_api.get_solver_release(
+                solver_name, data["solver"]["version"])
+        except ApiException as e:
+            raise SimulationException(
+                f"the requested solver could not be retrieved ({e})")
+
+        if solver_name == OPENCOR_SOLVER:
+            job_inputs = {
+                "model_url": data["opencor"]["model_url"],
+                "config_file": config_file
+            }
+        else:
+            if has_solver_input:
+                data["osparc"]["job_inputs"][data["solver"]["input"]["name"]] = input_file
+
+            job_inputs = data["osparc"]["job_inputs"]
+
+        job = solvers_api.create_job(
+            solver.id,
+            solver.version,
+            osparc.JobInputs(job_inputs)
+        )
+
+        # Start the simulation job.
+
+        status = solvers_api.start_job(solver.id, solver.version, job.id)
+
+        if status.state != "PUBLISHED":
+            raise SimulationException(
+                "the simulation job could not be submitted")
+
+        res = {
+            "status": "ok",
+            "data": {
+                "job_id": job.id,
+                "solver": {
+                    "name": solver.id,
+                    "version": solver.version
+                }
+            }
+        }
+    except SimulationException as e:
+        res = {
+            "status": "nok",
+            "description": e.args[0] if len(e.args) > 0 else "unknown"
+        }
+
+    return res
+
+
+def check_simulation(data):
+    try:
+        # Check whether the simulation has completed (or failed).
+
+        api_client = osparc.ApiClient(osparc.Configuration(
+            host=Config.OSPARC_API_URL,
+            username=Config.OSPARC_API_KEY,
+            password=Config.OSPARC_API_SECRET
+        ))
+        solvers_api = osparc.SolversApi(api_client)
+        job_id = data["job_id"]
+        solver_name = data["solver"]["name"]
+        solver_version = data["solver"]["version"]
+        status = solvers_api.inspect_job(solver_name, solver_version, job_id)
+
+        if status.progress == 100:
+            # The simulation has completed, but was it successful?
 
             if status.state != "SUCCESS":
                 raise SimulationException("the simulation failed")
@@ -74,33 +162,44 @@ def run_simulation(model_url, json_config):
 
             try:
                 outputs = solvers_api.get_job_outputs(
-                    solver.id, solver.version, job.id)
-            except:
+                    solver_name, solver_version, job_id)
+            except ApiException as e:
                 raise SimulationException(
-                    "the simulation job outputs could not be retrieved")
+                    f"the simulation job outputs could not be retrieved ({e})")
 
             # Download the simulation results.
 
             try:
+                files_api = osparc.FilesApi(api_client)
+
                 results_filename = files_api.download_file(
-                    outputs.results["output_1"].id)
-            except:
-                raise SimulationException("the simulation results could not be retrieved")
+                    outputs.results[list(outputs.results.keys())[0]].id)
+            except ApiException as e:
+                raise SimulationException(
+                    f"the simulation results could not be retrieved ({e})")
 
             results_file = open(results_filename, "r")
 
             res = {
                 "status": "ok",
-                "results": json.load(results_file)
             }
 
+            if solver_name == OPENCOR_SOLVER:
+                res["results"] = json.load(results_file)
+            else:
+                res["results"] = results_file.read()
+
             results_file.close()
-        except SimulationException as e:
+        else:
+            # The simulation is not complete yet.
+
             res = {
-                "status": "nok",
-                "description": e.args[0] if len(e.args) > 0 else "unknown"
+                "status": "ok"
             }
+    except SimulationException as e:
+        res = {
+            "status": "nok",
+            "description": e.args[0] if len(e.args) > 0 else "unknown"
+        }
 
-        temp_config_file.close()
-
-        return res
+    return res

tests/test_api.py

@@ -189,12 +189,12 @@ def test_onto_term_lookup(client):
     assert json_data['label'] == 'Human'
 
 
-def test_non_existing_simualtion_ui_file(client):
+def test_non_existing_simulation_ui_file(client):
     r = client.get('/simulation_ui_file/137')
     assert r.status_code == 404
 
 
-def test_simualtion_ui_file(client):
+def test_simulation_ui_file(client):
     r = client.get('/simulation_ui_file/135')
     assert r.status_code == 200
-    assert r.get_json()['input'][1]['enabled'] == '(sm == 1) || (sm == 2)'
+    assert r.get_json()['input'][1]['visible'] == 'sm == 1'