diff --git a/app/config.py b/app/config.py index bc8066c..499a468 100644 --- a/app/config.py +++ b/app/config.py @@ -31,6 +31,7 @@ class Config(object): BIOLUCIDA_PASSWORD = os.environ.get("BIOLUCIDA_PASSWORD", "local-password") KNOWLEDGEBASE_KEY = os.environ.get("KNOWLEDGEBASE_KEY", "secret-key") DEPLOY_ENV = os.environ.get("DEPLOY_ENV", "development") + SPARC_API_DEBUGGING = os.environ.get("SPARC_API_DEBUGGING", "TRUE") SPARC_APP_HOST = os.environ.get("SPARC_APP_HOST", "https://sparc-app.herokuapp.com") SCI_CRUNCH_HOST = os.environ.get("SCICRUNCH_HOST", "https://scicrunch.org/api/1/elastic/SPARC_PortalDatasets_pr") MAPSTATE_TABLENAME = os.environ.get("MAPSTATE_TABLENAME", "mapstates") diff --git a/app/main.py b/app/main.py index 97405a9..b81cfa3 100644 --- a/app/main.py +++ b/app/main.py @@ -37,7 +37,8 @@ reform_related_terms from app.serializer import ContactRequestSchema from app.utilities import img_to_base64_str -from app.osparc import run_simulation +from app.osparc import start_simulation as do_start_simulation +from app.osparc import check_simulation as do_check_simulation from app.biolucida_process_results import process_results as process_biolucida_results logging.basicConfig() @@ -171,14 +172,14 @@ def get_metrics(): if contentful: cf_response = get_funded_projects_count(contentful) usage_metrics['funded_projects_count'] = cf_response - + ps_response = get_download_count() usage_metrics['1year_download_count'] = ps_response if not metrics_scheduler.running: logging.info('Starting scheduler for metrics acquisition') metrics_scheduler.start() - + # Gets oSPARC viewers before the first request after startup and then once a day. viewers_scheduler.add_job(func=get_osparc_file_viewers, trigger="interval", days=1) @@ -643,9 +644,11 @@ def inject_template_data(resp): ) except ClientError: # If the file is not under folder 'files', check under folder 'packages' - logging.warning( - "Required file template.json was not found under /files folder, trying under /packages..." - ) + debugging = Config.SPARC_API_DEBUGGING == "TRUE" + if debugging: + logging.warning( + "Required file template.json was not found under /files folder, trying under /packages..." + ) try: response = s3.get_object( Bucket="pennsieve-prod-discover-publish-use1", @@ -653,7 +656,8 @@ def inject_template_data(resp): RequestPayer="requester", ) except ClientError as e: - logging.error(e) + if debugging: + logging.error(e) return template = response["Body"].read() @@ -899,7 +903,7 @@ def create_wrike_task(): hed = { 'Authorization': 'Bearer ' + Config.WRIKE_TOKEN } ## Updated Wrike Space info based off type of task. We default to drc_feedback folder if type is not present. url = 'https://www.wrike.com/api/v4/folders/' + Config.DRC_FEEDBACK_FOLDER_ID + '/tasks' - followers = [Config.CCB_HEAD_WRIKE_ID, Config.DAT_CORE_TECH_LEAD_WRIKE_ID, Config.MAP_CORE_TECH_LEAD_WRIKE_ID, Config.K_CORE_TECH_LEAD_WRIKE_ID, Config.SIM_CORE_TECH_LEAD_WRIKE_ID, Config.MODERATOR_WRIKE_ID] + followers = [Config.CCB_HEAD_WRIKE_ID, Config.DAT_CORE_TECH_LEAD_WRIKE_ID, Config.MAP_CORE_TECH_LEAD_WRIKE_ID, Config.K_CORE_TECH_LEAD_WRIKE_ID, Config.SIM_CORE_TECH_LEAD_WRIKE_ID, Config.MODERATOR_WRIKE_ID] responsibles = [Config.CCB_HEAD_WRIKE_ID, Config.DAT_CORE_TECH_LEAD_WRIKE_ID, Config.MAP_CORE_TECH_LEAD_WRIKE_ID, Config.K_CORE_TECH_LEAD_WRIKE_ID, Config.SIM_CORE_TECH_LEAD_WRIKE_ID, Config.MODERATOR_WRIKE_ID] customStatus = Config.DRC_WRIKE_CUSTOM_STATUS_ID taskType = "" @@ -1134,10 +1138,10 @@ def get_available_uberonids(): return jsonify(result) -# Get list of terms a level up/down from +# Get list of terms a level up/down from @app.route("/get-related-terms/") def get_related_terms(query): - + payload = { 'direction': request.args.get('direction', default='OUTGOING'), 'relationshipType': request.args.get('relationshipType', default='BFO:0000050'), @@ -1174,14 +1178,24 @@ def simulation_ui_file(identifier): abort(404, description="no simulation UI file could be found") -@app.route("/simulation", methods=["POST"]) -def simulation(): +@app.route("/start_simulation", methods=["POST"]) +def start_simulation(): + data = request.get_json() + + if data and "solver" in data and "name" in data["solver"] and "version" in data["solver"]: + return json.dumps(do_start_simulation(data)) + else: + abort(400, description="Missing solver name and/or solver version") + + +@app.route("/check_simulation", methods=["POST"]) +def check_simulation(): data = request.get_json() - if data and "model_url" in data and "json_config" in data: - return json.dumps(run_simulation(data["model_url"], data["json_config"])) + if data and "job_id" in data and "solver" in data and "name" in data["solver"] and "version" in data["solver"]: + return json.dumps(do_check_simulation(data)) else: - abort(400, description="Missing model URL and/or JSON configuration") + abort(400, description="Missing solver name, solver version and/or job id") @app.route("/pmr_latest_exposure", methods=["POST"]) diff --git a/app/osparc.py b/app/osparc.py index 1052f23..4de4f61 100644 --- a/app/osparc.py +++ b/app/osparc.py @@ -1,71 +1,159 @@ -from app.config import Config import json import osparc import tempfile + +from app.config import Config +from flask import abort +from osparc.rest import ApiException from time import sleep +OPENCOR_SOLVER = "simcore/services/comp/opencor" +DATASET_4_SOLVER = "simcore/services/comp/rabbit-ss-0d-cardiac-model" +DATASET_17_SOLVER = "simcore/services/comp/human-gb-0d-cardiac-model" +DATASET_78_SOLVER = "simcore/services/comp/kember-cardiac-model" + + class SimulationException(Exception): pass -def run_simulation(model_url, json_config): - with tempfile.NamedTemporaryFile(mode="w+") as temp_config_file: - json.dump(json_config, temp_config_file) +def start_simulation(data): + # Determine the type of simulation. - temp_config_file.seek(0) + solver_name = data["solver"]["name"] - try: - api_client = osparc.ApiClient(osparc.Configuration( - host=Config.OSPARC_API_URL, - username=Config.OSPARC_API_KEY, - password=Config.OSPARC_API_SECRET - )) + if solver_name == OPENCOR_SOLVER: + if not "opencor" in data: + abort(400, description="Missing OpenCOR settings") + else: + if "osparc" in data: + if ((solver_name != DATASET_4_SOLVER) + and (solver_name != DATASET_17_SOLVER) + and (solver_name != DATASET_78_SOLVER)): + abort(400, description="Unknown oSPARC solver") + else: + abort(400, description="Missing oSPARC settings") - # Upload the configuration file. + # Start the simulation. - files_api = osparc.FilesApi(api_client) + try: + api_client = osparc.ApiClient(osparc.Configuration( + host=Config.OSPARC_API_URL, + username=Config.OSPARC_API_KEY, + password=Config.OSPARC_API_SECRET + )) - try: - config_file = files_api.upload_file(temp_config_file.name) - except: - raise SimulationException( - "the simulation configuration file could not be uploaded") + # Upload the configuration file, in the case of an OpenCOR simulation or + # in the case of an oSPARC simulation input file. - # Create the simulation. + has_solver_input = "input" in data["solver"] - solvers_api = osparc.SolversApi(api_client) + if solver_name == OPENCOR_SOLVER: + temp_config_file = tempfile.NamedTemporaryFile(mode="w+") - solver = solvers_api.get_solver_release( - "simcore/services/comp/opencor", "1.0.3") + json.dump(data["opencor"]["json_config"], temp_config_file) + + temp_config_file.seek(0) + + try: + files_api = osparc.FilesApi(api_client) - job = solvers_api.create_job( - solver.id, - solver.version, - osparc.JobInputs({ - "model_url": model_url, - "config_file": config_file - }) - ) + config_file = files_api.upload_file(temp_config_file.name) + except ApiException as e: + raise SimulationException( + f"the simulation configuration file could not be uploaded ({e})") - # Start the simulation job. + temp_config_file.close() + elif has_solver_input: + temp_input_file = tempfile.NamedTemporaryFile(mode="w+") - status = solvers_api.start_job(solver.id, solver.version, job.id) + temp_input_file.write(data["solver"]["input"]["value"]) + temp_input_file.seek(0) - if status.state != "PUBLISHED": - raise SimulationException("the simulation job could not be submitted") + try: + files_api = osparc.FilesApi(api_client) - # Wait for the simulation job to be complete (or to fail). + input_file = files_api.upload_file(temp_input_file.name) + except ApiException as e: + raise SimulationException( + f"the solver input file could not be uploaded ({e})") - while True: - status = solvers_api.inspect_job(solver.id, solver.version, job.id) + temp_input_file.close() - if status.progress == 100: - break + # Create the simulation job with the job inputs that matches our + # simulation type. - sleep(1) + solvers_api = osparc.SolversApi(api_client) - status = solvers_api.inspect_job(solver.id, solver.version, job.id) + try: + solver = solvers_api.get_solver_release( + solver_name, data["solver"]["version"]) + except ApiException as e: + raise SimulationException( + f"the requested solver could not be retrieved ({e})") + + if solver_name == OPENCOR_SOLVER: + job_inputs = { + "model_url": data["opencor"]["model_url"], + "config_file": config_file + } + else: + if has_solver_input: + data["osparc"]["job_inputs"][data["solver"]["input"]["name"]] = input_file + + job_inputs = data["osparc"]["job_inputs"] + + job = solvers_api.create_job( + solver.id, + solver.version, + osparc.JobInputs(job_inputs) + ) + + # Start the simulation job. + + status = solvers_api.start_job(solver.id, solver.version, job.id) + + if status.state != "PUBLISHED": + raise SimulationException( + "the simulation job could not be submitted") + + res = { + "status": "ok", + "data": { + "job_id": job.id, + "solver": { + "name": solver.id, + "version": solver.version + } + } + } + except SimulationException as e: + res = { + "status": "nok", + "description": e.args[0] if len(e.args) > 0 else "unknown" + } + + return res + + +def check_simulation(data): + try: + # Check whether the simulation has completed (or failed). + + api_client = osparc.ApiClient(osparc.Configuration( + host=Config.OSPARC_API_URL, + username=Config.OSPARC_API_KEY, + password=Config.OSPARC_API_SECRET + )) + solvers_api = osparc.SolversApi(api_client) + job_id = data["job_id"] + solver_name = data["solver"]["name"] + solver_version = data["solver"]["version"] + status = solvers_api.inspect_job(solver_name, solver_version, job_id) + + if status.progress == 100: + # The simulation has completed, but was it successful? if status.state != "SUCCESS": raise SimulationException("the simulation failed") @@ -74,33 +162,44 @@ def run_simulation(model_url, json_config): try: outputs = solvers_api.get_job_outputs( - solver.id, solver.version, job.id) - except: + solver_name, solver_version, job_id) + except ApiException as e: raise SimulationException( - "the simulation job outputs could not be retrieved") + f"the simulation job outputs could not be retrieved ({e})") # Download the simulation results. try: + files_api = osparc.FilesApi(api_client) + results_filename = files_api.download_file( - outputs.results["output_1"].id) - except: - raise SimulationException("the simulation results could not be retrieved") + outputs.results[list(outputs.results.keys())[0]].id) + except ApiException as e: + raise SimulationException( + f"the simulation results could not be retrieved ({e})") results_file = open(results_filename, "r") res = { "status": "ok", - "results": json.load(results_file) } + if solver_name == OPENCOR_SOLVER: + res["results"] = json.load(results_file) + else: + res["results"] = results_file.read() + results_file.close() - except SimulationException as e: + else: + # The simulation is not complete yet. + res = { - "status": "nok", - "description": e.args[0] if len(e.args) > 0 else "unknown" + "status": "ok" } + except SimulationException as e: + res = { + "status": "nok", + "description": e.args[0] if len(e.args) > 0 else "unknown" + } - temp_config_file.close() - - return res + return res diff --git a/tests/test_api.py b/tests/test_api.py index 583962b..9a07685 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -189,12 +189,12 @@ def test_onto_term_lookup(client): assert json_data['label'] == 'Human' -def test_non_existing_simualtion_ui_file(client): +def test_non_existing_simulation_ui_file(client): r = client.get('/simulation_ui_file/137') assert r.status_code == 404 -def test_simualtion_ui_file(client): +def test_simulation_ui_file(client): r = client.get('/simulation_ui_file/135') assert r.status_code == 200 - assert r.get_json()['input'][1]['enabled'] == '(sm == 1) || (sm == 2)' + assert r.get_json()['input'][1]['visible'] == 'sm == 1' diff --git a/tests/test_osparc.py b/tests/test_osparc.py index 99138b7..63423d7 100644 --- a/tests/test_osparc.py +++ b/tests/test_osparc.py @@ -9,47 +9,118 @@ def client(): return app.test_client() -def test_osparc_no_post(client): - r = client.get('/simulation') +def test_osparc_start_simulation_no_post(client): + r = client.get('/start_simulation') assert r.status_code == 405 -def test_osparc_empty_post(client): - r = client.post("/simulation", json={}) +def test_osparc_check_simulation_no_post(client): + r = client.get('/check_simulation') + assert r.status_code == 405 + + +def test_osparc_start_simulation_empty_post(client): + r = client.post("/start_simulation", json={}) + assert r.status_code == 400 + + +def test_osparc_check_simulation_empty_post(client): + r = client.post("/check_simulation", json={}) + assert r.status_code == 400 + + +def test_osparc_start_simulation_no_data(client): + data = { + } + r = client.post("/start_simulation", json=data) + assert r.status_code == 400 + + +def test_osparc_start_simulation_no_opencor_data(client): + data = { + "solver": { + "name": "simcore/services/comp/opencor", + "version": "1.0.3" + } + } + r = client.post("/start_simulation", json=data) + assert r.status_code == 400 + + +def test_osparc_start_simulation_no_osparc_data(client): + data = { + "solver": { + "name": "simcore/services/comp/rabbit-ss-0d-cardiac-model", + "version": "1.0.1" + } + } + r = client.post("/start_simulation", json=data) + assert r.status_code == 400 + + +def test_osparc_check_simulation_no_job_id_data(client): + data = { + "solver": { + "name": "simcore/services/comp/opencor", + "version": "1.0.3" + } + } + r = client.post("/check_simulation", json=data) + assert r.status_code == 400 + + +def test_osparc_check_simulation_no_solver_data(client): + data = { + "job_id": "5026ff74-dc6d-4547-9166-6ae26d04b92e" + } + r = client.post("/check_simulation", json=data) assert r.status_code == 400 -def test_osparc_no_json_config(client): +def test_osparc_check_simulation_no_solver_name_data(client): data = { - "model_url": "https://models.physiomeproject.org/e/611/HumanSAN_Fabbri_Fantini_Wilders_Severi_2017.cellml" + "job_id": "5026ff74-dc6d-4547-9166-6ae26d04b92e", + "solver": { + "version": "1.0.3" + } } - r = client.post("/simulation", json=data) + r = client.post("/check_simulation", json=data) assert r.status_code == 400 -def test_osparc_no_model_url(client): +def test_osparc_check_simulation_no_solver_version_data(client): data = { - "json_config": { - "simulation": { - "Ending point": 0.003, - "Point interval": 0.001, - }, - "output": ["Membrane/V"] + "job_id": "5026ff74-dc6d-4547-9166-6ae26d04b92e", + "solver": { + "name": "simcore/services/comp/opencor" } } - r = client.post("/simulation", json=data) + r = client.post("/check_simulation", json=data) + assert r.status_code == 400 + + +def test_osparc_check_simulation_no_data(client): + data = { + } + r = client.post("/check_simulation", json=data) assert r.status_code == 400 -def test_osparc_valid_data(client): +def test_osparc_successful_simulation(client): data = { - "model_url": "https://models.physiomeproject.org/e/611/HumanSAN_Fabbri_Fantini_Wilders_Severi_2017.cellml", - "json_config": { - "simulation": { - "Ending point": 0.003, - "Point interval": 0.001, - }, - "output": ["Membrane/V"] + "opencor": { + "model_url": "https://models.physiomeproject.org/e/611/HumanSAN_Fabbri_Fantini_Wilders_Severi_2017.cellml", + "json_config": { + "simulation": { + "Ending point": 0.003, + "Point interval": 0.001, + }, + "output": ["Membrane/V"] + } + }, + "solver": { + "name": "simcore/services/comp/opencor", + "version": "1.0.3" } } res = { @@ -59,26 +130,47 @@ def test_osparc_valid_data(client): "Membrane/V": [-47.787168, -47.74547155339473, -47.72515226841376, -47.71370033208329] } } - r = client.post("/simulation", json=data) + r = client.post("/start_simulation", json=data) assert r.status_code == 200 - assert json.dumps(json.loads(r.data), sort_keys=True) == json.dumps(res, sort_keys=True) + check_simulation_data = json.loads(r.data)["data"] + while True: + r = client.post("/check_simulation", json=check_simulation_data) + assert r.status_code == 200 + json_data = json.loads(r.data) + assert json_data["status"] == "ok" + if "results" in json_data: + assert json.dumps(json_data, sort_keys=True) == json.dumps(res, sort_keys=True) + break def test_osparc_failing_simulation(client): data = { - "model_url": "https://models.physiomeproject.org/e/611/HumanSAN_Fabbri_Fantini_Wilders_Severi_2017.cellml", - "json_config": { - "simulation": { - "Ending point": 3.0, - "Point interval": 1.0, - }, - "output": ["Membrane/V"] + "opencor": { + "model_url": "https://models.physiomeproject.org/e/611/HumanSAN_Fabbri_Fantini_Wilders_Severi_2017.cellml", + "json_config": { + "simulation": { + "Ending point": 3.0, + "Point interval": 1.0, + }, + "output": ["Membrane/V"] + } + }, + "solver": { + "name": "simcore/services/comp/opencor", + "version": "1.0.3" } } res = { "status": "nok", "description": "the simulation failed" } - r = client.post("/simulation", json=data) + r = client.post("/start_simulation", json=data) assert r.status_code == 200 - assert json.dumps(json.loads(r.data), sort_keys=True) == json.dumps(res, sort_keys=True) + check_simulation_data = json.loads(r.data)["data"] + while True: + r = client.post("/check_simulation", json=check_simulation_data) + assert r.status_code == 200 + json_data = json.loads(r.data) + if json_data["status"] == "nok": + assert json.dumps(json_data, sort_keys=True) == json.dumps(res, sort_keys=True) + break