Feature/68 add remote links to results table and deal with engines that seem to pass but have the wrong results

SBML/tests/test_biosimulators_local.py

@@ -2,15 +2,15 @@
 
 """
 This script tests the compatibility of different biosimulation engines with a given SBML and SED-ML file.
-It runs each engine and records the result (pass/fail) and any error messages encountered during the simulation.
-The results are then displayed in a table and saved to a markdown file.
+It runs each engine and saves a JSON file with the log.yml file (as a dict), and the d1 plot for each engine.
 """
 
 import sys
 import os
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) # to import utils
 import utils
 import argparse
+import json
 
 # Save the current working directory
 cwd = os.getcwd()
@@ -38,8 +38,15 @@
 
 print('d1 plots will be saved in:', d1_plots_local_dir)
 
-results_local = utils.run_biosimulators_locally(sedml_file_name=sedml_file_name, 
+engine_keys = list(utils.ENGINES.keys())
+
+results_local = utils.run_biosimulators_locally(engine_keys,
+                                    sedml_file_name=sedml_file_name, 
                                     sbml_file_name=sbml_file_name,
                                     d1_plots_local_dir=d1_plots_local_dir, 
                                     test_folder=test_folder)
+
+results_local_path = os.path.join(path_to_sbml_folder, 'tests', 'results_local.json')
+with open(results_local_path, 'w') as fp:
+    json.dump(results_local, fp, indent=4)
 

SBML/tests/test_biosimulators_remote.py

@@ -2,15 +2,17 @@
 
 """
 This script tests the compatibility of different biosimulation engines with a given SBML and SED-ML file.
-It runs each engine and records the result (pass/fail) and any error messages encountered during the simulation.
-The results are then displayed in a table and saved to a markdown file.
+It runs each engine and saves the d1 plots, and a JSON file containing the log.yml file (as dict) and the 
+links for accessing the remote results, for each engine.
 """
 
 import sys
 import os
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) # to import utils
 import utils
 import argparse
+import json
+
 
 # Save the current working directory
 cwd = os.getcwd()
@@ -38,8 +40,14 @@
 
 print('d1 plots will be saved in:', d1_plots_remote_dir)
 
-results_remote = utils.run_biosimulators_remotely(sedml_file_name=sedml_file_name, 
+engine_keys = list(utils.ENGINES.keys())
+
+results_remote = utils.run_biosimulators_remotely(engine_keys, sedml_file_name=sedml_file_name, 
                                     sbml_file_name=sbml_file_name,
                                     d1_plots_remote_dir=d1_plots_remote_dir, 
                                     test_folder=test_folder)
+
+results_remote_path = os.path.join(path_to_sbml_folder, 'tests', 'results_remote.json')
+with open(results_remote_path, 'w') as fp:
+    json.dump(results_remote, fp, indent=4)
 

SBML/tests/test_compatibility_biosimulators.py

@@ -42,7 +42,10 @@
 
 print('d1 plots will be saved in:', d1_plots_remote_dir, 'and', d1_plots_local_dir)
 
-utils.run_biosimulators_remotely_and_locally(sedml_file_name, 
+engine_keys = list(utils.ENGINES.keys())
+
+utils.run_biosimulators_remotely_and_locally(engine_keys,
+                                 sedml_file_name, 
                                  sbml_file_name,
                                  d1_plots_remote_dir, 
                                  d1_plots_local_dir,

SBML/tests/test_creation_combined_results_table.py

@@ -3,6 +3,7 @@
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) # to import utils
 import utils
 import argparse
+import json
 
 engines = utils.ENGINES
 
@@ -31,11 +32,18 @@
 d1_plots_local_dir = os.path.join(test_folder, args.output_dir + '_local')
 d1_plots_remote_dir = os.path.join(test_folder, args.output_dir + '_remote')
 
-results_local = {'amici': 'pass', 'brian2': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/brian2' returned non-zero exit status 1```"], 'bionetgen': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/bionetgen' returned non-zero exit status 1```"], 'boolnet': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/boolnet' returned non-zero exit status 1```"], 'cbmpy': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/cbmpy' returned non-zero exit status 1```"], 'cobrapy': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/cobrapy' returned non-zero exit status 1```"], 'copasi': 'pass', 'gillespy2': 'pass', 'ginsim': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/ginsim' returned non-zero exit status 1```"], 'libsbmlsim': 'pass', 'masspy': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/masspy' returned non-zero exit status 1```"], 'netpyne': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/netpyne' returned non-zero exit status 1```"], 'neuron': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/neuron' returned non-zero exit status 1```"], 'opencor': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/opencor' returned non-zero exit status 1```"], 'pyneuroml': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/pyneuroml' returned non-zero exit status 1```"], 'pysces': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/pysces' returned non-zero exit status 1```"], 'rbapy': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/rbapy' returned non-zero exit status 1```"], 'smoldyn': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/smoldyn' returned non-zero exit status 1```"], 'tellurium': 'pass', 'vcell': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/vcell' returned non-zero exit status 1```"], 'xpp': ['other', "```Command '-i /root/in/LEMS_NML2_Ex9_FN_missing_xmlns.omex -o /root/out' in image 'ghcr.io/biosimulators/xpp' returned non-zero exit status 1```"]}
-results_remote = {'amici': ['pass', '', ''], 'brian2': ['FAIL', "No module named 'libsbml'", 'ModuleNotFoundError'], 'bionetgen': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    Language for model `net1` is not supported.\n      - Model language `urn:sedml:language:sbml` is not supported. Models must be in BNGL format (e.g., `sed:model/@language` must match `^urn:sedml:language:bngl(\\.|$)` such as `urn:sedml:language:bngl`).', 'CombineArchiveExecutionError'], 'boolnet': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    Simulation `sim1` is invalid.\n      - Number of points (20000) must be equal to the difference between the output end (200.0) and start times (0.0).', 'CombineArchiveExecutionError'], 'cbmpy': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    UniformTimeCourseSimulation `sim1` is not supported.\n      - Simulation sim1 of type `UniformTimeCourseSimulation` is not supported. Simulation must be an instance of one of the following:\n          - SteadyStateSimulation', 'CombineArchiveExecutionError'], 'cobrapy': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    UniformTimeCourseSimulation `sim1` is not supported.\n      - Simulation sim1 of type `UniformTimeCourseSimulation` is not supported. Simulation must be an instance of one of the following:\n          - SteadyStateSimulation', 'CombineArchiveExecutionError'], 'copasi': ['pass', '', ''], 'gillespy2': ['pass', '', ''], 'ginsim': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    Simulation `sim1` is invalid.\n      - The interval between the output start and time time must be an integer multiple of the number of steps, not `0.01`:\n          Output start time: 0.0\n          Output end time: 200.0\n          Number of steps: 20000', 'CombineArchiveExecutionError'], 'libsbmlsim': ['pass', '', ''], 'masspy': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    Something went wrong reading the SBML model. Most likely the SBML model is not valid. Please check that your model is valid using the `mass.io.sbml.validate_sbml_model` function or via the online validator at http://sbml.org/validator .\n    \t`(model, errors) = validate_sbml_model(filename)`\n    If the model is valid and cannot be read please open an issue at https://github.com/SBRG/masspy/issues .', 'CombineArchiveExecutionError'], 'netpyne': ['FAIL', "No module named 'libsbml'", 'ModuleNotFoundError'], 'neuron': ['FAIL', "No module named 'libsbml'", 'ModuleNotFoundError'], 'opencor': ['FAIL', "No module named 'libsbml'", 'ModuleNotFoundError'], 'pyneuroml': ['FAIL', "No module named 'libsbml'", 'ModuleNotFoundError'], 'pysces': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    Model at /tmp/tmp1bq_quiv/./LEMS_NML2_Ex9_FN.sbml could not be imported:\n      \n      File /tmp/tmp1bq_quiv/./LEMS_NML2_Ex9_FN.sbml.xml does not exist', 'CombineArchiveExecutionError'], 'rbapy': ['FAIL', 'The COMBINE/OMEX did not execute successfully:\n\n  The SED document did not execute successfully:\n  \n    Language for model `net1` is not supported.\n      - Model language `urn:sedml:language:sbml` is not supported. Models must be in RBA format (e.g., `sed:model/@language` must match `^urn:sedml:language:rba(\\.|$)` such as `urn:sedml:language:rba`).', 'CombineArchiveExecutionError'], 'smoldyn': ['FAIL', "No module named 'libsbml'", 'ModuleNotFoundError'], 'tellurium': ['pass', '', ''], 'vcell': [None, '', ''], 'xpp': ['FAIL', "No module named 'libsbml'", 'ModuleNotFoundError']}
+results_paths = {
+    "local": os.path.join(path_to_sbml_folder, 'tests', 'results_local.json'),
+    "remote": os.path.join(path_to_sbml_folder, 'tests', 'results_remote.json')
+}
 
-results_table = utils.create_combined_results_table(results_remote, 
-                                  results_local, 
+results = {}
+for key, path in results_paths.items():
+    with open(path, 'r') as f:
+        results[key] = json.load(f)
+
+results_table = utils.create_combined_results_table(results["remote"], 
+                                  results["local"], 
                                   sedml_file_name, 
                                   sbml_file_name, 
                                   d1_plots_local_dir, 

test_suite/test_test_suite_compatibility_biosimulators.py

@@ -134,7 +134,7 @@ def process_cases(args):
                                  os.path.basename(sbml_file_path),
                                  os.path.join(test_folder,'d1_plots_remote'), 
                                  os.path.join(test_folder,'d1_plots_local'),
-                                 engines=engines, test_folder=test_folder)
+                                 test_folder=test_folder)
 
 
 if __name__ == "__main__":