From d9aa0773542288a309e37bd208b8e05c625c0bb0 Mon Sep 17 00:00:00 2001 From: Lucas Rebscher Date: Thu, 24 Dec 2020 11:37:14 +0100 Subject: [PATCH 001/961] #190 Get parallel simulation code running --- netpyne_ui/netpyne_geppetto.py | 62 +++++++++++++++++++++++----------- netpyne_ui/template.py | 4 +-- netpyne_ui/template2.py | 4 ++- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/netpyne_ui/netpyne_geppetto.py b/netpyne_ui/netpyne_geppetto.py index 0e20b2d3..60de7927 100644 --- a/netpyne_ui/netpyne_geppetto.py +++ b/netpyne_ui/netpyne_geppetto.py @@ -90,7 +90,7 @@ def _filter(_file): def instantiateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): - if not 'usePrevInst' in args or not args['usePrevInst']: + if not args.get("usePrevInst", False): netpyne_model = self.instantiateNetPyNEModel() self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model) @@ -101,35 +101,53 @@ def instantiateNetPyNEModelInGeppetto(self, args): def simulateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): - # TODO mpi is not finding libmpi.dylib.. set LD_LIBRARY_PATH to openmpi bin folder, but nothing if args['parallelSimulation']: + # nrniv needs NRN_PYLIB to be set to python executable path + os.environ["NRN_PYLIB"] = sys.executable + logging.debug('Running parallel simulation') - if not 'usePrevInst' in args or not args['usePrevInst']: - self.netParams.save("netParams.json") - self.simConfig.saveJson = True - self.simConfig.save("simParams.json") - template = os.path.join(os.path.dirname(__file__), 'template.py') - else: + + if args.get('usePrevInst', False): sim.cfg.saveJson = True oldName = sim.cfg.filename sim.cfg.filename = 'model_output' + + # workaround for issue with empty LFP dict when calling saveData() + del sim.allSimData['LFP'] + sim.saveData() sim.cfg.filename = oldName template = os.path.join(os.path.dirname(__file__), 'template2.py') + else: + self.netParams.save("netParams.json") + self.simConfig.saveJson = True + self.simConfig.save("simParams.json") + template = os.path.join(os.path.dirname(__file__), 'template.py') + copyfile(template, './init.py') - cp = subprocess.run(["mpiexec", "-n", args['cores'], "nrniv", "-mpi", "-python", "init.py"], - capture_output=True) - print(cp.stdout.decode() + cp.stderr.decode()) - if cp.returncode != 0: return utils.getJSONError("Error while simulating the NetPyNE model", - cp.stderr.decode()) + cores = str(args.get("cores", "1")) + + # TODO on linux, mpi is not finding libmpi.dylib + # requires LD_LIBRARY_PATH to be set + cp = subprocess.run( + ["mpiexec", "-n", cores, "nrniv", "-python", "-mpi", "init.py"], + capture_output=True + ) + logging.info(cp.stdout.decode() + cp.stderr.decode()) + if cp.returncode != 0: + return utils.getJSONError( + "Error while simulating the NetPyNE model", + cp.stderr.decode() + ) + sim.load('model_output.json') self.geppetto_model = self.model_interpreter.getGeppettoModel(sim) netpyne_model = sim - else: # single cpu computation - logging.info("Starting simulation") - if not 'usePrevInst' in args or not args['usePrevInst']: + else: + # single cpu computation + if not args.get('usePrevInst', False): logging.debug('Instantiating single thread simulation') netpyne_model = self.instantiateNetPyNEModel() self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model) @@ -139,6 +157,7 @@ def simulateNetPyNEModelInGeppetto(self, args): return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) except: + logging.error(sys.exc_info()) return utils.getJSONError("Error while simulating the NetPyNE model", sys.exc_info()) def compileModMechFiles(self, compileMod, modFolder): @@ -158,7 +177,7 @@ def compileModMechFiles(self, compileMod, modFolder): def loadModel(self, args): # handles all data coming from a .json file (default file system for Netpyne) def remove(dictionary): - # remove reserved keys such as __dict__, __Method__, etc + # remove reserved keys such as __dict__, __Method__, etc # they appear when we do sim.loadAll(json_file) if isinstance(dictionary, dict): for key, value in list(dictionary.items()): @@ -244,7 +263,7 @@ def importModel(self, modelParameters): netParamsPath = str(modelParameters["netParamsPath"]) sys.path.append(netParamsPath) os.chdir(netParamsPath) - # Import Module + # Import Module netParamsModuleName = importlib.import_module(str(modelParameters["netParamsModuleName"])) # Import Model attributes self.netParams = getattr(netParamsModuleName, str(modelParameters["netParamsVariable"])) @@ -257,7 +276,7 @@ def importModel(self, modelParameters): simConfigPath = str(modelParameters["simConfigPath"]) sys.path.append(simConfigPath) os.chdir(simConfigPath) - # Import Module + # Import Module simConfigModuleName = importlib.import_module(str(modelParameters["simConfigModuleName"])) # Import Model attributes self.simConfig = getattr(simConfigModuleName, str(modelParameters["simConfigVariable"])) @@ -358,10 +377,13 @@ def instantiateNetPyNEModel(self): with redirect_stdout(sys.__stdout__): saveData = sim.allSimData if hasattr(sim, 'allSimData') and 'spkt' in sim.allSimData.keys() and len( sim.allSimData['spkt']) > 0 else False + sim.create(self.netParams, self.simConfig) sim.net.defineCellShapes() # creates 3d pt for cells with stylized geometries sim.gatherData(gatherLFP=False) - if saveData: sim.allSimData = saveData # preserve data from previous simulation + + if saveData: + sim.allSimData = saveData # preserve data from previous simulation return sim diff --git a/netpyne_ui/template.py b/netpyne_ui/template.py index 2a370fd2..a7c50d25 100644 --- a/netpyne_ui/template.py +++ b/netpyne_ui/template.py @@ -1,8 +1,6 @@ from netpyne import sim + netParams = sim.loadNetParams("./netParams.json", None, False) simConfig = sim.loadSimCfg("./simParams.json", None, False) sim.createSimulateAnalyze(netParams, simConfig) - - - diff --git a/netpyne_ui/template2.py b/netpyne_ui/template2.py index 17360fc2..df2c8178 100644 --- a/netpyne_ui/template2.py +++ b/netpyne_ui/template2.py @@ -1,3 +1,5 @@ from netpyne import sim + sim.load("./model_output.json") -sim.simulate() \ No newline at end of file +sim.simulate() +sim.analyze() From d62a8f6e414b0fccc2fa9feaf51b12e8ffd8b17d Mon Sep 17 00:00:00 2001 From: Lucas Rebscher Date: Thu, 24 Dec 2020 16:41:14 +0100 Subject: [PATCH 002/961] #190 Support for openmpi in docker image * Requires libopenmpi-dev to be installed * Need to test openmpi on different OSs and Python installations, depending on environment it requires different NRN_PYLIB env to be set, need some robust logic for this --- Dockerfile | 7 ++++++- netpyne_ui/netpyne_geppetto.py | 23 +++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1b2c17f..bef52da0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ FROM frodriguez4600/jupyter-neuron:v7.8.0 -USER $NB_USER ENV INSTALLATION_FOLDER=/home/jovyan/work/NetPyNE-UI ENV NETPYNE_VERSION=development @@ -8,6 +7,12 @@ ENV JUPYTER_GEPPETTO_VERSION=development ENV PYGEPPETTO_VERSION=development ENV BUILD_ARGS="" +# Install openmpi for parallel simulations +# Important: Have to switch to root to install a package and ensure to switch back to NB user afterwards +USER root +RUN apt-get update && apt-get install -y libopenmpi-dev +USER $NB_USER + WORKDIR /home/jovyan/work COPY --chown=1000:1000 requirements.txt ${INSTALLATION_FOLDER}/requirements.txt diff --git a/netpyne_ui/netpyne_geppetto.py b/netpyne_ui/netpyne_geppetto.py index 60de7927..f9ff7595 100644 --- a/netpyne_ui/netpyne_geppetto.py +++ b/netpyne_ui/netpyne_geppetto.py @@ -80,12 +80,12 @@ def getData(self): def find_tutorials(self): from os import listdir from os.path import isfile, join - onlyfiles = [f for f in listdir(NETPYNE_WORKDIR_PATH) if isfile(join(NETPYNE_WORKDIR_PATH, f))] + only_files = [f for f in listdir(NETPYNE_WORKDIR_PATH) if isfile(join(NETPYNE_WORKDIR_PATH, f))] def _filter(_file): return '.py' in _file and 'tut' in _file and 'gui' in _file - return list(filter(_filter, onlyfiles)) + return list(filter(_filter, only_files)) def instantiateNetPyNEModelInGeppetto(self, args): try: @@ -102,8 +102,12 @@ def simulateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): if args['parallelSimulation']: - # nrniv needs NRN_PYLIB to be set to python executable path - os.environ["NRN_PYLIB"] = sys.executable + + # TODO: need to understand when to set NRN_PYLIB + # it's required for pure Python installation, but breaks when using Conda + if "CONDA_VERSION" not in os.environ: + # nrniv needs NRN_PYLIB to be set to python executable path (MacOS, Python 3.7.6) + os.environ["NRN_PYLIB"] = sys.executable logging.debug('Running parallel simulation') @@ -127,9 +131,6 @@ def simulateNetPyNEModelInGeppetto(self, args): copyfile(template, './init.py') cores = str(args.get("cores", "1")) - - # TODO on linux, mpi is not finding libmpi.dylib - # requires LD_LIBRARY_PATH to be set cp = subprocess.run( ["mpiexec", "-n", cores, "nrniv", "-python", "-mpi", "init.py"], capture_output=True @@ -175,7 +176,13 @@ def compileModMechFiles(self, compileMod, modFolder): if modFolder: neuron.load_mechanisms(str(modFolder)) - def loadModel(self, args): # handles all data coming from a .json file (default file system for Netpyne) + def loadModel(self, args): + """ Handles all data coming from a .json file (default file format for Netpyne). + + :param args: + :return: + """ + def remove(dictionary): # remove reserved keys such as __dict__, __Method__, etc # they appear when we do sim.loadAll(json_file) From eb5b218313d5829e940804d3a2e1f8511cd15051 Mon Sep 17 00:00:00 2001 From: Lucas Rebscher Date: Mon, 28 Dec 2020 12:48:44 +0100 Subject: [PATCH 003/961] #190 Log all caught exceptions * Log exceptions in backend * Code cleanup --- netpyne_ui/api.py | 30 +- netpyne_ui/constants.py | 2 +- netpyne_ui/netpyne_geppetto.py | 152 +++++---- webapp/Utils.js | 19 +- .../definition/cellRules/NetPyNECellRules.js | 291 +++++++++--------- .../general/GeppettoJupyterUtils.js | 18 +- webapp/redux/reducers/errors.js | 4 +- 7 files changed, 278 insertions(+), 238 deletions(-) diff --git a/netpyne_ui/api.py b/netpyne_ui/api.py index 91cac361..2a110293 100644 --- a/netpyne_ui/api.py +++ b/netpyne_ui/api.py @@ -8,7 +8,8 @@ from tempfile import TemporaryDirectory from jupyter_geppetto.webapi import get, post from notebook.base.handlers import IPythonHandler -from netpyne_ui.constants import NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME, ALLOWED_EXTENSIONS, UPLOAD_FOLDER_PATH +from netpyne_ui.constants import ALLOWED_EXTENSIONS, UPLOAD_FOLDER_PATH + def allowed_file(filename, allowed_extensions=ALLOWED_EXTENSIONS): return '.' in filename and \ @@ -26,7 +27,7 @@ def send_files(handler, file_path, filename): if _buffer: handler.write(_buffer) else: - return + return except: handler.set_status(500, f"Error sending files") @@ -39,16 +40,17 @@ def get_file_paths(handler): for path in tmp_file_paths: if os.path.exists(path): file_paths.append(path) - + return file_paths + class NetPyNEController: # pytest: no cover @post('/uploads') def uploads(handler: IPythonHandler): files = handler.request.files files_saved = 0 - + if len(files) == 0 or 'file' not in files: handler.set_status(400, f"Can't find 'file' or filename is empty. Files received {len(files)}") else: @@ -58,13 +60,13 @@ def uploads(handler: IPythonHandler): logging.warn(f"Can't store file {f.filename}. Extension not allowed") continue - ## Save to file + # Save to file filename = f.filename file_path = os.path.join(UPLOAD_FOLDER_PATH, filename) - + with open(file_path, 'wb') as zf: zf.write(f['body']) - + files_saved += 1 if filename.endswith('.zip'): @@ -78,18 +80,18 @@ def uploads(handler: IPythonHandler): elif filename.endswith('.gz'): with gzip.open(file_path, "rb") as gz, open(file_path.replace('.gz', ''), 'wb') as ff: shutil.copyfileobj(gz, ff) - + handler.set_status(200, f"Number of files saved: {files_saved}. Number of files sent: {len(files['file'])}") handler.finish() - + @get('/downloads') def downloads(handler: IPythonHandler): file_paths = get_file_paths(handler) - + if file_paths: - + if len(file_paths) == 0: handler.set_status(400, f"Files not found.") handler.finish() @@ -97,8 +99,8 @@ def downloads(handler: IPythonHandler): if len(file_paths) == 1: send_files(handler, file_paths[0], file_paths[0].split('/')[-1]) - - else : + + else: with TemporaryDirectory() as dir_path: tar_gz_file_name = f'{str(uuid.uuid4())}.tar.gz' tar_gz_file_path = os.path.join(dir_path, tar_gz_file_name) @@ -107,5 +109,5 @@ def downloads(handler: IPythonHandler): tar.add(file_path, os.path.join('download', file_path.split('/')[-1])) send_files(handler, tar_gz_file_path, tar_gz_file_name) - + handler.finish() diff --git a/netpyne_ui/constants.py b/netpyne_ui/constants.py index 78eacb53..c57ed852 100644 --- a/netpyne_ui/constants.py +++ b/netpyne_ui/constants.py @@ -12,4 +12,4 @@ os.makedirs(UPLOAD_FOLDER_PATH) if not os.path.exists(NETPYNE_WORKDIR_PATH): - os.makedirs(NETPYNE_WORKDIR_PATH) \ No newline at end of file + os.makedirs(NETPYNE_WORKDIR_PATH) diff --git a/netpyne_ui/netpyne_geppetto.py b/netpyne_ui/netpyne_geppetto.py index f9ff7595..7bfa38a9 100644 --- a/netpyne_ui/netpyne_geppetto.py +++ b/netpyne_ui/netpyne_geppetto.py @@ -1,7 +1,9 @@ """ netpyne_geppetto.py + Initialise NetPyNE Geppetto, this class contains methods to connect NetPyNE with the Geppetto based UI """ + import json import os import importlib @@ -95,8 +97,10 @@ def instantiateNetPyNEModelInGeppetto(self, args): self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model) return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) - except: - return utils.getJSONError("Error while instantiating the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while instantiating the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) def simulateNetPyNEModelInGeppetto(self, args): try: @@ -144,7 +148,6 @@ def simulateNetPyNEModelInGeppetto(self, args): sim.load('model_output.json') self.geppetto_model = self.model_interpreter.getGeppettoModel(sim) - netpyne_model = sim else: # single cpu computation @@ -154,12 +157,13 @@ def simulateNetPyNEModelInGeppetto(self, args): self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model) logging.debug('Running single thread simulation') - netpyne_model = self.simulateNetPyNEModel() + self.simulateNetPyNEModel() return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) - except: - logging.error(sys.exc_info()) - return utils.getJSONError("Error while simulating the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while simulating the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) def compileModMechFiles(self, compileMod, modFolder): # Create Symbolic link @@ -199,8 +203,10 @@ def remove(dictionary): try: owd = os.getcwd() self.compileModMechFiles(args['compileMod'], args['modFolder']) - except: - return utils.getJSONError("Error while importing/compiling mods", sys.exc_info()) + except Exception: + message = "Error while importing/compiling mods" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) finally: os.chdir(owd) @@ -210,7 +216,8 @@ def remove(dictionary): wake_up_geppetto = False if all([args[option] for option in ['loadNetParams', 'loadSimCfg', 'loadSimData', 'loadNet']]): wake_up_geppetto = True - if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() + if self.doIhaveInstOrSimData()['haveInstance']: + sim.clearAll() sim.initialize() sim.loadAll(args['jsonModelFolder']) self.netParams = sim.net.params @@ -220,11 +227,13 @@ def remove(dictionary): else: if args['loadNet']: wake_up_geppetto = True - if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() + if self.doIhaveInstOrSimData()['haveInstance']: + sim.clearAll() sim.initialize() sim.loadNet(args['jsonModelFolder']) - if args['loadSimData']: # TODO (https://github.com/Neurosim-lab/netpyne/issues/360) + # TODO (https://github.com/Neurosim-lab/netpyne/issues/360) + if args['loadSimData']: wake_up_geppetto = True if not self.doIhaveInstOrSimData()['haveInstance']: sim.create(specs.NetParams(), specs.SimConfig()) @@ -238,7 +247,8 @@ def remove(dictionary): remove(self.simConfig.todict()) if args['loadNetParams']: - if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() + if self.doIhaveInstOrSimData()['haveInstance']: + sim.clearAll() sim.loadNetParams(args['jsonModelFolder']) self.netParams = sim.net.params remove(self.netParams.todict()) @@ -246,7 +256,7 @@ def remove(dictionary): if wake_up_geppetto: if len(sim.net.cells) > 0: section = list(sim.net.cells[0].secs.keys())[0] - if not 'pt3d' in list(sim.net.cells[0].secs[section].geom.keys()): + if 'pt3d' not in list(sim.net.cells[0].secs[section].geom.keys()): sim.net.defineCellShapes() sim.gatherData() sim.loadSimData(args['jsonModelFolder']) @@ -255,8 +265,10 @@ def remove(dictionary): return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) else: return utils.getJSONReply() - except: - return utils.getJSONError("Error while loading the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while loading the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) def importModel(self, modelParameters): try: @@ -270,8 +282,10 @@ def importModel(self, modelParameters): netParamsPath = str(modelParameters["netParamsPath"]) sys.path.append(netParamsPath) os.chdir(netParamsPath) + # Import Module netParamsModuleName = importlib.import_module(str(modelParameters["netParamsModuleName"])) + # Import Model attributes self.netParams = getattr(netParamsModuleName, str(modelParameters["netParamsVariable"])) @@ -283,14 +297,18 @@ def importModel(self, modelParameters): simConfigPath = str(modelParameters["simConfigPath"]) sys.path.append(simConfigPath) os.chdir(simConfigPath) + # Import Module simConfigModuleName = importlib.import_module(str(modelParameters["simConfigModuleName"])) + # Import Model attributes self.simConfig = getattr(simConfigModuleName, str(modelParameters["simConfigVariable"])) return utils.getJSONReply() - except: - return utils.getJSONError("Error while importing the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while importing the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) finally: os.chdir(owd) @@ -314,8 +332,10 @@ def importCellTemplate(self, modelParameters): self.netParams.cellParams[rule] = self.netParams.cellParams[rule].todict() return utils.getJSONReply() - except: - return utils.getJSONError("Error while importing the NetPyNE cell template", sys.exc_info()) + except Exception: + message = "Error while importing the NetPyNE cell template" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) finally: os.chdir(owd) @@ -335,17 +355,20 @@ def exportModel(self, args): data = json.load(json_file) return data - return utils.getJSONReply() - except: - return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while exporting the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) def exportNeuroML(self, modelParams): try: with redirect_stdout(sys.__stdout__): sim.exportNeuroML2(modelParams['fileName'], specs.SimConfig()) return utils.getJSONReply() - except: - return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while exporting the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) def importNeuroML(self, modelParams): try: @@ -356,27 +379,29 @@ def importNeuroML(self, modelParams): self.geppetto_model = self.model_interpreter.getGeppettoModel(sim) return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) - except: - return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while exporting the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) def deleteModel(self, modelParams): - try: with redirect_stdout(sys.__stdout__): self.netParams = specs.NetParams() self.simConfig = specs.SimConfig() sim.initialize(specs.NetParams(), specs.SimConfig()) self.geppetto_model = None - except: - return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while exporting the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) try: # This function fails is some keys don't exists # sim.clearAll() self.clearSim() - - except: - pass + except Exception: + logging.exception("Error while clearing simulation") return utils.getJSONReply() @@ -418,15 +443,18 @@ def doIhaveInstOrSimData(self): # return [bool, bool] telling if we have an ins def rename(self, path, oldValue, newValue): command = 'sim.rename(self.' + path + ',"' + oldValue + '","' + newValue + '")' logging.debug('renaming ' + command) + eval(command) for model, synched_component in list(jupyter_geppetto.synched_models.items()): if model != '' and oldValue in model and path in model: # jupyter_geppetto.synched_models.pop(model) - newModel = re.sub("(['])(?:(?=(\\?))\2.)*?\1", lambda x: x.group(0).replace(oldValue, newValue, 1), - model) - logging.debug("Rename funct - Model is " + model + " newModel is " + newModel) - jupyter_geppetto.synched_models[newModel] = synched_component + new_model = re.sub("(['])(?:(?=(\\?))\2.)*?\1", + lambda x: x.group(0).replace(oldValue, newValue, 1), + model) + logging.debug("Rename funct - Model is " + model + " newModel is " + new_model) + jupyter_geppetto.synched_models[new_model] = synched_component + with redirect_stdout(sys.__stdout__): if "popParams" in path: self.propagate_field_rename("pop", newValue, oldValue) @@ -444,7 +472,7 @@ def getPlotSettings(self, plot_name): def getDirList(self, dir=None, onlyDirs=False, filterFiles=False): # Get Current dir - if dir == None or dir == '': + if dir is None or dir == '': dir = os.path.join(os.getcwd(), NETPYNE_WORKDIR_PATH) dir_list = [] file_list = [] @@ -483,11 +511,9 @@ def getPlot(self, plotName, LFPflavour, theme='gui'): return html else: - - figData = getattr(analysis, plotName)(**args) - - if isinstance(figData, tuple): - fig = figData[0] + fig_data = getattr(analysis, plotName)(**args) + if isinstance(fig_data, tuple): + fig = fig_data[0] if fig == -1: return fig elif isinstance(fig, list): @@ -500,12 +526,8 @@ def getPlot(self, plotName, LFPflavour, theme='gui'): else: return [ui.getSVG(fig)] else: - return figData - - - + return fig_data except Exception as e: - # TODO: Extract these two lines as a function and call it in every catch clause err = "There was an exception in %s():" % (e.plotName) logging.exception(("%s \n %s \n%s" % (err, e, sys.exc_info()))) @@ -513,19 +535,19 @@ def getAvailablePops(self): return list(self.netParams.popParams.keys()) def getAvailableCellModels(self): - cellModels = set([]) + cell_models = set([]) for p in self.netParams.popParams: if 'cellModel' in self.netParams.popParams[p]: cm = self.netParams.popParams[p]['cellModel'] - if cm not in cellModels: - cellModels.add(cm) - return list(cellModels) + if cm not in cell_models: + cell_models.add(cm) + return list(cell_models) def getAvailableCellTypes(self): - cellTypes = set([]) + cell_types = set([]) for p in self.netParams.cellParams: - cellTypes.add(p) - return list(cellTypes) + cell_types.add(p) + return list(cell_types) def getAvailableSections(self): sections = {} @@ -605,7 +627,8 @@ def deleteParam(self, model, label): elif "synMechParams" in model: self.propagate_field_rename("synMech", None, label) return True - except: + except Exception: + logging.exception(f"Error while deleting parameter: {label}") return False def validateFunction(self, functionString): @@ -661,8 +684,10 @@ def header(title, spacer='-'): with open(fname) as f: return f.read() - except: - return utils.getJSONError("Error while importing the NetPyNE model", sys.exc_info()) + except Exception: + message = "Error while importing the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) def propagate(self, model, label, cond, new, old): with redirect_stdout(sys.__stdout__): @@ -762,7 +787,7 @@ def propagate_syn_mech_rename(self, new, old): if self.netParams.stimSourceParams[self.netParams.stimTargetParams[label]['source']][ 'type'] == 'NetStim': if old == self.netParams.stimTargetParams[label]['synMech']: - if new == None: + if new is None: self.netParams.stimTargetParams[label].pop('synMech') else: self.netParams.stimTargetParams[label]['synMech'] = new @@ -808,7 +833,7 @@ def clearSim(self): del sim.net - import gc; + import gc gc.collect() def create_celltype_from_template(self, label="CellType", conds={}, cell_template_name="Blank"): @@ -816,9 +841,10 @@ def create_celltype_from_template(self, label="CellType", conds={}, cell_templat with redirect_stdout(sys.__stdout__): self.netParams.addCellParamsTemplate(label=label, template=cell_template_name) return True - except: - return utils.getJSONError(f"Error while creating cellType from template {cell_template_name}", - sys.exc_info()) + except Exception: + message = f"Error while creating cellType from template {cell_template_name}" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) logging.info("Initialising NetPyNE UI") diff --git a/webapp/Utils.js b/webapp/Utils.js index 0f014914..ce25537b 100644 --- a/webapp/Utils.js +++ b/webapp/Utils.js @@ -9,11 +9,11 @@ const Utils = { return prefix; } // Get New Available ID - var id = prefix; + let id = prefix; if (model[id + 0] == undefined) { return id + 0 } - var i = 0; + let i = 0; id = prefix + i++ while (model[id] != undefined) { id = prefix + i++; @@ -25,8 +25,8 @@ const Utils = { if (key == undefined) { return; } - var currentObject; - var nextObject = window.metadata; + let currentObject; + let nextObject = window.metadata; key.split('.').forEach(item => { if (item in nextObject) { currentObject = nextObject[item]; @@ -42,14 +42,15 @@ const Utils = { }, getHTMLType: function (key) { - var type = this.getMetadataField(key, "type") + const type = this.getMetadataField(key, "type"); + let htmlType; switch (type) { case "int": - var htmlType = "number" + htmlType = "number"; break; default: - var htmlType = "text" + htmlType = "text"; break; } return htmlType; @@ -126,7 +127,7 @@ const Utils = { return name; }, - + // FIXME: Hack to remove scaped chars (\\ -> \ and \' -> ') manually convertToJSON (data){ if (typeof data === 'string' || data instanceof String){ @@ -136,7 +137,7 @@ const Utils = { }, getErrorResponse (data){ - var parsedData = this.convertToJSON(data) + const parsedData = this.convertToJSON(data); if (parsedData.type && parsedData['type'] == 'ERROR') { const error = { details: parsedData['details'] } if (Object.prototype.hasOwnProperty.call(parsedData, "message")) { diff --git a/webapp/components/definition/cellRules/NetPyNECellRules.js b/webapp/components/definition/cellRules/NetPyNECellRules.js index deccfd0c..3b295c3f 100644 --- a/webapp/components/definition/cellRules/NetPyNECellRules.js +++ b/webapp/components/definition/cellRules/NetPyNECellRules.js @@ -96,9 +96,9 @@ export default class NetPyNECellRules extends React.Component { // Create Cell Rule Client side Utils.execPythonMessage( 'netpyne_geppetto.netParams.cellParams["' - + cellRuleId - + '"] = ' - + JSON.stringify(value) + + cellRuleId + + '"] = ' + + JSON.stringify(value) ); model[cellRuleId] = newCellRule; // Update state @@ -133,17 +133,17 @@ export default class NetPyNECellRules extends React.Component { model[selectedCellRule]["secs"] = {}; Utils.execPythonMessage( 'netpyne_geppetto.netParams.cellParams["' - + selectedCellRule - + '"]["secs"] = {}' + + selectedCellRule + + '"]["secs"] = {}' ); } Utils.execPythonMessage( 'netpyne_geppetto.netParams.cellParams["' - + selectedCellRule - + '"]["secs"]["' - + sectionId - + '"] = ' - + JSON.stringify(value) + + selectedCellRule + + '"]["secs"]["' + + sectionId + + '"] = ' + + JSON.stringify(value) ); model[selectedCellRule]["secs"][sectionId] = newSection; // Update state @@ -172,7 +172,7 @@ export default class NetPyNECellRules extends React.Component { Object.keys(secs).forEach(sectionName => { const { mechs, ...secOthers } = this.state.value[cellRuleName].secs[ sectionName - ]; + ]; sections[sectionName] = { ...secOthers, mechs: { ...mechs } }; }); model[cellRuleName] = { ...cellOthers, secs: { ...sections } }; @@ -183,10 +183,10 @@ export default class NetPyNECellRules extends React.Component { model[selectedCellRule].secs[selectedSection]["mechs"] = {}; Utils.execPythonMessage( 'netpyne_geppetto.netParams.cellParams["' - + selectedCellRule - + '"]["secs"]["' - + selectedSection - + '"]["mechs"] = {}' + + selectedCellRule + + '"]["secs"]["' + + selectedSection + + '"]["mechs"] = {}' ); } Utils.evalPythonMessage("netpyne_geppetto.getMechParams", [mechanism]).then( @@ -195,13 +195,13 @@ export default class NetPyNECellRules extends React.Component { response.forEach(param => (params[param] = 0)); Utils.execPythonMessage( 'netpyne_geppetto.netParams.cellParams["' - + selectedCellRule - + '"]["secs"]["' - + selectedSection - + '"]["mechs"]["' - + mechanism - + '"] = ' - + JSON.stringify(params) + + selectedCellRule + + '"]["secs"]["' + + selectedSection + + '"]["mechs"]["' + + mechanism + + '"] = ' + + JSON.stringify(params) ); } ); @@ -324,7 +324,7 @@ export default class NetPyNECellRules extends React.Component { } else if ( prevState.value !== undefined && Object.keys(prevState.value).length - !== Object.keys(this.state.value).length + !== Object.keys(this.state.value).length ) { /* * logic into this if to check if the user added a new object from the python backend and @@ -353,7 +353,8 @@ export default class NetPyNECellRules extends React.Component { "netParams.cellParams", m, newValue, - (response, newValue) => {} + (response, newValue) => { + } ); }.bind(this) ); @@ -381,7 +382,7 @@ export default class NetPyNECellRules extends React.Component { if ( prevModel[n] !== undefined && Object.keys(model2[n]["secs"]).length - !== Object.keys(prevModel[n]["secs"]).length + !== Object.keys(prevModel[n]["secs"]).length ) { var cellRule = model2[n]["secs"]; for (var s in cellRule) { @@ -406,7 +407,8 @@ export default class NetPyNECellRules extends React.Component { 'netParams.cellParams["' + n + '"]["secs"]', s, newValue2, - (response, newValue) => {} + (response, newValue) => { + } ) ); } @@ -488,10 +490,11 @@ export default class NetPyNECellRules extends React.Component { } } }; + shouldComponentUpdate (nextProps, nextState) { var itemRenamed = this.hasSelectedCellRuleBeenRenamed(this.state, nextState) - !== undefined + !== undefined || this.hasSelectedSectionBeenRenamed(this.state, nextState) !== undefined || this.hasSelectedMechanismBeenRenamed(this.state, nextState) !== undefined; var newItemCreated = false; @@ -511,14 +514,14 @@ export default class NetPyNECellRules extends React.Component { ) { var oldLength = this.state.value[this.state.selectedCellRule] == undefined - ? 0 - : Object.keys(this.state.value[this.state.selectedCellRule].secs) - .length; + ? 0 + : Object.keys(this.state.value[this.state.selectedCellRule].secs) + .length; newItemCreated = newItemCreated || oldLength - != Object.keys(nextState.value[this.state.selectedCellRule].secs) - .length; + != Object.keys(nextState.value[this.state.selectedCellRule].secs) + .length; } if ( this.state.selectedSection != undefined @@ -526,26 +529,26 @@ export default class NetPyNECellRules extends React.Component { && nextState.value[this.state.selectedCellRule] != undefined && nextState.value[this.state.selectedCellRule].secs[ this.state.selectedSection - ] != undefined + ] != undefined ) { var oldLength = this.state.value[this.state.selectedCellRule].secs[ - this.state.selectedSection + this.state.selectedSection ] == undefined - ? 0 - : Object.keys( - this.state.value[this.state.selectedCellRule].secs[ - this.state.selectedSection + ? 0 + : Object.keys( + this.state.value[this.state.selectedCellRule].secs[ + this.state.selectedSection ].mechs - ).length; + ).length; newItemCreated = newItemCreated || oldLength - != Object.keys( - nextState.value[this.state.selectedCellRule].secs[ - this.state.selectedSection + != Object.keys( + nextState.value[this.state.selectedCellRule].secs[ + this.state.selectedSection ].mechs - ).length; + ).length; } } var errorDialogOpen = this.state.errorDetails !== nextState.errorDetails; @@ -573,7 +576,7 @@ export default class NetPyNECellRules extends React.Component { var model = this.state.value; delete model[this.state.selectedCellRule].secs[ this.state.selectedSection - ]["mechs"][name]; + ]["mechs"][name]; this.setState({ value: model, selectedMechanism: undefined }); }); } @@ -627,43 +630,43 @@ export default class NetPyNECellRules extends React.Component { } = this.state; switch (rule) { - case "cellRule": - if (page !== "main") { - if (selectedCellRule && selectedCellRule.length > 8) { - return selectedCellRule; + case "cellRule": + if (page !== "main") { + if (selectedCellRule && selectedCellRule.length > 8) { + return selectedCellRule; + } else { + return "Go back to cell type"; + } } else { - return "Go back to cell type"; + return "Create new cell type"; } - } else { - return "Create new cell type"; - } - case "section": - if (page === "mechanisms") { - if (!!selectedSection && selectedSection.length > 9) { - return selectedSection; - } else { - return "Go back to section"; - } - } else { - if (page == "sections") { - return "Create new section"; + case "section": + if (page === "mechanisms") { + if (!!selectedSection && selectedSection.length > 9) { + return selectedSection; + } else { + return "Go back to section"; + } } else { - if (selectedCellRule) { - if ( - !!model + if (page == "sections") { + return "Create new section"; + } else { + if (selectedCellRule) { + if ( + !!model && !!model[selectedCellRule] && Object.keys(model[selectedCellRule]["secs"]).length > 0 - ) { - return "Explore sections"; + ) { + return "Explore sections"; + } else { + return "Create first section"; + } } else { - return "Create first section"; + return "No cell type selected"; } - } else { - return "No cell type selected"; } } - } } } @@ -675,35 +678,35 @@ export default class NetPyNECellRules extends React.Component { selectedSection, } = this.state; switch (rule) { - case "cellRule": - if (page !== "main") { - return "CT"; - } else { - return ; - } + case "cellRule": + if (page !== "main") { + return "CT"; + } else { + return ; + } - case "sections": - if (page === "mechanisms") { - return "S"; - } else { - if (page == "sections") { - return ; + case "sections": + if (page === "mechanisms") { + return "S"; } else { - if (selectedCellRule) { - if ( - !!model + if (page == "sections") { + return ; + } else { + if (selectedCellRule) { + if ( + !!model && !!model[selectedCellRule] && Object.keys(model[selectedCellRule]["secs"]).length > 0 - ) { - return ; + ) { + return ; + } else { + return ; + } } else { - return ; + return ""; } - } else { - return ""; } } - } } } @@ -743,28 +746,28 @@ export default class NetPyNECellRules extends React.Component { return "undefined"; } switch (this.state.page) { - case "main": { - if (model[selectedCellRule]) { - return `${basePath}["${selectedCellRule}"]`; + case "main": { + if (model[selectedCellRule]) { + return `${basePath}["${selectedCellRule}"]`; + } + break; } - break; - } - case "sections": { - if (model[selectedCellRule].secs[selectedSection]) { - return `${basePath}["${selectedCellRule}"].secs["${selectedSection}"]`; + case "sections": { + if (model[selectedCellRule].secs[selectedSection]) { + return `${basePath}["${selectedCellRule}"].secs["${selectedSection}"]`; + } + break; } - break; - } - case "mechanisms": { - if ( - model[selectedCellRule].secs[selectedSection].mechs[selectedMechanism] - ) { - return `${basePath}["${selectedCellRule}"].secs["${selectedSection}"].mechs["${selectedMechanism}"]`; + case "mechanisms": { + if ( + model[selectedCellRule].secs[selectedSection].mechs[selectedMechanism] + ) { + return `${basePath}["${selectedCellRule}"].secs["${selectedSection}"].mechs["${selectedMechanism}"]`; + } + break; + } + default: { } - break; - } - default: { - } } return "undefined"; } @@ -789,35 +792,35 @@ export default class NetPyNECellRules extends React.Component { const dialogPop = errorMessage != undefined ? ( - - {errorMessage} - - - {errorDetails} - - - - - - - ) : ( - undefined - ); + + {errorMessage} + + + {errorDetails} + + + + + + + ) : ( + undefined + ); if (page == "main") { if ( @@ -1044,16 +1047,16 @@ export default class NetPyNECellRules extends React.Component { - - + + this.setState({ filterValue: newValue }) diff --git a/webapp/components/general/GeppettoJupyterUtils.js b/webapp/components/general/GeppettoJupyterUtils.js index 14cc003c..fa6b7b92 100644 --- a/webapp/components/general/GeppettoJupyterUtils.js +++ b/webapp/components/general/GeppettoJupyterUtils.js @@ -16,7 +16,11 @@ const handle_output = function (data) { } catch (error) { var response = data.content.data['text/plain'].replace(/^'(.*)'$/, '$1'); } - GEPPETTO.trigger(GEPPETTO.Events.Receive_Python_Message, { id: data.parent_header.msg_id, type: data.msg_type, response: response }); + GEPPETTO.trigger(GEPPETTO.Events.Receive_Python_Message, { + id: data.parent_header.msg_id, + type: data.msg_type, + response: response + }); break; case "display_data": // FIXME @@ -28,8 +32,12 @@ const handle_output = function (data) { const execPythonMessage = function (command, callback = handle_output) { GEPPETTO.CommandController.log('Executing Python command: ' + command, true); - var kernel = IPython.notebook.kernel; - var messageID = kernel.execute(command, { iopub: { output: callback } }, { silent: false, stop_on_error: true, store_history: true }); + const kernel = IPython.notebook.kernel; + const messageID = kernel.execute(command, { iopub: { output: callback } }, { + silent: false, + stop_on_error: true, + store_history: true + }); return new Promise((resolve, reject) => GEPPETTO.on(GEPPETTO.Events.Receive_Python_Message, function (data) { @@ -41,7 +49,7 @@ const execPythonMessage = function (command, callback = handle_output) { }; const evalPythonMessage = function (command, parameters, parse = true) { - var parametersString = ''; + let parametersString = ''; if (parameters) { if (parameters.length > 0) { parametersString = "(" + parameters.map(parameter => "utils.convertToPython('" + JSON.stringify(parameter) + "')").join(",") + ")"; @@ -50,7 +58,7 @@ const evalPythonMessage = function (command, parameters, parse = true) { } } - var finalCommand = command + parametersString; + let finalCommand = command + parametersString; if (parse) { finalCommand = 'utils.convertToJS(' + finalCommand + ')' } diff --git a/webapp/redux/reducers/errors.js b/webapp/redux/reducers/errors.js index 4ebb289d..4a1ed5bb 100644 --- a/webapp/redux/reducers/errors.js +++ b/webapp/redux/reducers/errors.js @@ -2,7 +2,7 @@ import { OPEN_BACKEND_ERROR_DIALOG, CLOSE_BACKEND_ERROR_DIALOG } from '../actions/errors'; // Default state for general -export const ERROR_DEFAULT_STATE = { +export const ERROR_DEFAULT_STATE = { openDialog: false, errorMessage: '', errorDetails: '' @@ -13,7 +13,7 @@ export const ERROR_DEFAULT_STATE = { export default function reduceError (state = ERROR_DEFAULT_STATE, action) { switch (action.type) { case OPEN_BACKEND_ERROR_DIALOG: - return { + return { ...state, openDialog: true, errorMessage: action.payload, From b537ae0ba8fcdfa19ea9b90007cfb0f7d6fc02ed Mon Sep 17 00:00:00 2001 From: Lucas Rebscher Date: Mon, 28 Dec 2020 12:54:46 +0100 Subject: [PATCH 004/961] #190 Correct mistake in README --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 82a06054..12b956d0 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,18 @@ ![Screenshot](https://github.com/MetaCell/NetPyNE-UI/raw/documentation/docs/netpyneui.png) -This repository hosts the User Interface for [NetPyNE](http://www.neurosimlab.org/netpyne/). NetPyNE is a python package to facilitate the development, parallel simulation and analysis of biological neuronal networks using the NEURON simulator. +This repository hosts the User Interface for [NetPyNE](http://www.neurosimlab.org/netpyne/). NetPyNE is a python package +to facilitate the development, parallel simulation and analysis of biological neuronal networks using the NEURON +simulator. ## Install NetPyNE User Interface -Select one option to install the NetPyNE User Interface. +Select one option to install the NetPyNE User Interface. -If you are familiar with NEURON and have already NEURON installed in your machine you can proceed using Pip. If you want a container which comes with everything preinstalled including NEURON you can use the Docker image. Using docker you will still be able to mount a local folder which will be your NetPyNE workspace. If you don't have docker installed in your system and you have had troubles installing it you can opt for the Virtual Machine installation. +If you are familiar with NEURON and have already NEURON installed in your machine you can proceed using Pip. If you want +a container which comes with everything preinstalled including NEURON you can use the Docker image. Using docker you +will still be able to mount a local folder which will be your NetPyNE workspace. If you don't have docker installed in +your system and you have had troubles installing it you can opt for the Virtual Machine installation.

Pip @@ -57,7 +62,6 @@ For debugging you can use `run.py` instead python run.py ``` - ## Run NetPyNE User Interface in Docker Ensure that you have Docker installed on your system. @@ -65,7 +69,7 @@ Ensure that you have Docker installed on your system. Build the image ```bash -docker build -t netpyne-ui +docker build -t netpyne-ui . ``` Run the image @@ -76,8 +80,8 @@ docker run -p 8888:8888 netpyne-ui ## End-to-end tests -End-to-end tests are located in `tests/deployment/frontend/e2e`. -Ensure that the application is running in a blank state, since end-to-end tests interact with the running application. +End-to-end tests are located in `tests/deployment/frontend/e2e`. Ensure that the application is running in a blank +state, since end-to-end tests interact with the running application. Install packages @@ -92,11 +96,9 @@ Start tests npm run test ``` - #### Containerized tests -You can also use `docker-compose` to run the tests. -Ensure that you have Docker installed on your system. +You can also use `docker-compose` to run the tests. Ensure that you have Docker installed on your system. Build the images @@ -111,8 +113,7 @@ Run the tests docker-compose up --abort-on-container-exit --exit-code-from netpyne-ui-e2e ``` - ## Additional Notes -NetPyNE-UI is being developed in collaboration with the [Neurosim Lab](http://neurosimlab.org/). -See the [Wiki](https://github.com/MetaCell/NetPyNE-UI/wiki) for more info! \ No newline at end of file +NetPyNE-UI is being developed in collaboration with the [Neurosim Lab](http://neurosimlab.org/). See +the [Wiki](https://github.com/MetaCell/NetPyNE-UI/wiki) for more info! \ No newline at end of file From 16e6ed523d86026f14a8b7e97edb75cf28df25e9 Mon Sep 17 00:00:00 2001 From: Lucas Rebscher Date: Thu, 14 Jan 2021 10:45:00 +0100 Subject: [PATCH 005/961] #190 Add simulation module --- netpyne_ui/netpyne_geppetto.py | 96 ++++++++++++++-------------------- netpyne_ui/simulations.py | 39 ++++++++++++++ 2 files changed, 78 insertions(+), 57 deletions(-) create mode 100644 netpyne_ui/simulations.py diff --git a/netpyne_ui/netpyne_geppetto.py b/netpyne_ui/netpyne_geppetto.py index 7bfa38a9..ef4708ed 100644 --- a/netpyne_ui/netpyne_geppetto.py +++ b/netpyne_ui/netpyne_geppetto.py @@ -16,6 +16,8 @@ from netpyne.specs.utils import validateFunction from netpyne.conversion.neuronPyHoc import mechVarList from netpyne.metadata import metadata + +from netpyne_ui import simulations from netpyne_ui.netpyne_model_interpreter import NetPyNEModelInterpreter from pygeppetto.model.model_serializer import GeppettoModelSerializer import matplotlib.pyplot as plt @@ -106,64 +108,51 @@ def simulateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): if args['parallelSimulation']: + self._run_parallel(args) + else: + self._run_in_single_core(args) + return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) + except Exception: + message = "Error while simulating the NetPyNE model" + logging.exception(message) + return utils.getJSONError(message, sys.exc_info()) - # TODO: need to understand when to set NRN_PYLIB - # it's required for pure Python installation, but breaks when using Conda - if "CONDA_VERSION" not in os.environ: - # nrniv needs NRN_PYLIB to be set to python executable path (MacOS, Python 3.7.6) - os.environ["NRN_PYLIB"] = sys.executable + def _run_parallel(self, args): + if args.get('usePrevInst', False): + sim.cfg.saveJson = True + oldName = sim.cfg.filename + sim.cfg.filename = 'model_output' - logging.debug('Running parallel simulation') + # workaround for issue with empty LFP dict when calling saveData() + del sim.allSimData['LFP'] - if args.get('usePrevInst', False): - sim.cfg.saveJson = True - oldName = sim.cfg.filename - sim.cfg.filename = 'model_output' + sim.saveData() + sim.cfg.filename = oldName + template = os.path.join(os.path.dirname(__file__), 'template2.py') + else: + self.netParams.save("netParams.json") + self.simConfig.saveJson = True + self.simConfig.save("simParams.json") + template = os.path.join(os.path.dirname(__file__), 'template.py') - # workaround for issue with empty LFP dict when calling saveData() - del sim.allSimData['LFP'] + # TODO: Instead of copying here files depending on usePrevInst, we can simply parameterize the python script + copyfile(template, './init.py') - sim.saveData() - sim.cfg.filename = oldName - template = os.path.join(os.path.dirname(__file__), 'template2.py') - else: - self.netParams.save("netParams.json") - self.simConfig.saveJson = True - self.simConfig.save("simParams.json") - template = os.path.join(os.path.dirname(__file__), 'template.py') - - copyfile(template, './init.py') - - cores = str(args.get("cores", "1")) - cp = subprocess.run( - ["mpiexec", "-n", cores, "nrniv", "-python", "-mpi", "init.py"], - capture_output=True - ) - logging.info(cp.stdout.decode() + cp.stderr.decode()) - if cp.returncode != 0: - return utils.getJSONError( - "Error while simulating the NetPyNE model", - cp.stderr.decode() - ) - - sim.load('model_output.json') - self.geppetto_model = self.model_interpreter.getGeppettoModel(sim) + cores = str(args.get("cores", "1")) + error = simulations.run(parallel=True, cores=cores) + if error: + return utils.getJSONError("Error while simulating the NetPyNE model", error) - else: - # single cpu computation - if not args.get('usePrevInst', False): - logging.debug('Instantiating single thread simulation') - netpyne_model = self.instantiateNetPyNEModel() - self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model) + sim.load('model_output.json') + self.geppetto_model = self.model_interpreter.get_geppetto_model(sim) - logging.debug('Running single thread simulation') - self.simulateNetPyNEModel() + def _run_in_single_core(self, args): + if not args.get('usePrevInst', False): + logging.debug('Instantiating single thread simulation') + netpyne_model = self.instantiateNetPyNEModel() + self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model) - return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) - except Exception: - message = "Error while simulating the NetPyNE model" - logging.exception(message) - return utils.getJSONError(message, sys.exc_info()) + simulations.run() def compileModMechFiles(self, compileMod, modFolder): # Create Symbolic link @@ -419,13 +408,6 @@ def instantiateNetPyNEModel(self): return sim - def simulateNetPyNEModel(self): - with redirect_stdout(sys.__stdout__): - sim.setupRecording() - sim.simulate() - sim.saveData() - return sim - def doIhaveInstOrSimData(self): # return [bool, bool] telling if we have an instance and simulated data with redirect_stdout(sys.__stdout__): out = [False, False] diff --git a/netpyne_ui/simulations.py b/netpyne_ui/simulations.py new file mode 100644 index 00000000..be09b45e --- /dev/null +++ b/netpyne_ui/simulations.py @@ -0,0 +1,39 @@ +import logging +import subprocess +import os +import sys + +from netpyne import sim + + +def run(parallel=False, cores=None): + if parallel: + return _run_with_mpi(cores) + else: + return _run_in_same_process() + + +def _run_with_mpi(cores): + logging.debug('Running parallel simulation') + + # TODO: need to understand when to set NRN_PYLIB + # it's required for pure Python installation, but breaks when using Conda + if "CONDA_VERSION" not in os.environ: + # nrniv needs NRN_PYLIB to be set to python executable path (MacOS, Python 3.7.6) + os.environ["NRN_PYLIB"] = sys.executable + + completed_process = subprocess.run( + ["mpiexec", "-n", cores, "nrniv", "-python", "-mpi", "init.py"], + capture_output=True + ) + logging.info(completed_process.stdout.decode() + completed_process.stderr.decode()) + + return None if completed_process == 0 else completed_process.stderr.decode() + + +def _run_in_same_process(): + logging.debug('Running single core simulation') + + sim.setupRecording() + sim.simulate() + sim.saveData() From 3adf018128e54963772714d43befe677832aee4a Mon Sep 17 00:00:00 2001 From: Lucas Rebscher Date: Fri, 15 Jan 2021 12:51:52 +0100 Subject: [PATCH 006/961] #193 Implement POC for batch support * Introduce new templates package to store all templates in * Add dummy batch_config to backend and implement batch templates, configuration mapping and running of a batch --- netpyne_ui/__init__.py | 15 ++- netpyne_ui/netpyne_geppetto.py | 97 ++++++++++++++++--- netpyne_ui/simulations.py | 9 +- netpyne_ui/templates/__init__.py | 0 netpyne_ui/templates/batch.py | 40 ++++++++ netpyne_ui/templates/batch_cfg.py | 5 + netpyne_ui/templates/batch_netParams.py | 19 ++++ netpyne_ui/templates/batch_run_single.py | 7 ++ netpyne_ui/{template.py => templates/run.py} | 0 .../run_instantiated_net.py} | 0 .../configuration/NetPyNESimConfig.js | 12 +-- webapp/components/general/NetPyNEField.js | 2 +- 12 files changed, 184 insertions(+), 22 deletions(-) create mode 100644 netpyne_ui/templates/__init__.py create mode 100644 netpyne_ui/templates/batch.py create mode 100644 netpyne_ui/templates/batch_cfg.py create mode 100644 netpyne_ui/templates/batch_netParams.py create mode 100644 netpyne_ui/templates/batch_run_single.py rename netpyne_ui/{template.py => templates/run.py} (100%) rename netpyne_ui/{template2.py => templates/run_instantiated_net.py} (100%) diff --git a/netpyne_ui/__init__.py b/netpyne_ui/__init__.py index cb0bf72e..b50e723f 100644 --- a/netpyne_ui/__init__.py +++ b/netpyne_ui/__init__.py @@ -1,6 +1,17 @@ import logging -from jupyter_geppetto.webapi import RouteManager +import sys +from jupyter_geppetto.webapi import RouteManager from netpyne_ui import api -RouteManager.add_controller(api.NetPyNEController) \ No newline at end of file +RouteManager.add_controller(api.NetPyNEController) + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + +logging.info("Merry Christmas") +logger = logging.getLogger(__name__) +stream_handler = logging.StreamHandler(stream=sys.stdout) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(message)s') +stream_handler.setFormatter(formatter) +logger.addHandler(stream_handler) +logger.info("netpyne-ui logger set up") diff --git a/netpyne_ui/netpyne_geppetto.py b/netpyne_ui/netpyne_geppetto.py index ef4708ed..9811172d 100644 --- a/netpyne_ui/netpyne_geppetto.py +++ b/netpyne_ui/netpyne_geppetto.py @@ -11,23 +11,27 @@ import subprocess import logging import re +import pickle + +import matplotlib.pyplot as plt +import numpy as np +import neuron from netpyne import specs, sim, analysis from netpyne.specs.utils import validateFunction from netpyne.conversion.neuronPyHoc import mechVarList from netpyne.metadata import metadata -from netpyne_ui import simulations -from netpyne_ui.netpyne_model_interpreter import NetPyNEModelInterpreter from pygeppetto.model.model_serializer import GeppettoModelSerializer -import matplotlib.pyplot as plt from pygeppetto import ui -import numpy as np -import neuron -from shutil import copyfile + +from shutil import copyfile, move from jupyter_geppetto import jupyter_geppetto, synchronization, utils from contextlib import redirect_stdout + from netpyne_ui.constants import NETPYNE_WORKDIR_PATH +from netpyne_ui import simulations +from netpyne_ui.netpyne_model_interpreter import NetPyNEModelInterpreter os.chdir(NETPYNE_WORKDIR_PATH) @@ -39,6 +43,33 @@ def __init__(self): self.netParams = specs.NetParams() self.simConfig = specs.SimConfig() + + # TODO: need new methods to store field data in these dictionaries + self.batch_config = { + "enabled": True, + "params": [ + { + # set in cfg.py to cfg.paramX = defaultValue of field + "label": "weight", + # exact path to parameter that will be overwritten + "mapsTo": "netParams.connParams['E->E']['weight']", + # or 'range' with min, max, steps fields + "type": "list", + "values": [1, 2, 3, 4], + } + ], + # possible values: grid|list|evol|asd|optuna + "method": "grid", + "name": "my_batch", + "seed": None, + "saveFolder": "batches" + } + + self.run_config = { + "asynchronous": True, + "cores": 1, + } + synchronization.startSynchronization(self.__dict__) logging.debug("Initializing the original model") @@ -107,16 +138,56 @@ def instantiateNetPyNEModelInGeppetto(self, args): def simulateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): - if args['parallelSimulation']: - self._run_parallel(args) + if self.batch_config["enabled"]: + self._run_as_batch() else: - self._run_in_single_core(args) + if args['parallelSimulation']: + self._run_parallel(args) + else: + self._run_in_single_core(args) + return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model)) except Exception: message = "Error while simulating the NetPyNE model" logging.exception(message) return utils.getJSONError(message, sys.exc_info()) + def _run_as_batch(self): + """Runs the configured simulation as a batch of variations of this simulation. + + * Store netParams and cfg as pkl files + * Prepare py templates for netParams.py, cfg.py, batch.py, run.py + * Copy all to workspace + * Submit batch simulation + """ + # Can store param mapping in simConfig + self.simConfig.mapping = self.batch_config["params"] + + # Configuration of batch.py in json format + batch_config = os.path.join(os.path.dirname(__file__), "templates", "batchConfig.json") + json.dump(self.batch_config, open(batch_config, 'w')) + move(batch_config, './batchConfig.json') + + # Pickle files of netParams and cfg dicts + net_params_pkl = os.path.join(os.path.dirname(__file__), "templates", 'netParams.pkl') + cfg_pkl = os.path.join(os.path.dirname(__file__), "templates", 'cfg.pkl') + pickle.dump(self.netParams, open(net_params_pkl, "wb")) + pickle.dump(self.simConfig, open(cfg_pkl, "wb")) + move(net_params_pkl, './netParams.pkl') + move(cfg_pkl, './cfg.pkl') + + # Python template files + template_single_run = os.path.join(os.path.dirname(__file__), "templates", 'batch_run_single.py') + template_batch = os.path.join(os.path.dirname(__file__), "templates", 'batch.py') + cfg = os.path.join(os.path.dirname(__file__), "templates", 'batch_cfg.py') + net_params = os.path.join(os.path.dirname(__file__), "templates", 'batch_netParams.py') + copyfile(template_single_run, './run.py') + copyfile(template_batch, './init.py') + copyfile(cfg, './cfg.py') + copyfile(net_params, './netParams.py') + + simulations.run_in_subprocess("init.py") + def _run_parallel(self, args): if args.get('usePrevInst', False): sim.cfg.saveJson = True @@ -128,14 +199,16 @@ def _run_parallel(self, args): sim.saveData() sim.cfg.filename = oldName - template = os.path.join(os.path.dirname(__file__), 'template2.py') + template_name = "run_instantiated_net.py" else: + # Saved in netpyne_workspace self.netParams.save("netParams.json") self.simConfig.saveJson = True self.simConfig.save("simParams.json") - template = os.path.join(os.path.dirname(__file__), 'template.py') + template_name = "run.py" # TODO: Instead of copying here files depending on usePrevInst, we can simply parameterize the python script + template = os.path.join(os.path.dirname(__file__), "templates", template_name) copyfile(template, './init.py') cores = str(args.get("cores", "1")) @@ -667,7 +740,7 @@ def header(title, spacer='-'): return f.read() except Exception: - message = "Error while importing the NetPyNE model" + message = "Error while exporting NetPyNE model to python" logging.exception(message) return utils.getJSONError(message, sys.exc_info()) diff --git a/netpyne_ui/simulations.py b/netpyne_ui/simulations.py index be09b45e..6a505443 100644 --- a/netpyne_ui/simulations.py +++ b/netpyne_ui/simulations.py @@ -13,6 +13,14 @@ def run(parallel=False, cores=None): return _run_in_same_process() +def run_in_subprocess(script): + completed_process = subprocess.run(["python", "./" + script]) + if completed_process.returncode == 0: + logging.info("Batch finished with success") + else: + logging.error(f"Batch run failed ...") + + def _run_with_mpi(cores): logging.debug('Running parallel simulation') @@ -26,7 +34,6 @@ def _run_with_mpi(cores): ["mpiexec", "-n", cores, "nrniv", "-python", "-mpi", "init.py"], capture_output=True ) - logging.info(completed_process.stdout.decode() + completed_process.stderr.decode()) return None if completed_process == 0 else completed_process.stderr.decode() diff --git a/netpyne_ui/templates/__init__.py b/netpyne_ui/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/netpyne_ui/templates/batch.py b/netpyne_ui/templates/batch.py new file mode 100644 index 00000000..213f4b09 --- /dev/null +++ b/netpyne_ui/templates/batch.py @@ -0,0 +1,40 @@ +import json + +from netpyne import specs +from netpyne.batch import Batch + +with open("batchConfig.json", "r") as f: + batch_config = json.load(f) + print(batch_config) + +params = specs.ODict() +for param in batch_config["params"]: + if param["type"] == "list": + params[param["label"]] = param['values'] + else: + print("Not supported yet...") + +batch = Batch( + cfgFile="cfg.py", + netParamsFile="netParams.py", + params=params, + seed=None +) + +# Label will be subfolder of saveFolder +batch.batchLabel = batch_config.get("name", "batch_template_run") + +# Have to set the saveFolder, default goes to root folder which is not always allowed by OS +batch.saveFolder = batch_config.get("saveFolder", "./batch") + +# For now, we only support grid|list +batch.method = batch_config.get("method", "grid") + +# for now, we only support mpi_direct or bulletinboard +batch.runCfg = { + 'type': 'mpi_direct', + 'script': 'run.py', + 'skip': True +} + +batch.run() diff --git a/netpyne_ui/templates/batch_cfg.py b/netpyne_ui/templates/batch_cfg.py new file mode 100644 index 00000000..e0110ec5 --- /dev/null +++ b/netpyne_ui/templates/batch_cfg.py @@ -0,0 +1,5 @@ +import pickle + +with open("cfg.pkl", "rb") as f: + simConfig = pickle.load(f) + cfg = simConfig diff --git a/netpyne_ui/templates/batch_netParams.py b/netpyne_ui/templates/batch_netParams.py new file mode 100644 index 00000000..9d9a6bbf --- /dev/null +++ b/netpyne_ui/templates/batch_netParams.py @@ -0,0 +1,19 @@ +import pickle + +try: + from __main__ import cfg # import SimConfig object with params from parent module +except: + from cfg import cfg # if no simConfig in parent module, import directly from tut8_cfg module + +with open("netParams.pkl", "rb") as f: + netParams = pickle.load(f) + + if getattr(cfg, "mapping", None): + print(cfg.mapping) + + for param in cfg.mapping: + # Value of current combination is stored in __main__.cfg + value = getattr(cfg, param["label"]) + + # Update value in netParams + exec(f"{param['maps_to']} = {value}") diff --git a/netpyne_ui/templates/batch_run_single.py b/netpyne_ui/templates/batch_run_single.py new file mode 100644 index 00000000..0de55679 --- /dev/null +++ b/netpyne_ui/templates/batch_run_single.py @@ -0,0 +1,7 @@ +from netpyne import sim + +# read cfg and netParams from command line arguments if available; otherwise use default +simConfig, netParams = sim.readCmdLineArgs(simConfigDefault='tut8_cfg.py', netParamsDefault='tut8_netParams.py') + +# Create network and run simulation +sim.createSimulateAnalyze(netParams=netParams, simConfig=simConfig) \ No newline at end of file diff --git a/netpyne_ui/template.py b/netpyne_ui/templates/run.py similarity index 100% rename from netpyne_ui/template.py rename to netpyne_ui/templates/run.py diff --git a/netpyne_ui/template2.py b/netpyne_ui/templates/run_instantiated_net.py similarity index 100% rename from netpyne_ui/template2.py rename to netpyne_ui/templates/run_instantiated_net.py diff --git a/webapp/components/definition/configuration/NetPyNESimConfig.js b/webapp/components/definition/configuration/NetPyNESimConfig.js index 8e534ef9..b23e6958 100644 --- a/webapp/components/definition/configuration/NetPyNESimConfig.js +++ b/webapp/components/definition/configuration/NetPyNESimConfig.js @@ -29,10 +29,10 @@ class NetPyNESimConfig extends React.Component { this.setState({ selectedIndex: index, sectionId: sectionId }); render () { - var contentLeft =

; - var contentRight =
; + let contentLeft =
; + let contentRight =
; const { classes } = this.props; - if (this.state.sectionId == "General") { + if (this.state.sectionId === "General") { contentLeft = (
@@ -170,7 +170,7 @@ class NetPyNESimConfig extends React.Component {
); - } else if (this.state.sectionId == "SaveConfiguration") { + } else if (this.state.sectionId === "SaveConfiguration") { contentLeft = (
@@ -313,7 +313,7 @@ class NetPyNESimConfig extends React.Component {
); } else if (this.state.sectionId == "netParams") { - var contentLeft = ( + contentLeft = (
- +
); diff --git a/webapp/components/general/NetPyNEField.js b/webapp/components/general/NetPyNEField.js index 738c5784..96d6a0e3 100644 --- a/webapp/components/general/NetPyNEField.js +++ b/webapp/components/general/NetPyNEField.js @@ -179,7 +179,7 @@ export default class NetPyNEField extends Component { {(help != undefined && help != "") ? + leaveTouchDelay={0} disableTouchListener={true} disableFocusListener={true}> {childWithProp} From 823f22b15b2e4590f3d451ca23533cf049d1fa10 Mon Sep 17 00:00:00 2001 From: Lucas Rebscher Date: Fri, 15 Jan 2021 16:12:30 +0100 Subject: [PATCH 007/961] #193 Add batch flag & cleanup frontend components --- netpyne_ui/netpyne_geppetto.py | 30 ++++- .../configuration/NetPyNESimConfig.js | 100 +++++++++------ webapp/components/general/Checkbox.js | 2 +- webapp/components/general/NetPyNEField.js | 118 +++++------------- .../general/PythonControlledCapability.js | 24 ++-- 5 files changed, 136 insertions(+), 138 deletions(-) diff --git a/netpyne_ui/netpyne_geppetto.py b/netpyne_ui/netpyne_geppetto.py index 9811172d..8ff72041 100644 --- a/netpyne_ui/netpyne_geppetto.py +++ b/netpyne_ui/netpyne_geppetto.py @@ -44,7 +44,9 @@ def __init__(self): self.netParams = specs.NetParams() self.simConfig = specs.SimConfig() - # TODO: need new methods to store field data in these dictionaries + # default values + self.simConfig.batch = False + self.batch_config = { "enabled": True, "params": [ @@ -99,6 +101,31 @@ def getData(self): "type": "float" } + metadata['simConfig']['children']['batch'] = { + "label": "Batch enabled", + "help": "Activates batch", + "suggestions": "", + "hintText": "", + "type": "bool" + } + + metadata['batch_config'] = { + "label": "Batch configuration", + "help": "", + "suggestions": "", + "hintText": "", + 'children': { + 'enabled': { + "label": "Batch enabled", + "help": "Activates batch", + "suggestions": "", + "hintText": "", + "type": "bool" + }, + # TODO: add here metadata for remaining fields + } + } + self.netParams.cellsVisualizationSpacingMultiplierX = 1 self.netParams.cellsVisualizationSpacingMultiplierY = 1 self.netParams.cellsVisualizationSpacingMultiplierZ = 1 @@ -107,6 +134,7 @@ def getData(self): "metadata": metadata, "netParams": self.netParams.todict(), "simConfig": self.simConfig.todict(), + "batch_config": self.batch_config, "isDocker": os.path.isfile('/.dockerenv'), "currentFolder": os.getcwd(), "tuts": self.find_tutorials() diff --git a/webapp/components/definition/configuration/NetPyNESimConfig.js b/webapp/components/definition/configuration/NetPyNESimConfig.js index b23e6958..e7964d85 100644 --- a/webapp/components/definition/configuration/NetPyNESimConfig.js +++ b/webapp/components/definition/configuration/NetPyNESimConfig.js @@ -60,25 +60,25 @@ class NetPyNESimConfig extends React.Component { - + - + - + - +
); @@ -88,85 +88,85 @@ class NetPyNESimConfig extends React.Component { id="simConfig.createNEURONObj" className={"netpyneCheckbox"} > - + - + - + - + - + - + - + - + - + - + - + - + - +
); @@ -200,7 +200,7 @@ class NetPyNESimConfig extends React.Component { - + @@ -215,36 +215,36 @@ class NetPyNESimConfig extends React.Component { contentRight = (
- + - + - + - + - + - + {/* // TODO: can this be removed? @@ -265,23 +265,23 @@ class NetPyNESimConfig extends React.Component { */} - +
); - } else if (this.state.sectionId == "Record") { + } else if (this.state.sectionId === "Record") { contentLeft = (
- + - + - + @@ -300,7 +300,7 @@ class NetPyNESimConfig extends React.Component { className={"netpyneCheckbox"} style={{ marginTop: 25 }} > - + - +
); - } else if (this.state.sectionId == "netParams") { + } else if (this.state.sectionId === "netParams") { contentLeft = (
@@ -398,7 +398,7 @@ class NetPyNESimConfig extends React.Component { - + @@ -432,6 +432,23 @@ class NetPyNESimConfig extends React.Component {
); + } else if (this.state.sectionId === "Batch") { + contentLeft = ( +
+ + + + + + +
+ ) } return (
@@ -444,33 +461,40 @@ class NetPyNESimConfig extends React.Component { id={"configGeneral"} key={"General"} label={"General"} - icon={} + icon={} onClick={() => this.select(0, "General")} /> } + icon={} onClick={() => this.select(1, "Record")} /> } + icon={} onClick={() => this.select(2, "SaveConfiguration")} /> } + icon={} onClick={() => this.select(3, "netParams")} /> + } + onClick={() => this.select(4, "Batch")} + /> -
+
{contentLeft} {contentRight} diff --git a/webapp/components/general/Checkbox.js b/webapp/components/general/Checkbox.js index ef3ed2b3..8a96fe4d 100644 --- a/webapp/components/general/Checkbox.js +++ b/webapp/components/general/Checkbox.js @@ -4,7 +4,7 @@ import FormGroup from '@material-ui/core/FormGroup'; import MuiCheckbox from '@material-ui/core/Checkbox'; import FormControlLabel from '@material-ui/core/FormControlLabel'; -import { bgLight } from '../../theme' +import { bgLight } from 'root/theme' export default class Checkbox extends Component { render () { diff --git a/webapp/components/general/NetPyNEField.js b/webapp/components/general/NetPyNEField.js index 96d6a0e3..e5fdc7bc 100644 --- a/webapp/components/general/NetPyNEField.js +++ b/webapp/components/general/NetPyNEField.js @@ -1,35 +1,21 @@ import React, { Component } from "react"; -import Dialog from "@material-ui/core/Dialog"; -import Button from "@material-ui/core/Button"; import MenuItem from "@material-ui/core/MenuItem"; -import Utils from "../../Utils"; - -import DialogActions from "@material-ui/core/DialogActions"; -import DialogContent from "@material-ui/core/DialogContent"; -import DialogContentText from "@material-ui/core/DialogContentText"; -import DialogTitle from "@material-ui/core/DialogTitle"; -import { Tooltip } from "netpyne/components"; import Grid from "@material-ui/core/Grid"; import Box from "@material-ui/core/Box"; +import Utils from "../../Utils"; +import { Tooltip } from "netpyne/components"; export default class NetPyNEField extends Component { + constructor (props) { super(props); this.state = { openHelp: false }; } - handleOpenHelp = help => { - this.setState({ openHelp: true, helpText: help }); - }; - - handleCloseHelp = () => { - this.setState({ openHelp: false }); - }; - setErrorMessage (value) { return new Promise((resolve, reject) => { - if (this.realType == "func") { - if (value != "" && value != undefined) { + if (this.realType === "func") { + if (value !== "" && value !== undefined) { Utils.evalPythonMessage("netpyne_geppetto.validateFunction", [ value, ]).then(response => { @@ -42,7 +28,7 @@ export default class NetPyNEField extends Component { } else { resolve({ errorMsg: "" }); } - } else if (this.realType == "float") { + } else if (this.realType === "float") { if (isNaN(value)) { resolve({ errorMsg: "Only float allowed" }); } else { @@ -53,8 +39,8 @@ export default class NetPyNEField extends Component { } prePythonSyncProcessing (value) { - if (value == "") { - if (this.default != undefined) { + if (value === "") { + if (this.default !== undefined) { return this.default; } else if ( !this.model.split(".")[0].startsWith("simConfig") @@ -67,45 +53,11 @@ export default class NetPyNEField extends Component { } render () { - var help = Utils.getMetadataField(this.props.id, "help"); - if (help != undefined && help != "") { - var helpComponent = ( -
- -
-