From 6e0caf760f1c507c8eb52e512b0b8b5b3db7be81 Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Mon, 7 Apr 2014 19:12:30 +1000 Subject: [PATCH 01/20] Initial pythonic API teaser --- pymatbridge/matlab/matlabserver.m | 110 +++- pymatbridge/matlab/util/pymat_eval.m | 76 --- pymatbridge/matlab/util/pymat_feval.m | 42 -- pymatbridge/matlab/util/pymat_get_variable.m | 32 - pymatbridge/matlab/util/pymat_set_variable.m | 7 - pymatbridge/pymatbridge.py | 584 ++++++++++++++----- 6 files changed, 513 insertions(+), 338 deletions(-) delete mode 100644 pymatbridge/matlab/util/pymat_eval.m delete mode 100644 pymatbridge/matlab/util/pymat_feval.m delete mode 100644 pymatbridge/matlab/util/pymat_get_variable.m delete mode 100644 pymatbridge/matlab/util/pymat_set_variable.m diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 55b223b..279f54d 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -1,41 +1,95 @@ function matlabserver(socket_address) -% This function takes a socket address as input and initiates a ZMQ session -% over the socket. I then enters the listen-respond mode until it gets an -% "exit" command +% MATLABSERVER Run a Matlab server to handle requests from Python over ZMQ +% +% MATLABSERVER(SOCKET_ADDRESS) initiates a ZMQ session over the provided +% SOCKET_ADDRESS. Once started, it executes client requests and returns +% the response. +% +% The recognized requests are: +% 'ping': Elicit a response from the server +% 'exit': Request the server to shutdown +% 'call': Call a Matlab function with the provdided arguments -json.startup -messenger('init', socket_address); + json.startup + messenger('init', socket_address); -while(1) - msg_in = messenger('listen'); - req = json.load(msg_in); + while true + % don't let any errors escape (and crash the server) + try + msg_in = messenger('listen'); + req = json.load(msg_in); - switch(req.cmd) - case {'connect'} - messenger('respond', 'connected'); + switch(req.cmd) + case {'ping'} + messenger('respond', 'pong'); - case {'exit'} - messenger('exit'); - clear mex; - break; + case {'exit'} + messenger('exit'); + clear mex; + break; - case {'run_function'} - fhandle = str2func('pymat_feval'); - resp = feval(fhandle, req); - messenger('respond', resp); + case {'call'} + resp = call(req); + json_resp = json.dump(resp); + messenger('respond', json_resp); - case {'run_code'} - fhandle = str2func('pymat_eval'); - resp = feval(fhandle, req); - messenger('respond', resp); + otherwise + throw(MException('MATLAB:matlabserver', ['Unrecognized command ' req.cmd])) + end - case {'get_var'} - fhandle = str2func('pymat_get_variable'); - resp = feval(fhandle, req); - messenger('respond', resp); + catch exception + % format the exception and pass it back to the client + resp.success = false; + resp.result = exception.identifier; + resp.message = exception.message; + json_resp = json.dump(resp); + messenger('respond', json_resp); + end + end +end + + +function resp = call(req) +% CALL Call a Matlab function +% +% RESPONSE = CALL(REQUEST) calls Matlab's FEVAL function, intelligently +% handling the number of input and output arguments so that the argument +% spec is satisfied. +% +% The REQUEST is a struct with three fields: +% 'func': The name of the function to execute +% 'args': A cell array of args to expand into the function arguments +% 'nout': The number of output arguments requested + + % function of no arguments + if ~isfield(req, 'args') + req.args = {} + end + + % determine the number of output arguments + % TODO: What should the default behaviour be? + func = str2func(req.func); + nout = req.nout; + if isempty(nout) + try + nout = min(abs(nargout(func)), 1); + catch + nout = 1; + end + end + + % call the function, taking care of broadcasting outputs + switch nout + case 0 + func(req.args{:}); + case 1 + resp.result = func(req.args{:}); otherwise - messenger('respond', 'i dont know what you want'); + [resp.result{1:nout}] = func(req.args{:}); end + % build the response + resp.success = true; + resp.message = 'Successfully completed request'; end diff --git a/pymatbridge/matlab/util/pymat_eval.m b/pymatbridge/matlab/util/pymat_eval.m deleted file mode 100644 index b071cb8..0000000 --- a/pymatbridge/matlab/util/pymat_eval.m +++ /dev/null @@ -1,76 +0,0 @@ -function json_response = web_eval(req); -%WEB_EVAL: Returns a json object of the result of calling the function -% -% json_response = WEB_EVAL(headers); -% json_response = WEB_EVAL(headers, config); -% -% This allows you to run any matlab code. To be used with webserver.m. -% HTTP POST to /web_eval.m with the following parameters: -% code: a string which contains the code to be run in the matlab session -% -% Should return a json object containing the result -% -% Based on Max Jaderberg's web_feval - -response.success = 'false'; -field_names = fieldnames(req); - -response.content = ''; - -code_check = false; -if size(field_names) - if isfield(req, 'code') - code_check = true; - end -end - -if ~code_check - response.message = 'No code provided as POST parameter'; - json_response = json.dump(response); - return; -end - -code = req.code; - -try - % tempname is less likely to get bonked by another process. - diary_file = [tempname() '_diary.txt']; - diary(diary_file); - evalin('base', code); - diary('off'); - - datadir = fullfile(tempdir(),'MatlabData'); - response.content.datadir = [datadir, filesep()]; - if ~exist(datadir, 'dir') - mkdir(datadir); - end - - fig_files = make_figs(datadir); - - response.success = 'true'; - response.content.figures = fig_files; - - % this will not work on Windows: - %[ignore_status, stdout] = system(['cat ' diary_file]); - % cf. http://rosettacode.org/wiki/Read_entire_file#MATLAB_.2F_Octave - FID = fopen(diary_file,'r'); - if (FID > 0) - [stdout,count] = fread(FID, [1,inf], 'uint8=>char'); - fclose(FID); - response.content.stdout = stdout; - else - response.success = 'false'; - response.content.stdout = sprintf('could not open %s for read',diary_file); - end - delete(diary_file) -catch ME - diary('off'); - response.success = 'false'; - response.content.stdout = ME.message; -end - -response.content.code = code; - -json_response = json.dump(response); - -end %function diff --git a/pymatbridge/matlab/util/pymat_feval.m b/pymatbridge/matlab/util/pymat_feval.m deleted file mode 100644 index 2fd3a48..0000000 --- a/pymatbridge/matlab/util/pymat_feval.m +++ /dev/null @@ -1,42 +0,0 @@ -% Max Jaderberg 2011 - -function json_response = matlab_feval(req) - - response.success = 'false'; - field_names = fieldnames(req); - - response.result = ''; - - func_path_check = false; - arguments_check = false; - if size(field_names) - if isfield(req, 'func_path') - func_path_check = true; - end - if isfield(req, 'func_args') - arguments_check = true; - end - end - - if ~func_path_check - response.message = 'No function given as func_path POST parameter'; - json_response = json.dump(response); - return - end - - func_path = req.func_path; - if arguments_check - arguments = req.func_args; - else - arguments = ''; - end - - response.result = run_dot_m(func_path, arguments); - response.success = 'true'; - response.message = 'Successfully completed request'; - - json_response = json.dump(response); - - return - -end diff --git a/pymatbridge/matlab/util/pymat_get_variable.m b/pymatbridge/matlab/util/pymat_get_variable.m deleted file mode 100644 index c37fa90..0000000 --- a/pymatbridge/matlab/util/pymat_get_variable.m +++ /dev/null @@ -1,32 +0,0 @@ -function json_response = pymat_get_variable(req) -% Reach into the current namespace get a variable in json format that can -% be returned as part of a response - -response.success = 'false'; - -field_names = fieldnames(req); - -response.content = ''; - -varname_check = false; -if size(field_names) - if isfield(req, 'varname') - varname_check = true; - end -end - -if ~varname_check - response.message = 'No variable name provided as input argument'; - json_response = json.dump(response); - return -end - - -varname = req.varname; - -response.var = evalin('base', varname); - -json_response = json.dump(response); - -return -end diff --git a/pymatbridge/matlab/util/pymat_set_variable.m b/pymatbridge/matlab/util/pymat_set_variable.m deleted file mode 100644 index e3936f4..0000000 --- a/pymatbridge/matlab/util/pymat_set_variable.m +++ /dev/null @@ -1,7 +0,0 @@ -function res = pymat_set_variable(args) -% Setup a variable in Matlab workspace - - assignin('base', args.name, args.value); - res = 1; - -end %function diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 9e9a347..91ba717 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -1,231 +1,509 @@ -""" -pymatbridge -=========== - -This is a module for communicating and running +"""Main module for running and communicating with Matlab subprocess -Part of Python-MATLAB-bridge, Max Jaderberg 2012 +pymatbridge.py provides Matlab, the main class used to communicate +with the Matlab executable. Each instance starts and manages its +own Matlab subprocess. -This is a modified version using ZMQ, Haoxing Zhang Jan.2014 """ - +from __future__ import print_function +import collections +import functools +import json import numpy as np -import os, time -import zmq -import subprocess +import os import platform +import subprocess import sys +import time +import types +import weakref +import zmq +try: + # Python 2 + basestring + DEVNULL = open(os.devnull, 'w') +except: + # Python 3 + basestring = str + DEVNULL = subprocess.DEVNULL + + +# ---------------------------------------------------------------------------- +# HELPERS +# ---------------------------------------------------------------------------- +def chain(*iterables): + """Yield elements from each iterable in order + + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. Unlike itertools.chain, strings are not treated + as iterable, and thus not expanded into characters: + chain([1, 2, 3, 4], 'string') --> 1, 2, 3, 4, 'string' + + Returns: + generator: A generator which yields items from the iterables -import json + """ + for iterable in iterables: + if not isinstance(iterable, collections.Iterable) or isinstance(iterable, basestring): + yield iterable + else: + for item in iterable: + yield item -# JSON encoder extension to handle complex numbers -class ComplexEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, complex): - return {'real':obj.real, 'imag':obj.imag} - # Handle the default case - return json.JSONEncoder.default(self, obj) -# JSON decoder for complex numbers -def as_complex(dct): - if 'real' in dct and 'imag' in dct: - return complex(dct['real'], dct['imag']) - return dct +class AttributeDict(dict): + """A dictionary with attribute-like access + Values within an AttributeDict can be accessed either via + d[key] or d.key. + See: http://stackoverflow.com/a/14620633 -MATLAB_FOLDER = '%s/matlab' % os.path.realpath(os.path.dirname(__file__)) + """ + def __init__(self, *args, **kwargs): + super(AttributeDict, self).__init__(*args, **kwargs) + self.__dict__ = self -# Start a Matlab server and bind it to a ZMQ socket(TCP/IPC) -def _run_matlab_server(matlab_bin, matlab_socket_addr, matlab_log, matlab_id, matlab_startup_options): - command = matlab_bin - command += ' %s ' % matlab_startup_options - command += ' -r "' - command += "addpath(genpath(" - command += "'%s'" % MATLAB_FOLDER - command += ')), matlabserver(\'%s\'),exit"' % matlab_socket_addr - if matlab_log: - command += ' -logfile ./pymatbridge/logs/matlablog_%s.txt > ./pymatbridge/logs/bashlog_%s.txt' % (matlab_id, matlab_id) +# ---------------------------------------------------------------------------- +# JSON EXTENSION +# ---------------------------------------------------------------------------- +class MatlabEncoder(json.JSONEncoder): + """A JSON extension for encoding numpy arrays to Matlab format - subprocess.Popen(command, shell = True, stdin=subprocess.PIPE) + Numpy arrays are converted to nested lists. Complex numbers + (either standalone or scalars within an array) are converted to JSON + objects. + complex --> {'real': complex.real, 'imag': complex.imag} + """ + def default(self, obj): + if isinstance(obj, np.ndarray): + ordering = range(0, obj.ndim) + ordering[0:2] = ordering[1::-1] + return np.transpose(obj, axes=ordering[::-1]).tolist() + if isinstance(obj, complex): + return {'real':obj.real, 'imag':obj.imag} + # Handle the default case + return super(MatlabEncoder, self).default(obj) - return True +class MatlabDecoder(json.JSONDecoder): + """A JSON extension for decoding Matlab arrays into numpy arrays -class Matlab(object): + The default JSON decoder is called first, then elements within the + resulting object tree are greedily marshalled to numpy arrays. If this + fails, the function recurses into """ - A class for communicating with a matlab session - """ - + def __init__(self, encoding='UTF-8', **kwargs): + # register the complex object decoder with the super class + kwargs['object_hook'] = self.decode_complex + # allowable scalar types we can coerce to (not strings or objects) + super(MatlabDecoder, self).__init__(encoding=encoding, **kwargs) + + def decode(self, s): + # decode the string using the default decoder first + tree = super(MatlabDecoder, self).decode(s) + # recursively attempt to build numpy arrays (top-down) + return self.coerce_to_numpy(tree) + + def decode_complex(self, d): + try: + return complex(d['real'], d['imag']) + except KeyError: + return d + + def coerce_to_numpy(self, tree): + """Greedily attempt to coerce an object into a numeric numpy""" + if isinstance(tree, dict): + return dict((key, self.coerce_to_numpy(val)) for key, val in tree.items()) + if isinstance(tree, list): + array = np.array(tree) + if isinstance(array.dtype.type(), (bool, int, float, complex)): + ordering = range(0, array.ndim) + ordering[-2:] = ordering[:-3:-1] + return np.transpose(array, axes=ordering[::-1]) + else: + return [self.coerce_to_numpy(item) for item in tree] + return tree + + +# ---------------------------------------------------------------------------- +# MATLAB +# ---------------------------------------------------------------------------- +class Matlab(object): def __init__(self, matlab='matlab', socket_addr=None, - id='python-matlab-bridge', log=False, maxtime=60, + id='python-matlab-bridge', log=False, timeout=30, platform=None, startup_options=None): - """ - Initialize this thing. - - Parameters - ---------- - - matlab : str - A string that woul start matlab at the terminal. Per default, this - is set to 'matlab', so that you can alias in your bash setup - - socket_addr : str - A string that represents a valid ZMQ socket address, such as - "ipc:///tmp/pymatbridge", "tcp://127.0.0.1:55555", etc. - - id : str - An identifier for this instance of the pymatbridge - - log : bool - Whether to save a log file in some known location. - - maxtime : float - The maximal time to wait for a response from matlab (optional, - Default is 10 sec) - - platform : string - The OS of the machine on which this is running. Per default this - will be taken from sys.platform. + """Execute functions in a Matlab subprocess via Python + + Matlab provides a pythonic interface for accessing functions in Matlab. + It works by starting a Matlab subprocess and establishing a connection + to it using ZMQ. Function calls are serialized and executed remotely. + + Keyword Args: + matlab (str): A string to the Matlab executable. This defaults + to 'matlab', assuming the executable is on your PATH + socket_addr (str): A string the represents a valid ZMQ socket + address, such as "ipc:///tmp/pymatbridge", "tcp://127.0.0.1:5555" + id (str): An identifier for this instance of the pymatbridge + log (bool): Log status and error messages + timeout: The maximum time to wait for a response from the Matlab + process before timing out (default 30 seconds) + platform (str): The OS of the machine running Matlab. By default + this is determined automatically from sys.platform + startup_options (list): A list of switches that should be passed + to the Matlab subprocess at startup. By default, switches are + passed to disable the graphical session. For a full list of + available switches see: + Windows: http://www.mathworks.com.au/help/matlab/ref/matlabwindows.html + UNIX: http://www.mathworks.com.au/help/matlab/ref/matlabunix.html """ + self.MATLAB_FOLDER = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'matlab') + # Setup internal state variables self.started = False - self.running = False self.matlab = matlab self.socket_addr = socket_addr - self.id = id self.log = log - self.maxtime = maxtime + self.timeout = timeout - if platform is None: - self.platform = sys.platform + # determine the platform-specific options + self.platform = platform if platform else sys.platform + if self.platform == 'win32': + default_socket_addr = "tcp://127.0.0.1:55555" + default_options = ['-automation', '-nofigureWindows'] else: - self.platform = platform + default_socket_addr = "ipc:///tmp/pymatbridge" + default_options = ['-nodesktop', '-nodisplay'] - if self.socket_addr is None: # use the default - self.socket_addr = "tcp://127.0.0.1:55555" if self.platform == "win32" else "ipc:///tmp/pymatbridge" + self.socket_addr = socket_addr if socket_addr else default_socket_addr + self.startup_options = startup_options if startup_options else default_options - if startup_options: - self.startup_options = startup_options - elif self.platform == 'win32': - self.startup_options = ' -automation -noFigureWindows' - else: - self.startup_options = ' -nodesktop -nodisplay' + # initialize the ZMQ socket + self.context = zmq.Context() + self.socket = self.context.socket(zmq.REQ) + + # auto-generate some useful matlab builtins + self.bind_method('exist', unconditionally=True) + self.bind_method('addpath', unconditionally=True) + self.bind_method('eval', unconditionally=True) + self.bind_method('help', unconditionally=True) + self.bind_method('license', unconditionally=True) + self.bind_method('run', unconditionally=True) + self.bind_method('version', unconditionally=True) - self.context = None - self.socket = None + def __del__(self): + """Forcibly cleanup resources + + The user should always call Matlab.stop to gracefully shutdown the Matlab + process before Matlab leaves scope, but in case they don't, attempt + to cleanup so we don't leave an orphaned process lying around. + + """ + self.stop() - # Start server/client session and make the connection def start(self): - # Start the MATLAB server in a new process - print "Starting MATLAB on ZMQ socket %s" % (self.socket_addr) - print "Send 'exit' command to kill the server" - _run_matlab_server(self.matlab, self.socket_addr, self.log, self.id, self.startup_options) + """Start a new Matlab subprocess and attempt to connect to it via ZMQ + + Raises: + RuntimeError: If Matlab is already running, or failed to start + + """ + if self.started: + raise RuntimeError('Matlab is already running') + + # build the command + command = chain( + self.matlab, + self.startup_options, + '-r', + "\"addpath(genpath('%s')),matlabserver('%s'),exit\"" % (self.MATLAB_FOLDER, self.socket_addr) + ) + + command = ' '.join(command) + print('Starting Matlab subprocess', end='') + self.matlab_process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=DEVNULL) # Start the client - self.context = zmq.Context() self.socket = self.context.socket(zmq.REQ) self.socket.connect(self.socket_addr) - self.started = True # Test if connection is established if (self.is_connected()): - print "MATLAB started and connected!" - return True + print('started') else: - print "MATLAB failed to start" - return False + self.started = False + raise RuntimeError('Matlab failed to start') + def stop(self, timeout=1): + """Stop the Matlab subprocess - # Stop the Matlab server - def stop(self): - req = json.dumps(dict(cmd="exit"), cls=ComplexEncoder) - self.socket.send(req) - resp = self.socket.recv_string() + Attempt to gracefully shutdown the Matlab subprocess. If it fails to + stop within the timeout period, terminate it forcefully. - # Matlab should respond with "exit" if successful - if resp == "exit": - print "MATLAB closed" + Args: + timeout: Time in seconds before SIGKILL is sent + """ + if not self.started: + return + + req = json.dumps({'cmd': 'exit'}) + try: + # the user might be stopping Matlab because the socket is in a bad state + self.socket.send(req) + except: + pass + + start_time = time.time() + while time.time() - start_time < timeout: + if self.matlab_process.poll() is not None: + break + time.sleep(0.1) + else: + self.matlab_process.kill() + + # finalize + self.socket.close() self.started = False - return True - # To test if the client can talk to the server + def restart(self): + """Restart the Matlab subprocess if the state becomes bad + + Aliases the following command: + >> matlab.stop() + >> matlab.start() + + """ + self.stop() + self.start() + def is_connected(self): + """Test if the client can talk to the server + + Raises: + RuntimeError: If there is no running Matlab subprocess to connect to + + Returns: + bool: True if the Matlab subprocess ZMQ server can be contacted, + False, otherwise + + """ if not self.started: - time.sleep(2) - return False + raise RuntimeError('No running Matlab subprocess to connect to') - req = json.dumps(dict(cmd="connect"), cls=ComplexEncoder) + req = json.dumps({'cmd': 'ping'}) self.socket.send(req) start_time = time.time() - while(True): + while time.time() - start_time < self.timeout: + time.sleep(0.5) try: - resp = self.socket.recv_string(flags=zmq.NOBLOCK) - if resp == "connected": - return True - else: - return False + self.socket.recv_string(flags=zmq.NOBLOCK) + return True except zmq.ZMQError: - np.disp(".", linefeed=False) - time.sleep(1) - if (time.time() - start_time > self.maxtime) : - print "Matlab session timed out after %d seconds" % (self.maxtime) - return False + print('.', end='') + sys.stdout.flush() + print('failed to connect to Matlab after %d seconds' % self.timeout) + return False def is_function_processor_working(self): - result = self.run_func('%s/test_functions/test_sum.m' % MATLAB_FOLDER, {'echo': 'Matlab: Function processor is working!'}) - if result['success'] == 'true': + """Check whether the Matlab subprocess can evaluate functions + + First check whether the Python client can talk to the Matlab + server, then if the server is in a state where it can successfully + evaluate a function + + Raises: + RuntimeError: If there is no running Matlab subprocess to connect to + + Returns: + bool: True if Matlab can evaluate a function, False otherwise + + """ + if not self.is_connected(): + return False + + try: + self.abs(2435) return True - else: + except: return False + def execute_in_matlab(self, req): + """Execute a request in the Matlab subprocess - # Run a function in Matlab and return the result - def run_func(self, func_path, func_args=None, maxtime=None): - if self.running: - time.sleep(0.05) + Args: + req (dict): A dictionary containing the request to evaluate in + Matlab. The request should contain the 'cmd' key, and the + corresponding command recognized by matlabserver.m, as well + as any arguments required by that command - req = dict(cmd="run_function") - req['func_path'] = func_path - req['func_args'] = func_args + Returns: + resp (dict): A dictionary containing the response from the Matlab + server containing the keys 'success', 'result', and 'message' - req = json.dumps(req, cls=ComplexEncoder) + Raises: + RuntimeError: If the 'success' field of the resp object is False, + then an exception is raised with the value of the 'result' + field which will contain the identifier of the exception + raised in Matlab, and the 'message' field, which will contain + the reason for the exception + + """ + # send the request + req = json.dumps(req, cls=MatlabEncoder) self.socket.send(req) - resp = self.socket.recv_string() - resp = json.loads(resp, object_hook=as_complex) + # receive the response + resp = self.socket.recv_string() + resp = AttributeDict(json.loads(resp, cls=MatlabDecoder)) + if not resp.success: + raise RuntimeError(resp.result +': '+ resp.message) return resp - # Run some code in Matlab command line provide by a string - def run_code(self, code, maxtime=None): - if self.running: - time.sleep(0.05) + def __getattr__(self, name): + return self.bind_method(name) + def bind_method(self, name, unconditionally=False): + """Generate a Matlab function and bind it to the instance - req = dict(cmd="run_code") - req['code'] = code - req = json.dumps(req, cls=ComplexEncoder) - self.socket.send(req) - resp = self.socket.recv_string() - resp = json.loads(resp, object_hook=as_complex) + This is where the magic happens. When an unknown attribute of the + Matlab class is requested, it is assumed to be a call to a + Matlab function, and is generated and bound to the instance. - return resp + This works because getattr() falls back to __getattr__ only if no + attributes of the requested name can be found through normal + routes (__getattribute__, __dict__, class tree). + + bind_method first checks whether the requested name is a callable + Matlab function before generating a binding. + + Args: + name (str): The name of the Matlab function to call + e.g. 'sqrt', 'sum', 'svd', etc + unconditionally (bool): Bind the method without performing + checks. Used to bootstrap methods that are required and + know to exist + + Returns: + Method: a reference to a newly bound Method instance if the + requested name is determined to be a callable function + + Raises: + AttributeError: if the requested name is not a callable + Matlab function + + """ + # TODO: This does not work if the function is a mex function inside a folder of the same name + if not unconditionally and not self.exist(name): + raise AttributeError("'Matlab' object has no attribute '%s'" % name) + + # create a new method instance + method_instance = Method(weakref.ref(self), name) + method_instance.__name__ = name + + # bind to the Matlab instance with a weakref (to avoid circular references) + setattr(self, name, types.MethodType(method_instance, weakref.ref(self), Matlab)) + return getattr(self, name) + + + def run_func(self, func_path, func_args=None, maxtime=None): + path, filename = os.path.split(func_path) + func, ext = filename.split('.') + + self.addpath(path) + + def run_code(self, code, maxtime=None): + try: + return {'result': self.eval(code), 'success': 'true', 'message': ''} + except RuntimeError as e: + return {'result': '', 'success': 'false', 'message': e} def get_variable(self, varname, maxtime=None): - if self.running: - time.sleep(0.05) + return self.evalin('base',varname) - req = dict(cmd="get_var") - req['varname'] = varname - req = json.dumps(req, cls=ComplexEncoder) - self.socket.send(req) - resp = self.socket.recv_string() - resp = json.loads(resp, object_hook=as_complex) - return resp['var'] +# ---------------------------------------------------------------------------- +# MATLAB METHOD +# ---------------------------------------------------------------------------- +class Method(object): + + def __init__(self, parent, name): + """An object representing a Matlab function + + Methods are dynamically bound to instances of Matlab objects and + represent a callable function in the Matlab subprocess. + + Args: + parent: A reference to the parent (Matlab instance) to which the + Method is being bound + name: The name of the Matlab function this represents + + """ + self.name = name + self._parent = parent + self.doc = None + + def __call__(self, unused_parent_weakref, *args, **kwargs): + """Call a function with the supplied arguments in the Matlab subprocess + Args: + The *args parameter is unpacked and forwarded verbatim to Matlab. + It contains arguments in the order that they would appear in a + native function call. + + Keyword Args: + Keyword arguments are passed to Matlab in the form [key, val] so + that matlab.plot(x, y, '--', LineWidth=2) would be translated into + plot(x, y, '--', 'LineWidth', 2) + + nout (int): The number of arguments to output. By default this is + 1 for functions that return 1 or more values, and 0 for + functions that return no values. This is useful for functions + that change their behvaiour depending on the number of inputs: + U, S, V = matlab.svd(A, nout=3) + + """ + # parse out number of output arguments + nout = kwargs.pop('nout', None) + + # convert keyword arguments to arguments + args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) + + # build request + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout} + resp = self.parent.execute_in_matlab(req) + + # return the result + return resp.get('result', None) + + @property + def parent(self): + """Get the actual parent from the stored weakref + + The parent (Matlab instance) is stored as a weak reference + to eliminate circular references from dynamically binding Methods + to Matlab. + + """ + parent = self._parent() + if parent is None: + raise AttributeError('Stale reference to attribute of non-existent Matlab object') + return parent + + @property + def __doc__(self): + """Fetch the docstring from Matlab + + Get the documentation for a Matlab function by calling Matlab's builtin + help() then returning it as the Python docstring. The result is cached + so Matlab is only ever polled on the first request + + """ + if self.doc is None: + self.doc = self.parent.help(self.name) + return self.doc From dbdc5abe0d6e6e1035829414d6b966d367bd2650 Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Thu, 10 Apr 2014 11:48:13 +1000 Subject: [PATCH 02/20] Allow graphical plots by default. * draw the plots with a call to matlab.drawnow(). --- pymatbridge/pymatbridge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 91ba717..13142ff 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -173,10 +173,10 @@ def __init__(self, matlab='matlab', socket_addr=None, self.platform = platform if platform else sys.platform if self.platform == 'win32': default_socket_addr = "tcp://127.0.0.1:55555" - default_options = ['-automation', '-nofigureWindows'] + default_options = ['-automation'] else: default_socket_addr = "ipc:///tmp/pymatbridge" - default_options = ['-nodesktop', '-nodisplay'] + default_options = ['-nodesktop', '-nosplash'] self.socket_addr = socket_addr if socket_addr else default_socket_addr self.startup_options = startup_options if startup_options else default_options From fb9cd281c1a2ab103b3f6f1644234d0a71af426d Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:03:30 +0200 Subject: [PATCH 03/20] Send a better exception report to the python side Shows a full MATLAB traceback in case of error --- pymatbridge/matlab/matlabserver.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 279f54d..31d1596 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -41,7 +41,7 @@ function matlabserver(socket_address) % format the exception and pass it back to the client resp.success = false; resp.result = exception.identifier; - resp.message = exception.message; + resp.message = getReport(exception) json_resp = json.dump(resp); messenger('respond', json_resp); From 86921fa1be88bb48ffbbd47a4ed65206e0380850 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:04:15 +0200 Subject: [PATCH 04/20] Add saveout to function arguments This allows python callers to optionally save variables returned from functions on the MATLAB side. This improves performance as large objects do not need to be serialized so frequently. In combination with changes to run_code and run_script it allows more performant execution of line-by-line MATLAB from python --- pymatbridge/matlab/matlabserver.m | 15 +++++++++++++++ pymatbridge/pymatbridge.py | 12 +++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 31d1596..a7b0faa 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -71,6 +71,7 @@ function matlabserver(socket_address) % TODO: What should the default behaviour be? func = str2func(req.func); nout = req.nout; + [saveout nsaveout] = regexp(req.saveout, '(?:;)+', 'split', 'match'); if isempty(nout) try nout = min(abs(nargout(func)), 1); @@ -89,6 +90,20 @@ function matlabserver(socket_address) [resp.result{1:nout}] = func(req.args{:}); end + if length(nsaveout) + if nout == 1 + assignin('base',saveout{1},resp.result); + resp.result = ['__VAR=' saveout{1} '|' class(resp.result)]; + elseif nout > 1 + tmp_result = ''; + for i=1:nout + assignin('base',saveout{i},resp.result{i}); + tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) ';' tmp_result]; + end + resp.result = tmp_result; + end + end + % build the response resp.success = true; resp.message = 'Successfully completed request'; diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 13142ff..432d51c 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -470,12 +470,22 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): """ # parse out number of output arguments nout = kwargs.pop('nout', None) + saveout = kwargs.pop('saveout',None) + + if nout is None: + saveout = [] + else: + if saveout is not None: + if len(saveout) != nout: + raise ValueError('saveout should be the same length as nout') # convert keyword arguments to arguments args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) # build request - req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout} + so = ';'.join(saveout) + ';' if saveout else '' + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + resp = self.parent.execute_in_matlab(req) # return the result From 865a4a96c400a240f5f62df03904b8a2cf8f0bab Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:04:51 +0200 Subject: [PATCH 05/20] Fix run_func to actually call the function --- pymatbridge/pymatbridge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 432d51c..fff1c66 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -411,11 +411,10 @@ def bind_method(self, name, unconditionally=False): return getattr(self, name) - def run_func(self, func_path, func_args=None, maxtime=None): + def run_func(self, func_path, *args, **kwargs): path, filename = os.path.split(func_path) func, ext = filename.split('.') - - self.addpath(path) + return self.bind_method(func)(*args, **kwargs) def run_code(self, code, maxtime=None): try: From 4a78255988c5fcc245c1951b0e51cf5517f3a5a9 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:01:53 +0200 Subject: [PATCH 06/20] Optionally capture stdout Add capture_stdout to constructor. Passing False allows one to see display() etc messages --- pymatbridge/pymatbridge.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index fff1c66..ac3f8f2 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -133,7 +133,8 @@ def coerce_to_numpy(self, tree): class Matlab(object): def __init__(self, matlab='matlab', socket_addr=None, id='python-matlab-bridge', log=False, timeout=30, - platform=None, startup_options=None): + platform=None, startup_options=None, + capture_stdout=True): """Execute functions in a Matlab subprocess via Python Matlab provides a pythonic interface for accessing functions in Matlab. @@ -157,6 +158,8 @@ def __init__(self, matlab='matlab', socket_addr=None, available switches see: Windows: http://www.mathworks.com.au/help/matlab/ref/matlabwindows.html UNIX: http://www.mathworks.com.au/help/matlab/ref/matlabunix.html + capture_stdout: capture (hide) matlab stdout, such as disp() + and redirect to /dev/null/ """ self.MATLAB_FOLDER = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'matlab') @@ -168,6 +171,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.id = id self.log = log self.timeout = timeout + self.capture_stdout = capture_stdout # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -224,7 +228,9 @@ def start(self): command = ' '.join(command) print('Starting Matlab subprocess', end='') - self.matlab_process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=DEVNULL) + self.matlab_process = subprocess.Popen(command, shell=True, + stdin=subprocess.PIPE, + stdout=DEVNULL if self.capture_stdout else None) # Start the client self.socket = self.context.socket(zmq.REQ) From efca4283184c6e5fce584f17fbe0f386d95e85a9 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:02:15 +0200 Subject: [PATCH 07/20] Add logging Add log to constructor arguments which shows which functions are called, and what their arguments are --- pymatbridge/pymatbridge.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index ac3f8f2..3afe1e4 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -169,9 +169,9 @@ def __init__(self, matlab='matlab', socket_addr=None, self.matlab = matlab self.socket_addr = socket_addr self.id = id - self.log = log self.timeout = timeout self.capture_stdout = capture_stdout + self._log = log # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -208,6 +208,10 @@ def __del__(self): """ self.stop() + def log(self, msg): + if self._log: + print(msg) + def start(self): """Start a new Matlab subprocess and attempt to connect to it via ZMQ @@ -473,6 +477,8 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): U, S, V = matlab.svd(A, nout=3) """ + self.parent.log("CALL: %s" % self.name) + # parse out number of output arguments nout = kwargs.pop('nout', None) saveout = kwargs.pop('saveout',None) @@ -491,6 +497,8 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): so = ';'.join(saveout) + ';' if saveout else '' req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + self.parent.log("REQ: %r:"%req) + resp = self.parent.execute_in_matlab(req) # return the result From abeba521536dcf2c44d3fab7836b37f649667e15 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:02:23 +0200 Subject: [PATCH 08/20] Add a path cache Because run_code, run_script, and run_function all manipulate the MATLAB path add a local python path cache to reduce the number of addpath calls --- pymatbridge/pymatbridge.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 3afe1e4..5aab904 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -172,6 +172,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.timeout = timeout self.capture_stdout = capture_stdout self._log = log + self._path_cache = None # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -208,6 +209,21 @@ def __del__(self): """ self.stop() + def _ensure_in_path(self, path): + if not os.path.isfile(path): + raise ValueError("not a valid matlab file: %s" % path) + + path, filename = os.path.split(path) + funcname, ext = os.path.splitext(filename) + + if self._path_cache is None: + self._path_cache = self.path().split(os.pathsep) + if path not in self._path_cache: + self.addpath(path) + self._path_cache.append(path) + + return path,funcname + def log(self, msg): if self._log: print(msg) From b0d79ac034c3021a7b1875ef299d822a46d70ee9 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:05 +0200 Subject: [PATCH 09/20] Remove weakref * It didnt actually work with functions with multiple arguments * Log function proxy creation --- pymatbridge/pymatbridge.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 5aab904..6cbd058 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -433,7 +433,7 @@ def bind_method(self, name, unconditionally=False): method_instance.__name__ = name # bind to the Matlab instance with a weakref (to avoid circular references) - setattr(self, name, types.MethodType(method_instance, weakref.ref(self), Matlab)) + setattr(self, name, types.MethodType(method_instance, Matlab)) return getattr(self, name) @@ -469,11 +469,13 @@ def __init__(self, parent, name): name: The name of the Matlab function this represents """ - self.name = name self._parent = parent + self.name = name self.doc = None - def __call__(self, unused_parent_weakref, *args, **kwargs): + self.parent.log("CREATED: %s" % self.name) + + def __call__(self, _, *args, **kwargs): """Call a function with the supplied arguments in the Matlab subprocess Args: From b84be0e6ba19e59e93bccc5f31f72aa50edc230b Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:30 +0200 Subject: [PATCH 10/20] Fix run_func to add the path --- pymatbridge/pymatbridge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 6cbd058..db068fb 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -438,9 +438,8 @@ def bind_method(self, name, unconditionally=False): def run_func(self, func_path, *args, **kwargs): - path, filename = os.path.split(func_path) - func, ext = filename.split('.') - return self.bind_method(func)(*args, **kwargs) + path, funcname = self._ensure_in_path(func_path) + return self.bind_method(funcname)(*args, **kwargs) def run_code(self, code, maxtime=None): try: From d0f46f05da443260e2b5de187d258a0b81ba8e83 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:52 +0200 Subject: [PATCH 11/20] Run_code in the base workspace * So we can get to the variables created within --- pymatbridge/pymatbridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index db068fb..b3a7fc4 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -443,7 +443,7 @@ def run_func(self, func_path, *args, **kwargs): def run_code(self, code, maxtime=None): try: - return {'result': self.eval(code), 'success': 'true', 'message': ''} + return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} except RuntimeError as e: return {'result': '', 'success': 'false', 'message': e} From 5a294fa8e36e776399c0f4b772fed814e6853dc8 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 18:56:00 +0200 Subject: [PATCH 12/20] Pass and return args as dictionaries Due to the inherent ambiguity in matlab's JSON decoding ability this is the only way to make passing >1 same-sized array arguments work. Otherwise the matlab json library coerces them to a matrix --- pymatbridge/matlab/matlabserver.m | 29 +++++++++++++++++++++-------- pymatbridge/pymatbridge.py | 12 +++++++++++- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index a7b0faa..066e185 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -62,9 +62,11 @@ function matlabserver(socket_address) % 'args': A cell array of args to expand into the function arguments % 'nout': The number of output arguments requested - % function of no arguments - if ~isfield(req, 'args') - req.args = {} + args = {}; + if req.nin > 0 + for i=1:req.nin + args{i} = req.args.(['a' num2str(i-1)]); + end end % determine the number of output arguments @@ -83,22 +85,33 @@ function matlabserver(socket_address) % call the function, taking care of broadcasting outputs switch nout case 0 - func(req.args{:}); + func(args{:}); case 1 - resp.result = func(req.args{:}); + resp.result = func(args{:}); otherwise - [resp.result{1:nout}] = func(req.args{:}); + [resp.result{1:nout}] = func(args{:}); + if ~length(nsaveout) + %because of ambiguity of json encoding arrays of matrices + %convert multiple output arguments into a structure with + %fields name a0..aN. Convert these back to a list of matrices + %at the python end + result_struct = struct('nout',nout); + for i=1:nout + result_struct.(['a' num2str(i-1)]) = resp.result{i}; + end + resp.result = result_struct; + end end if length(nsaveout) if nout == 1 assignin('base',saveout{1},resp.result); - resp.result = ['__VAR=' saveout{1} '|' class(resp.result)]; + resp.result = ['__VAR=' saveout{1} '|' class(resp.result) '(' mat2str(size(resp.result)) ')']; elseif nout > 1 tmp_result = ''; for i=1:nout assignin('base',saveout{i},resp.result{i}); - tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) ';' tmp_result]; + tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) '(' mat2str(size(resp.result{i})) ');' tmp_result]; end resp.result = tmp_result; end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index b3a7fc4..79c6008 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -390,6 +390,10 @@ def execute_in_matlab(self, req): resp = AttributeDict(json.loads(resp, cls=MatlabDecoder)) if not resp.success: raise RuntimeError(resp.result +': '+ resp.message) + + if hasattr(resp, 'result') and isinstance(resp.result, dict) and 'nout' in resp.result: + resp.result = [resp.result['a%d'%i] for i in range(resp.result['nout'])] + return resp def __getattr__(self, name): @@ -510,9 +514,15 @@ def __call__(self, _, *args, **kwargs): # convert keyword arguments to arguments args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) + #now convert to a dict with string(num) keys because of the ambiguity + #of JSON wrt decoding [[1,2],[3,4]] (2 array args get decoded as a single + #matrix argument + nin = len(args) + args = {'a%d'%i:a for i,a in enumerate(args)} + # build request so = ';'.join(saveout) + ';' if saveout else '' - req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nin': nin, 'nout': nout, 'saveout': so} self.parent.log("REQ: %r:"%req) From 35ca51b1a2da88c6c31165791081772e39ba980f Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:05:49 +0200 Subject: [PATCH 13/20] Add run_script Runs a m script file and stores any variables in the base workspace --- pymatbridge/pymatbridge.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 79c6008..bfc91da 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -445,6 +445,10 @@ def run_func(self, func_path, *args, **kwargs): path, funcname = self._ensure_in_path(func_path) return self.bind_method(funcname)(*args, **kwargs) + def run_script(self, script_path): + path, funcname = self._ensure_in_path(script_path) + self.evalin('base',"run('%s')" % funcname, nout=0) + def run_code(self, code, maxtime=None): try: return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} From 9d5166ee4ac960eeedefb8f0a438a213b0d0db27 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Mon, 4 Aug 2014 16:58:19 +0200 Subject: [PATCH 14/20] Rework run_code Write a temporary m file and execute it via run_script to allows variables to persist in the base workspace. --- pymatbridge/pymatbridge.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index bfc91da..f5b18b8 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -18,6 +18,9 @@ import types import weakref import zmq +import tempfile +import hashlib +import shutil try: # Python 2 basestring @@ -199,6 +202,9 @@ def __init__(self, matlab='matlab', socket_addr=None, self.bind_method('run', unconditionally=True) self.bind_method('version', unconditionally=True) + #generate a temporary directory to run code in + self.tempdir_code = tempfile.mkdtemp(prefix='pymatlabridge',suffix='code') + def __del__(self): """Forcibly cleanup resources @@ -296,6 +302,8 @@ def stop(self, timeout=1): self.socket.close() self.started = False + shutil.rmtree(self.tempdir_code) + def restart(self): """Restart the Matlab subprocess if the state becomes bad @@ -447,13 +455,16 @@ def run_func(self, func_path, *args, **kwargs): def run_script(self, script_path): path, funcname = self._ensure_in_path(script_path) - self.evalin('base',"run('%s')" % funcname, nout=0) - - def run_code(self, code, maxtime=None): - try: - return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} - except RuntimeError as e: - return {'result': '', 'success': 'false', 'message': e} + return self.evalin('base',"run('%s')" % funcname, nout=0) + + def run_code(self, code): + #write a temporary file + fn = os.path.join(self.tempdir_code, + 'code_' + hashlib.md5(code).hexdigest() + '.m') + if not os.path.isfile(fn): + with open(fn,'w') as f: + f.write(code) + return self.run_script(fn) def get_variable(self, varname, maxtime=None): return self.evalin('base',varname) From ec9757bb1238fa5e83fa0ac01fe89b1f566f9e7d Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 7 Aug 2014 21:19:44 +0200 Subject: [PATCH 15/20] Pass large matrices via the filesystem --- pymatbridge/matlab/matlabserver.m | 33 +++++++++++++++++++++++++++---- pymatbridge/pymatbridge.py | 32 +++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 066e185..51f87f0 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -13,6 +13,9 @@ function matlabserver(socket_address) json.startup messenger('init', socket_address); + state.tmp_mat_dir = tempname; + mkdir(state.tmp_mat_dir); + while true % don't let any errors escape (and crash the server) try @@ -29,7 +32,8 @@ function matlabserver(socket_address) break; case {'call'} - resp = call(req); + resp = call(req, state); + resp.state = state; json_resp = json.dump(resp); messenger('respond', json_resp); @@ -47,10 +51,13 @@ function matlabserver(socket_address) messenger('respond', json_resp); end end + + rmdir(state.tmp_mat_dir, 's'); + end -function resp = call(req) +function resp = call(req, state) % CALL Call a Matlab function % % RESPONSE = CALL(REQUEST) calls Matlab's FEVAL function, intelligently @@ -65,7 +72,15 @@ function matlabserver(socket_address) args = {}; if req.nin > 0 for i=1:req.nin - args{i} = req.args.(['a' num2str(i-1)]); + fn = num2str(i-1); + if isfield(req.args,['a' fn]) + args{i} = req.args.(['a' num2str(i-1)]); + elseif isfield(req.args,['b' fn]) + data = load(req.args.(['b' fn]), 'v'); + args{i} = data.v; + else + throw(MException('MATLAB:matlabserver', ['Unrecognized field ' 'b' fn])); + end end end @@ -97,7 +112,17 @@ function matlabserver(socket_address) %at the python end result_struct = struct('nout',nout); for i=1:nout - result_struct.(['a' num2str(i-1)]) = resp.result{i}; + v = resp.result{i}; + if isa(v,'numeric') & (numel(v) > 100) + key = ['b' num2str(i-1)]; + val = fullfile(state.tmp_mat_dir, [key '.mat']); + save(val,'v','-v6'); + clear v; + else + key = ['a' num2str(i-1)]; + val = v; + end + result_struct.(key) = val; end resp.result = result_struct; end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index f5b18b8..c59709b 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -10,6 +10,7 @@ import functools import json import numpy as np +import scipy.io import os import platform import subprocess @@ -399,8 +400,21 @@ def execute_in_matlab(self, req): if not resp.success: raise RuntimeError(resp.result +': '+ resp.message) + self.log("RESP: %r:"%resp) + if hasattr(resp, 'result') and isinstance(resp.result, dict) and 'nout' in resp.result: - resp.result = [resp.result['a%d'%i] for i in range(resp.result['nout'])] + array_result = [] + for i in range(resp.result['nout']): + try: + array_result.append(resp.result['a%d'%i]) + except KeyError: + #passed as matlab array + key = 'b%d'%i + matname = os.path.join(resp['state']['tmp_mat_dir'], key+'.mat') + val = scipy.io.loadmat(matname)['v'] + array_result.append(val) + + resp.result = array_result return resp @@ -532,14 +546,22 @@ def __call__(self, _, *args, **kwargs): #now convert to a dict with string(num) keys because of the ambiguity #of JSON wrt decoding [[1,2],[3,4]] (2 array args get decoded as a single #matrix argument - nin = len(args) - args = {'a%d'%i:a for i,a in enumerate(args)} + dargs = {} + for i,a in enumerate(args): + if isinstance(a,np.ndarray) and (a.nbytes > 1000): + key = 'b%d' % i + val = os.path.join(self.parent.tempdir_code,'%s.mat' % key) + scipy.io.savemat(val, {'v':a}, oned_as='row') + else: + key = 'a%d' % i + val = a + dargs[key] = val # build request so = ';'.join(saveout) + ';' if saveout else '' - req = {'cmd': 'call', 'func': self.name, 'args': args, 'nin': nin, 'nout': nout, 'saveout': so} + req = {'cmd': 'call', 'func': self.name, 'args': dargs, 'nin': len(dargs), 'nout': nout, 'saveout': so} - self.parent.log("REQ: %r:"%req) + self.parent.log("REQ : %r:"%req) resp = self.parent.execute_in_matlab(req) From 34f6303415a875fd8021e80cc0e0224210257d76 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Wed, 24 Sep 2014 14:07:07 +0200 Subject: [PATCH 16/20] Add proxy variables for flexible passing of data Proxy variables will are now returned from calls to functions with saveout=(...). These variables can be passed to subsequent function calls and will be transparently reconstituted on the MATLAB side --- pymatbridge/matlab/matlabserver.m | 2 + pymatbridge/pymatbridge.py | 67 +++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 51f87f0..3ef2c6c 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -78,6 +78,8 @@ function matlabserver(socket_address) elseif isfield(req.args,['b' fn]) data = load(req.args.(['b' fn]), 'v'); args{i} = data.v; + elseif isfield(req.args,['p' fn]) + args{i} = evalin('base',req.args.(['p' fn])); else throw(MException('MATLAB:matlabserver', ['Unrecognized field ' 'b' fn])); end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index c59709b..61db148 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -9,9 +9,11 @@ import collections import functools import json +import string import numpy as np import scipy.io import os +import random import platform import subprocess import sys @@ -22,6 +24,7 @@ import tempfile import hashlib import shutil +import re try: # Python 2 basestring @@ -55,6 +58,27 @@ def chain(*iterables): for item in iterable: yield item +class ProxyVariable(object): + PROXY_RE = re.compile("__VAR=(?P[a-zA-Z_]+[a-zA-Z0-9_]*)\|(?P[a-zA-Z0-9[\]() ]+)") + def __init__(self, parent, desc): + self._parent = parent + self._desc = desc + self._info = self.PROXY_RE.match(desc).groupdict() + self.name = self._info['name'] + + def __call__(self): + parent = self._parent() + return parent.get_variable(self.name) + + def __repr__(self): + return "" % self._desc + + def __str__(self): + return self.name + + @staticmethod + def matches(placeholder): + return ProxyVariable.PROXY_RE.match(placeholder) is not None class AttributeDict(dict): """A dictionary with attribute-like access @@ -177,6 +201,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.capture_stdout = capture_stdout self._log = log self._path_cache = None + self._name_cache = {} # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -197,11 +222,12 @@ def __init__(self, matlab='matlab', socket_addr=None, # auto-generate some useful matlab builtins self.bind_method('exist', unconditionally=True) self.bind_method('addpath', unconditionally=True) - self.bind_method('eval', unconditionally=True) + self.bind_method('evalin', unconditionally=True) self.bind_method('help', unconditionally=True) self.bind_method('license', unconditionally=True) self.bind_method('run', unconditionally=True) self.bind_method('version', unconditionally=True) + self.bind_method('assignin',unconditionally=True) #generate a temporary directory to run code in self.tempdir_code = tempfile.mkdtemp(prefix='pymatlabridge',suffix='code') @@ -391,8 +417,8 @@ def execute_in_matlab(self, req): """ # send the request - req = json.dumps(req, cls=MatlabEncoder) - self.socket.send(req) + reqs = json.dumps(req, cls=MatlabEncoder) + self.socket.send(reqs) # receive the response resp = self.socket.recv_string() @@ -415,6 +441,18 @@ def execute_in_matlab(self, req): array_result.append(val) resp.result = array_result + elif hasattr(resp, 'result') and isinstance(resp.result, (str, unicode)): + if req.get('saveout') and ProxyVariable.matches(resp.result): + proxies = [] + #filter the last empty split match (there is a trialing ;) + for p in filter(len, resp.result.split(';')): + proxies.append( ProxyVariable(weakref.ref(self), p) ) + + #sort according to the order of saveout + saveout = req['saveout'].split(';') + proxies.sort(key=lambda x: saveout.index(x.name)) + + resp.result = proxies[0] if len(proxies) == 1 else proxies return resp @@ -480,8 +518,26 @@ def run_code(self, code): f.write(code) return self.run_script(fn) + def proxy_variable(self, varname): + name = '__VAR=' + varname + '|' + \ + self.evalin('base','class(%s)' % varname) + \ + '(' + self.evalin('base','mat2str(size(%s))' % varname) + ')' + return ProxyVariable(weakref.ref(self), name) + def get_variable(self, varname, maxtime=None): - return self.evalin('base',varname) + #str(varname) converts proxyvariables to their name + return self.evalin('base',str(varname)) + + def set_variable(self, varname, var): + self.assignin('base',varname,var) + return self.proxy_variable(varname) + + def varname(self, prefix='', postfix=''): + while True: + s = prefix + ''.join(random.choice(string.ascii_uppercase) for _ in range(10)) + postfix + if s not in self._name_cache: + self._name_cache[s] = True + return s # ---------------------------------------------------------------------------- @@ -552,6 +608,9 @@ def __call__(self, _, *args, **kwargs): key = 'b%d' % i val = os.path.join(self.parent.tempdir_code,'%s.mat' % key) scipy.io.savemat(val, {'v':a}, oned_as='row') + elif isinstance(a,ProxyVariable): + key = 'p%d' % i + val = a.name else: key = 'a%d' % i val = a From 2ac61cc7ed6da8ae18d0c597faff91416bde4404 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Mon, 29 Sep 2014 18:32:04 +0200 Subject: [PATCH 17/20] run_code supports calling functions --- pymatbridge/pymatbridge.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 61db148..2d46d51 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -159,6 +159,9 @@ def coerce_to_numpy(self, tree): # MATLAB # ---------------------------------------------------------------------------- class Matlab(object): + + STRING_IS_FUNCTION = re.compile("[\w\s%]*function.*") + def __init__(self, matlab='matlab', socket_addr=None, id='python-matlab-bridge', log=False, timeout=30, platform=None, startup_options=None, @@ -500,7 +503,6 @@ def bind_method(self, name, unconditionally=False): setattr(self, name, types.MethodType(method_instance, Matlab)) return getattr(self, name) - def run_func(self, func_path, *args, **kwargs): path, funcname = self._ensure_in_path(func_path) return self.bind_method(funcname)(*args, **kwargs) @@ -509,14 +511,18 @@ def run_script(self, script_path): path, funcname = self._ensure_in_path(script_path) return self.evalin('base',"run('%s')" % funcname, nout=0) - def run_code(self, code): + def run_code(self, code, *args, **kwargs): #write a temporary file fn = os.path.join(self.tempdir_code, 'code_' + hashlib.md5(code).hexdigest() + '.m') if not os.path.isfile(fn): with open(fn,'w') as f: f.write(code) - return self.run_script(fn) + + if self.STRING_IS_FUNCTION.match(code): + return self.run_func(fn, *args, **kwargs) + else: + return self.run_script(fn) def proxy_variable(self, varname): name = '__VAR=' + varname + '|' + \ From 650c747bcc4daacf3300264ba5f0edc77a6c39da Mon Sep 17 00:00:00 2001 From: John Stowers Date: Wed, 1 Oct 2014 11:28:37 +0200 Subject: [PATCH 18/20] Add util function for saving MATLAB figures Allows one to do with mlab.fig('foo.jpg') as f: mlab.plot(...) --- pymatbridge/pymatbridge.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 2d46d51..e1319c1 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -7,6 +7,7 @@ """ from __future__ import print_function import collections +import contextlib import functools import json import string @@ -545,6 +546,20 @@ def varname(self, prefix='', postfix=''): self._name_cache[s] = True return s + @contextlib.contextmanager + def fig(self, destination, name=None): + dirname,figname = os.path.split(destination) + if not os.path.exists(dirname): + os.makedirs(dirname) + + if name is not None: + f = self.figure() + else: + f = self.figure('Name',str(name)) + + yield f + + self.saveas(f, destination) # ---------------------------------------------------------------------------- # MATLAB METHOD From 446ee24cb52d0a6986ba39b5b9e3a712006e6f0d Mon Sep 17 00:00:00 2001 From: John Stowers Date: Wed, 24 Sep 2014 14:18:44 +0200 Subject: [PATCH 19/20] Add examples of recent functionality --- pymatbridge/examples/example_func.m | 3 + pymatbridge/examples/example_script.m | 3 + pymatbridge/examples/outvariables.py | 103 ++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 pymatbridge/examples/example_func.m create mode 100644 pymatbridge/examples/example_script.m create mode 100644 pymatbridge/examples/outvariables.py diff --git a/pymatbridge/examples/example_func.m b/pymatbridge/examples/example_func.m new file mode 100644 index 0000000..063fbaf --- /dev/null +++ b/pymatbridge/examples/example_func.m @@ -0,0 +1,3 @@ +function lol = bar_func(name) + lol=['hello from ' name]; +end diff --git a/pymatbridge/examples/example_script.m b/pymatbridge/examples/example_script.m new file mode 100644 index 0000000..ae468e9 --- /dev/null +++ b/pymatbridge/examples/example_script.m @@ -0,0 +1,3 @@ +q=1:10; +f=45; + diff --git a/pymatbridge/examples/outvariables.py b/pymatbridge/examples/outvariables.py new file mode 100644 index 0000000..7da00d9 --- /dev/null +++ b/pymatbridge/examples/outvariables.py @@ -0,0 +1,103 @@ +import time +import numpy as np +import numpy.testing as npt +import os.path +import pymatbridge + +_dir = os.path.dirname(os.path.abspath(__file__)) + +mlab = pymatbridge.Matlab(matlab='/opt/matlab/R2013a/bin/matlab', log=True, capture_stdout=True) +mlab.start() + +if 1: + with mlab.fig('/tmp/bob/test.png') as f: + mlab.plot(range(1000)) + mlab.suptitle('test') + mlab.ylabel('y label') + +if 1: + px = mlab.set_variable('v1',5) + assert repr(px) == '' + assert str(px) == 'v1' + assert px() == 5 + +if 1: + #conventional behaviour, perform the matlab command and return the result + z = mlab.zeros(5) + npt.assert_equal(z, np.zeros((5,5))) + + #perform the same command, and save the 1 output variable on the matlab side + #with the name 'z'. return a placeholder containing some metadata about it + _z = mlab.zeros(5,nout=1,saveout=('z',)) + assert repr(_z) == '' + + #now return the real variable, not the proxy + z = _z() + npt.assert_equal(z, np.zeros((5,5))) + + #now return the result + z = mlab.get_variable('z') + npt.assert_equal(z, np.zeros((5,5))) + +if 1: + #this time the matlab command returns two variables + x,y = mlab.meshgrid(range(1,4),range(10,15),nout=2) + npx,npy = np.meshgrid(range(1,4),range(10,15)) + npt.assert_equal(x,npx); npt.assert_equal(y,npy) + + #perform the same command, but leave the result in matlab + _x,_y = mlab.meshgrid(range(1,4),range(10,15),nout=2,saveout=('X','Y')) + assert repr(_x) == '' + assert repr(_y) == '' + + #now return the real variable, not the proxy + x = _x() + npt.assert_equal(x,npx) + + #now return the result + x = mlab.get_variable('X') + npt.assert_equal(x,npx) + + #or pass the proxy to get_variable to do the same + x = mlab.get_variable(_x) + npt.assert_equal(x,npx) + + #now pass the proxy to a matlab command and check it decodes it + z = mlab.cat(3,_x,_y) + npz = np.dstack((npx,npy)) + npt.assert_equal(z,npz) + + #annother approach, mixing variables on the matlab side + #(calling str(proxy) will return the name) + mlab.run_code('zb=cat(3,%s,%s);' % (_x, _y)) + zb = mlab.get_variable('zb') + npt.assert_equal(z,zb) + +if 1: + mlab.run_func(os.path.join(_dir,'example_func.m'), 'john', nout=1, saveout=('lol',)) + assert 'hello from john' == mlab.get_variable('lol') + +if 1: + mlab.run_script(os.path.join(_dir,'example_script.m')) + q = mlab.get_variable('q') + npt.assert_equal(q,range(1,11)) + f = mlab.get_variable('f') + npt.assert_equal(f,45) + +if 1: + mlab.run_code('foo=1:100;') + m = mlab.get_variable('foo') + npt.assert_equal(m,range(1,101)) + + mlab.run_code('foo=1:100;') + +if 1: + a,b = mlab.run_code(""" +function [a, b] = cheese(c, d) + a = 2; + b = c + d; +end""", 3, 9, nout=2) + print a + print b + +mlab.stop() From 073b4b5ffa6d1d63b448b33d3c2c9cf8d3b2cca6 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Wed, 15 Oct 2014 14:07:02 +0200 Subject: [PATCH 20/20] SQUASH fig decorator --- pymatbridge/pymatbridge.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index e1319c1..542d924 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -547,7 +547,7 @@ def varname(self, prefix='', postfix=''): return s @contextlib.contextmanager - def fig(self, destination, name=None): + def fig(self, destination, name=None, driver=None): dirname,figname = os.path.split(destination) if not os.path.exists(dirname): os.makedirs(dirname) @@ -559,7 +559,10 @@ def fig(self, destination, name=None): yield f - self.saveas(f, destination) + if driver is not None: + self.saveas(f, destination, driver) + else: + self.saveas(f, destination) # ---------------------------------------------------------------------------- # MATLAB METHOD