diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m index e582047..ed8ef62 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m @@ -63,7 +63,6 @@ % mapped to the same json string '[1,2]'. % % See also json.load json.write - json_startup('WarnOnAddpath', true); options = get_options_(varargin{:}); obj = dump_data_(value, options); @@ -102,7 +101,20 @@ elseif ~isscalar(value) obj = javaObject('org.json.JSONArray'); - if ndims(value) > 2 + if isnumeric(value) + % encode arrays as a struct + double_struct = struct; + double_struct.ndarray = 1; + value = double(value); + if isreal(value) + double_struct.data = base64encode(typecast(value(:), 'uint8')); + else + double_struct.real = base64encode(typecast(real(value(:)), 'uint8')); + double_struct.imag = base64encode(typecast(imag(value(:)), 'uint8')); + end + double_struct.shape = base64encode(typecast(size(value), 'uint8')); + obj = dump_data_(double_struct, options); + elseif ndims(value) > 2 split_value = num2cell(value, 1:ndims(value)-1); for i = 1:numel(split_value) obj.put(dump_data_(split_value{i}, options)); @@ -151,3 +163,165 @@ error('json:typeError', 'Unsupported data type: %s', class(value)); end end + + +function y = base64encode(x, eol) +%BASE64ENCODE Perform base64 encoding on a string. +% +% BASE64ENCODE(STR, EOL) encode the given string STR. EOL is the line ending +% sequence to use; it is optional and defaults to '\n' (ASCII decimal 10). +% The returned encoded string is broken into lines of no more than 76 +% characters each, and each line will end with EOL unless it is empty. Let +% EOL be empty if you do not want the encoded string broken into lines. +% +% STR and EOL don't have to be strings (i.e., char arrays). The only +% requirement is that they are vectors containing values in the range 0-255. +% +% This function may be used to encode strings into the Base64 encoding +% specified in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The +% Base64 encoding is designed to represent arbitrary sequences of octets in a +% form that need not be humanly readable. A 65-character subset +% ([A-Za-z0-9+/=]) of US-ASCII is used, enabling 6 bits to be represented per +% printable character. +% +% Examples +% -------- +% +% If you want to encode a large file, you should encode it in chunks that are +% a multiple of 57 bytes. This ensures that the base64 lines line up and +% that you do not end up with padding in the middle. 57 bytes of data fills +% one complete base64 line (76 == 57*4/3): +% +% If ifid and ofid are two file identifiers opened for reading and writing, +% respectively, then you can base64 encode the data with +% +% while ~feof(ifid) +% fwrite(ofid, base64encode(fread(ifid, 60*57))); +% end +% +% or, if you have enough memory, +% +% fwrite(ofid, base64encode(fread(ifid))); +% +% See also BASE64DECODE. + +% Author: Peter J. Acklam +% Time-stamp: 2004-02-03 21:36:56 +0100 +% E-mail: pjacklam@online.no +% URL: http://home.online.no/~pjacklam + + % check number of input arguments + error(nargchk(1, 2, nargin)); + + % make sure we have the EOL value + if nargin < 2 + eol = ''; %sprintf('\n'); + else + if sum(size(eol) > 1) > 1 + error('EOL must be a vector.'); + end + if any(eol(:) > 255) + error('EOL can not contain values larger than 255.'); + end + end + + if sum(size(x) > 1) > 1 + error('STR must be a vector.'); + end + + x = uint8(x); + eol = uint8(eol); + + ndbytes = length(x); % number of decoded bytes + nchunks = ceil(ndbytes / 3); % number of chunks/groups + nebytes = 4 * nchunks; % number of encoded bytes + + % add padding if necessary, to make the length of x a multiple of 3 + if rem(ndbytes, 3) + x(end+1 : 3*nchunks) = 0; + end + + x = reshape(x, [3, nchunks]); % reshape the data + y = repmat(uint8(0), 4, nchunks); % for the encoded data + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Split up every 3 bytes into 4 pieces + % + % aaaaaabb bbbbcccc ccdddddd + % + % to form + % + % 00aaaaaa 00bbbbbb 00cccccc 00dddddd + % + y(1,:) = bitshift(x(1,:), -2); % 6 highest bits of x(1,:) + + y(2,:) = bitshift(bitand(x(1,:), 3), 4); % 2 lowest bits of x(1,:) + y(2,:) = bitor(y(2,:), bitshift(x(2,:), -4)); % 4 highest bits of x(2,:) + + y(3,:) = bitshift(bitand(x(2,:), 15), 2); % 4 lowest bits of x(2,:) + y(3,:) = bitor(y(3,:), bitshift(x(3,:), -6)); % 2 highest bits of x(3,:) + + y(4,:) = bitand(x(3,:), 63); % 6 lowest bits of x(3,:) + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Now perform the following mapping + % + % 0 - 25 -> A-Z + % 26 - 51 -> a-z + % 52 - 61 -> 0-9 + % 62 -> + + % 63 -> / + % + % We could use a mapping vector like + % + % ['A':'Z', 'a':'z', '0':'9', '+/'] + % + % but that would require an index vector of class double. + % + z = repmat(uint8(0), size(y)); + i = y <= 25; z(i) = 'A' + double(y(i)); + i = 26 <= y & y <= 51; z(i) = 'a' - 26 + double(y(i)); + i = 52 <= y & y <= 61; z(i) = '0' - 52 + double(y(i)); + i = y == 62; z(i) = '+'; + i = y == 63; z(i) = '/'; + y = z; + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Add padding if necessary. + % + npbytes = 3 * nchunks - ndbytes; % number of padding bytes + if npbytes + y(end-npbytes+1 : end) = '='; % '=' is used for padding + end + + if isempty(eol) + + % reshape to a row vector + y = reshape(y, [1, nebytes]); + + else + + nlines = ceil(nebytes / 76); % number of lines + neolbytes = length(eol); % number of bytes in eol string + + % pad data so it becomes a multiple of 76 elements + y(nebytes + 1 : 76 * nlines) = 0; + y = reshape(y, 76, nlines); + + % insert eol strings + eol = eol(:); + y(end + 1 : end + neolbytes, :) = eol(:, ones(1, nlines)); + + % remove padding, but keep the last eol string + m = nebytes + neolbytes * (nlines - 1); + n = (76+neolbytes)*nlines - neolbytes; + y(m+1 : n) = ''; + + % extract and reshape to row vector + y = reshape(y, 1, m+neolbytes); + + end + + % output is a character array + y = char(y); +end diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m index 39dcf6d..ed2660f 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m @@ -122,8 +122,17 @@ value.(safe_field) = parse_data_(node.get(javaObject('java.lang.String', key)), ... options); end - % Check if the struct just decoded represents a complex number - if isfield(value,'real') && isfield(value, 'imag') + % Check if the struct just decoded represents an array or complex number + if isfield(value,'ndarray') && isfield(value, 'shape') + if isfield(value, 'data') + arr = typecast(base64decode(value.data), 'double'); + else + r = typecast(base64decode(value.real), 'double'); + im = typecast(base64decode(value.imag), 'double'); + arr = complex(r, im); + end + value = reshape(arr, value.shape); + elseif isfield(value,'real') && isfield(value, 'imag') complex_value = complex(value.real, value.imag); value = complex_value; end @@ -198,3 +207,92 @@ vec = [vec, uint8([fields{:}])]; end end + + + +function y = base64decode(x) + %BASE64DECODE Perform base64 decoding on a string. + % + % BASE64DECODE(STR) decodes the given base64 string STR. + % + % Any character not part of the 65-character base64 subset set is silently + % ignored. + % + % This function is used to decode strings from the Base64 encoding specified + % in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The Base64 + % encoding is designed to represent arbitrary sequences of octets in a form + % that need not be humanly readable. A 65-character subset ([A-Za-z0-9+/=]) + % of US-ASCII is used, enabling 6 bits to be represented per printable + % character. + % + % See also BASE64ENCODE. + + % Author: Peter J. Acklam + % Time-stamp: 2004-09-20 08:20:50 +0200 + % E-mail: pjacklam@online.no + % URL: http://home.online.no/~pjacklam + + % Modified by Guillaume Flandin, May 2008 + + % check number of input arguments + %-------------------------------------------------------------------------- + + error(nargchk(1, 1, nargin)); + + % Perform the following mapping + %-------------------------------------------------------------------------- + % A-Z -> 0 - 25 a-z -> 26 - 51 0-9 -> 52 - 61 + % + -> 62 / -> 63 = -> 64 + % anything else -> NaN + + base64chars = NaN(1,256); + base64chars('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=') = 0:64; + x = base64chars(x); + + % Remove/ignore any characters not in the base64 characters list or '=' + %-------------------------------------------------------------------------- + + x = x(~isnan(x)); + + % Replace any incoming padding ('=' -> 64) with a zero pad + %-------------------------------------------------------------------------- + + if x(end-1) == 64, p = 2; x(end-1:end) = 0; + elseif x(end) == 64, p = 1; x(end) = 0; + else p = 0; + end + + % Allocate decoded data array + %-------------------------------------------------------------------------- + + n = length(x) / 4; % number of groups + x = reshape(uint8(x), 4, n); % input data + y = zeros(3, n, 'uint8'); % decoded data + + % Rearrange every 4 bytes into 3 bytes + %-------------------------------------------------------------------------- + % 00aaaaaa 00bbbbbb 00cccccc 00dddddd + % + % to form + % + % aaaaaabb bbbbcccc ccdddddd + + y(1,:) = bitshift(x(1,:), 2); % 6 highest bits of y(1,:) + y(1,:) = bitor(y(1,:), bitshift(x(2,:), -4)); % 2 lowest bits of y(1,:) + + y(2,:) = bitshift(x(2,:), 4); % 4 highest bits of y(2,:) + y(2,:) = bitor(y(2,:), bitshift(x(3,:), -2)); % 4 lowest bits of y(2,:) + + y(3,:) = bitshift(x(3,:), 6); % 2 highest bits of y(3,:) + y(3,:) = bitor(y(3,:), x(4,:)); % 6 lowest bits of y(3,:) + + % Remove any zero pad that was added to make this a multiple of 24 bits + %-------------------------------------------------------------------------- + + if p, y(end-p+1:end) = []; end + + % Reshape to a row vector + %-------------------------------------------------------------------------- + + y = reshape(y, 1, []); +end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index c0667c0..4aad82e 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -9,7 +9,9 @@ This is a modified version using ZMQ, Haoxing Zhang Jan.2014 """ -import os, time +import os +import time +import codecs import zmq import subprocess import sys @@ -17,7 +19,7 @@ from uuid import uuid4 try: - from numpy import ndarray, generic + from numpy import ndarray, generic, float64, frombuffer, asfortranarray except ImportError: class ndarray: pass @@ -30,21 +32,61 @@ class spmatrix: pass +def encode_ndarray(obj): + """Write a numpy array and its shape to base64 buffers""" + shape = obj.shape + if len(shape) == 1: + shape = (1, obj.shape[0]) + if obj.flags.c_contiguous: + obj = obj.T + elif not obj.flags.f_contiguous: + obj = asfortranarray(obj) + data = obj.astype(float64).tobytes() + data = codecs.encode(data, 'base64').decode('utf-8') + return data, shape + + # JSON encoder extension to handle complex numbers and numpy arrays class PymatEncoder(json.JSONEncoder): + def default(self, obj): - if isinstance(obj, complex): - return {'real':obj.real, 'imag':obj.imag} - if isinstance(obj, ndarray): + if isinstance(obj, ndarray) and obj.dtype.kind in 'uif': + data, shape = encode_ndarray(obj) + return {'ndarray': True, 'shape': shape, 'data': data} + elif isinstance(obj, ndarray) and obj.dtype.kind == 'c': + real, shape = encode_ndarray(obj.real.copy()) + imag, _ = encode_ndarray(obj.imag.copy()) + return {'ndarray': True, 'shape': shape, + 'real': real, 'imag': imag} + elif isinstance(obj, ndarray): return obj.tolist() - if isinstance(obj, generic): + elif isinstance(obj, complex): + return {'real': obj.real, 'imag': obj.imag} + elif isinstance(obj, generic): return obj.item() # Handle the default case return json.JSONEncoder.default(self, obj) -# JSON decoder for complex numbers -def as_complex(dct): - if 'real' in dct and 'imag' in dct: + +def decode_arr(data): + """Extract a numpy array from a base64 buffer""" + data = data.encode('utf-8') + return frombuffer(codecs.decode(data, 'base64'), float64) + + +# JSON decoder for arrays and complex numbers +def decode_pymat(dct): + if 'ndarray' in dct and 'data' in dct: + value = decode_arr(dct['data']) + shape = decode_arr(dct['shape']) + return value.reshape(shape, order='F') + elif 'ndarray' in dct and 'imag' in dct: + real = decode_arr(dct['real']) + imag = decode_arr(dct['imag']) + shape = decode_arr(dct['shape']) + data = real + 1j * imag + return data.reshape(shape, order='F') + elif 'real' in dct and 'imag' in dct: return complex(dct['real'], dct['imag']) return dct @@ -201,7 +243,7 @@ def is_function_processor_working(self): def _json_response(self, **kwargs): if self.running: time.sleep(0.05) - return json.loads(self._response(**kwargs), object_hook=as_complex) + return json.loads(self._response(**kwargs), object_hook=decode_pymat) # Run a function in Matlab and return the result def run_func(self, func_path, func_args=None): diff --git a/pymatbridge/tests/test_array.py b/pymatbridge/tests/test_array.py index 4d271e3..8e95080 100644 --- a/pymatbridge/tests/test_array.py +++ b/pymatbridge/tests/test_array.py @@ -19,7 +19,7 @@ def teardown_class(cls): # Pass a 1000*1000 array to Matlab def test_array_size(self): - array = np.random.random_sample((50,50)).tolist() + array = np.random.random_sample((50,50)) res = self.mlab.run_func("array_size.m",{'val':array})['result'] npt.assert_almost_equal(res, array, decimal=8, err_msg = "test_array_size: error") diff --git a/pymatbridge/tests/test_get_variable.py b/pymatbridge/tests/test_get_variable.py index b344b0e..be38ea0 100644 --- a/pymatbridge/tests/test_get_variable.py +++ b/pymatbridge/tests/test_get_variable.py @@ -30,8 +30,8 @@ def test_get_array(self): self.mlab.run_code("a = [1 2 3 4]") self.mlab.run_code("b = [1 2; 3 4]") - npt.assert_equal(self.mlab.get_variable('a'), [1,2,3,4]) - npt.assert_equal(self.mlab.get_variable('b'), [[1,2],[3,4]]) + npt.assert_equal(self.mlab.get_variable('a'), [[1.,2.,3.,4.]]) + npt.assert_equal(self.mlab.get_variable('b'), [[1.,2.],[3.,4.]]) # Try to get a non-existent variable diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 450b108..57e3677 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -62,7 +62,7 @@ def test_line_magic(self): # Get the result back to Python self.ip.run_cell_magic('matlab', '-o actual', 'actual = res') - self.ip.run_cell("expected = np.array([2, 4, 6])") + self.ip.run_cell("expected = np.array([[2., 4., 6.]])") npt.assert_almost_equal(self.ip.user_ns['actual'], self.ip.user_ns['expected'], decimal=7) @@ -88,5 +88,5 @@ def test_struct(self): 'obj.num = num;obj.num_array = num_array;obj.str = str;') npt.assert_equal(isinstance(self.ip.user_ns['obj'], dict), True) npt.assert_equal(self.ip.user_ns['obj']['num'], self.ip.user_ns['num']) - npt.assert_equal(self.ip.user_ns['obj']['num_array'], self.ip.user_ns['num_array']) + npt.assert_equal(self.ip.user_ns['obj']['num_array'].squeeze(), self.ip.user_ns['num_array']) npt.assert_equal(self.ip.user_ns['obj']['str'], self.ip.user_ns['str'])