From e34c10dad07a118ad13ae0079d06ede7864053ff Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 16 Jan 2015 16:56:17 -0600 Subject: [PATCH 01/12] Implement efficient handling of large array passing to Matlab --- pymatbridge/matlab/util/json_v0.2.2/json/json_load.m | 5 +++++ pymatbridge/pymatbridge.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m index 39dcf6d..22f37f3 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m @@ -127,6 +127,11 @@ complex_value = complex(value.real, value.imag); value = complex_value; end + % Check if the struct just decoded represents a numpy array + if isfield(value,'ndarray') && isfield(value, 'shape') + arr = typecast(unicode2native(value.data, 'latin-1'), 'double') + value = reshape(arr, value.shape); + end % In MATLAB, nested classes end up with a $ in the name, in Octave it's a . elseif isa(node, 'org.json.JSONObject$Null') || isa(node, 'org.json.JSONObject.Null') value = []; diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index c0667c0..0111d0c 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -36,7 +36,11 @@ def default(self, obj): if isinstance(obj, complex): return {'real':obj.real, 'imag':obj.imag} if isinstance(obj, ndarray): - return obj.tolist() + if obj.size > 100: + return {'ndarray': True, 'shape': list(obj.shape), + 'data': obj.astype(float).tobytes().encode('latin-1')} + else: + return obj.tolist() if isinstance(obj, generic): return obj.item() # Handle the default case From 0af4ccd7c32359024143eed2ad91f5616853d69f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 16 Jan 2015 21:00:38 -0600 Subject: [PATCH 02/12] Use base64 encoding for large arrays --- .../matlab/util/json_v0.2.2/json/json_load.m | 94 ++++++++++++++++++- pymatbridge/pymatbridge.py | 10 +- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m index 22f37f3..8efb850 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m @@ -129,8 +129,9 @@ end % Check if the struct just decoded represents a numpy array if isfield(value,'ndarray') && isfield(value, 'shape') - arr = typecast(unicode2native(value.data, 'latin-1'), 'double') - value = reshape(arr, value.shape); + disp(value.shape) + arr = typecast(base64decode(value.data), 'double'); + value = arr; %reshape(arr, value.shape); end % In MATLAB, nested classes end up with a $ in the name, in Octave it's a . elseif isa(node, 'org.json.JSONObject$Null') || isa(node, 'org.json.JSONObject.Null') @@ -203,3 +204,92 @@ vec = [vec, uint8([fields{:}])]; end end + + + +function y = base64decode(x) +%BASE64DECODE Perform base64 decoding on a string. +% +% BASE64DECODE(STR) decodes the given base64 string STR. +% +% Any character not part of the 65-character base64 subset set is silently +% ignored. +% +% This function is used to decode strings from the Base64 encoding specified +% in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The Base64 +% encoding is designed to represent arbitrary sequences of octets in a form +% that need not be humanly readable. A 65-character subset ([A-Za-z0-9+/=]) +% of US-ASCII is used, enabling 6 bits to be represented per printable +% character. +% +% See also BASE64ENCODE. + +% Author: Peter J. Acklam +% Time-stamp: 2004-09-20 08:20:50 +0200 +% E-mail: pjacklam@online.no +% URL: http://home.online.no/~pjacklam + +% Modified by Guillaume Flandin, May 2008 + +% check number of input arguments +%-------------------------------------------------------------------------- + +error(nargchk(1, 1, nargin)); + +% Perform the following mapping +%-------------------------------------------------------------------------- +% A-Z -> 0 - 25 a-z -> 26 - 51 0-9 -> 52 - 61 +% + -> 62 / -> 63 = -> 64 +% anything else -> NaN + +base64chars = NaN(1,256); +base64chars('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=') = 0:64; +x = base64chars(x); + +% Remove/ignore any characters not in the base64 characters list or '=' +%-------------------------------------------------------------------------- + +x = x(~isnan(x)); + +% Replace any incoming padding ('=' -> 64) with a zero pad +%-------------------------------------------------------------------------- + +if x(end-1) == 64, p = 2; x(end-1:end) = 0; +elseif x(end) == 64, p = 1; x(end) = 0; +else p = 0; +end + +% Allocate decoded data array +%-------------------------------------------------------------------------- + +n = length(x) / 4; % number of groups +x = reshape(uint8(x), 4, n); % input data +y = zeros(3, n, 'uint8'); % decoded data + +% Rearrange every 4 bytes into 3 bytes +%-------------------------------------------------------------------------- +% 00aaaaaa 00bbbbbb 00cccccc 00dddddd +% +% to form +% +% aaaaaabb bbbbcccc ccdddddd + +y(1,:) = bitshift(x(1,:), 2); % 6 highest bits of y(1,:) +y(1,:) = bitor(y(1,:), bitshift(x(2,:), -4)); % 2 lowest bits of y(1,:) + +y(2,:) = bitshift(x(2,:), 4); % 4 highest bits of y(2,:) +y(2,:) = bitor(y(2,:), bitshift(x(3,:), -2)); % 4 lowest bits of y(2,:) + +y(3,:) = bitshift(x(3,:), 6); % 2 highest bits of y(3,:) +y(3,:) = bitor(y(3,:), x(4,:)); % 6 lowest bits of y(3,:) + +% Remove any zero pad that was added to make this a multiple of 24 bits +%-------------------------------------------------------------------------- + +if p, y(end-p+1:end) = []; end + +% Reshape to a row vector +%-------------------------------------------------------------------------- + +y = reshape(y, 1, []); +end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 0111d0c..9440c44 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -9,7 +9,9 @@ This is a modified version using ZMQ, Haoxing Zhang Jan.2014 """ -import os, time +import os +import time +import codecs import zmq import subprocess import sys @@ -17,7 +19,7 @@ from uuid import uuid4 try: - from numpy import ndarray, generic + from numpy import ndarray, generic, float64 except ImportError: class ndarray: pass @@ -37,8 +39,8 @@ def default(self, obj): return {'real':obj.real, 'imag':obj.imag} if isinstance(obj, ndarray): if obj.size > 100: - return {'ndarray': True, 'shape': list(obj.shape), - 'data': obj.astype(float).tobytes().encode('latin-1')} + return {'ndarray': True, 'shape': obj.shape, + 'data': codecs.encode(obj.astype(float64).tobytes(), 'base64').decode('utf-8')} else: return obj.tolist() if isinstance(obj, generic): From 516b5d8394fb3aa29547d994fd1efef3dcf94d7a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 16 Jan 2015 23:23:11 -0600 Subject: [PATCH 03/12] Finish implementing efficient arrays --- .../matlab/util/json_v0.2.2/json/json_dump.m | 172 +++++++++++++++++- .../matlab/util/json_v0.2.2/json/json_load.m | 131 +++++++------ pymatbridge/pymatbridge.py | 19 +- 3 files changed, 248 insertions(+), 74 deletions(-) diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m index e582047..a200b4c 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m @@ -63,7 +63,6 @@ % mapped to the same json string '[1,2]'. % % See also json.load json.write - json_startup('WarnOnAddpath', true); options = get_options_(varargin{:}); obj = dump_data_(value, options); @@ -102,7 +101,14 @@ elseif ~isscalar(value) obj = javaObject('org.json.JSONArray'); - if ndims(value) > 2 + if strcmp(class(value), 'double') + % encode double arrays as a struct + double_struct = struct; + double_struct.ndarray = 1; + double_struct.data = base64encode(typecast(value(:), 'uint8')); + double_struct.shape = base64encode(typecast(size(value), 'uint8')); + obj = dump_data_(double_struct, options); + elseif ndims(value) > 2 split_value = num2cell(value, 1:ndims(value)-1); for i = 1:numel(split_value) obj.put(dump_data_(split_value{i}, options)); @@ -151,3 +157,165 @@ error('json:typeError', 'Unsupported data type: %s', class(value)); end end + + +function y = base64encode(x, eol) +%BASE64ENCODE Perform base64 encoding on a string. +% +% BASE64ENCODE(STR, EOL) encode the given string STR. EOL is the line ending +% sequence to use; it is optional and defaults to '\n' (ASCII decimal 10). +% The returned encoded string is broken into lines of no more than 76 +% characters each, and each line will end with EOL unless it is empty. Let +% EOL be empty if you do not want the encoded string broken into lines. +% +% STR and EOL don't have to be strings (i.e., char arrays). The only +% requirement is that they are vectors containing values in the range 0-255. +% +% This function may be used to encode strings into the Base64 encoding +% specified in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The +% Base64 encoding is designed to represent arbitrary sequences of octets in a +% form that need not be humanly readable. A 65-character subset +% ([A-Za-z0-9+/=]) of US-ASCII is used, enabling 6 bits to be represented per +% printable character. +% +% Examples +% -------- +% +% If you want to encode a large file, you should encode it in chunks that are +% a multiple of 57 bytes. This ensures that the base64 lines line up and +% that you do not end up with padding in the middle. 57 bytes of data fills +% one complete base64 line (76 == 57*4/3): +% +% If ifid and ofid are two file identifiers opened for reading and writing, +% respectively, then you can base64 encode the data with +% +% while ~feof(ifid) +% fwrite(ofid, base64encode(fread(ifid, 60*57))); +% end +% +% or, if you have enough memory, +% +% fwrite(ofid, base64encode(fread(ifid))); +% +% See also BASE64DECODE. + +% Author: Peter J. Acklam +% Time-stamp: 2004-02-03 21:36:56 +0100 +% E-mail: pjacklam@online.no +% URL: http://home.online.no/~pjacklam + + % check number of input arguments + error(nargchk(1, 2, nargin)); + + % make sure we have the EOL value + if nargin < 2 + eol = ''; %sprintf('\n'); + else + if sum(size(eol) > 1) > 1 + error('EOL must be a vector.'); + end + if any(eol(:) > 255) + error('EOL can not contain values larger than 255.'); + end + end + + if sum(size(x) > 1) > 1 + error('STR must be a vector.'); + end + + x = uint8(x); + eol = uint8(eol); + + ndbytes = length(x); % number of decoded bytes + nchunks = ceil(ndbytes / 3); % number of chunks/groups + nebytes = 4 * nchunks; % number of encoded bytes + + % add padding if necessary, to make the length of x a multiple of 3 + if rem(ndbytes, 3) + x(end+1 : 3*nchunks) = 0; + end + + x = reshape(x, [3, nchunks]); % reshape the data + y = repmat(uint8(0), 4, nchunks); % for the encoded data + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Split up every 3 bytes into 4 pieces + % + % aaaaaabb bbbbcccc ccdddddd + % + % to form + % + % 00aaaaaa 00bbbbbb 00cccccc 00dddddd + % + y(1,:) = bitshift(x(1,:), -2); % 6 highest bits of x(1,:) + + y(2,:) = bitshift(bitand(x(1,:), 3), 4); % 2 lowest bits of x(1,:) + y(2,:) = bitor(y(2,:), bitshift(x(2,:), -4)); % 4 highest bits of x(2,:) + + y(3,:) = bitshift(bitand(x(2,:), 15), 2); % 4 lowest bits of x(2,:) + y(3,:) = bitor(y(3,:), bitshift(x(3,:), -6)); % 2 highest bits of x(3,:) + + y(4,:) = bitand(x(3,:), 63); % 6 lowest bits of x(3,:) + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Now perform the following mapping + % + % 0 - 25 -> A-Z + % 26 - 51 -> a-z + % 52 - 61 -> 0-9 + % 62 -> + + % 63 -> / + % + % We could use a mapping vector like + % + % ['A':'Z', 'a':'z', '0':'9', '+/'] + % + % but that would require an index vector of class double. + % + z = repmat(uint8(0), size(y)); + i = y <= 25; z(i) = 'A' + double(y(i)); + i = 26 <= y & y <= 51; z(i) = 'a' - 26 + double(y(i)); + i = 52 <= y & y <= 61; z(i) = '0' - 52 + double(y(i)); + i = y == 62; z(i) = '+'; + i = y == 63; z(i) = '/'; + y = z; + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Add padding if necessary. + % + npbytes = 3 * nchunks - ndbytes; % number of padding bytes + if npbytes + y(end-npbytes+1 : end) = '='; % '=' is used for padding + end + + if isempty(eol) + + % reshape to a row vector + y = reshape(y, [1, nebytes]); + + else + + nlines = ceil(nebytes / 76); % number of lines + neolbytes = length(eol); % number of bytes in eol string + + % pad data so it becomes a multiple of 76 elements + y(nebytes + 1 : 76 * nlines) = 0; + y = reshape(y, 76, nlines); + + % insert eol strings + eol = eol(:); + y(end + 1 : end + neolbytes, :) = eol(:, ones(1, nlines)); + + % remove padding, but keep the last eol string + m = nebytes + neolbytes * (nlines - 1); + n = (76+neolbytes)*nlines - neolbytes; + y(m+1 : n) = ''; + + % extract and reshape to row vector + y = reshape(y, 1, m+neolbytes); + + end + + % output is a character array + y = char(y); +end diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m index 8efb850..428e666 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m @@ -129,9 +129,8 @@ end % Check if the struct just decoded represents a numpy array if isfield(value,'ndarray') && isfield(value, 'shape') - disp(value.shape) arr = typecast(base64decode(value.data), 'double'); - value = arr; %reshape(arr, value.shape); + value = reshape(arr, value.shape); end % In MATLAB, nested classes end up with a $ in the name, in Octave it's a . elseif isa(node, 'org.json.JSONObject$Null') || isa(node, 'org.json.JSONObject.Null') @@ -208,88 +207,88 @@ function y = base64decode(x) -%BASE64DECODE Perform base64 decoding on a string. -% -% BASE64DECODE(STR) decodes the given base64 string STR. -% -% Any character not part of the 65-character base64 subset set is silently -% ignored. -% -% This function is used to decode strings from the Base64 encoding specified -% in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The Base64 -% encoding is designed to represent arbitrary sequences of octets in a form -% that need not be humanly readable. A 65-character subset ([A-Za-z0-9+/=]) -% of US-ASCII is used, enabling 6 bits to be represented per printable -% character. -% -% See also BASE64ENCODE. + %BASE64DECODE Perform base64 decoding on a string. + % + % BASE64DECODE(STR) decodes the given base64 string STR. + % + % Any character not part of the 65-character base64 subset set is silently + % ignored. + % + % This function is used to decode strings from the Base64 encoding specified + % in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The Base64 + % encoding is designed to represent arbitrary sequences of octets in a form + % that need not be humanly readable. A 65-character subset ([A-Za-z0-9+/=]) + % of US-ASCII is used, enabling 6 bits to be represented per printable + % character. + % + % See also BASE64ENCODE. -% Author: Peter J. Acklam -% Time-stamp: 2004-09-20 08:20:50 +0200 -% E-mail: pjacklam@online.no -% URL: http://home.online.no/~pjacklam + % Author: Peter J. Acklam + % Time-stamp: 2004-09-20 08:20:50 +0200 + % E-mail: pjacklam@online.no + % URL: http://home.online.no/~pjacklam -% Modified by Guillaume Flandin, May 2008 + % Modified by Guillaume Flandin, May 2008 -% check number of input arguments -%-------------------------------------------------------------------------- + % check number of input arguments + %-------------------------------------------------------------------------- -error(nargchk(1, 1, nargin)); + error(nargchk(1, 1, nargin)); -% Perform the following mapping -%-------------------------------------------------------------------------- -% A-Z -> 0 - 25 a-z -> 26 - 51 0-9 -> 52 - 61 -% + -> 62 / -> 63 = -> 64 -% anything else -> NaN + % Perform the following mapping + %-------------------------------------------------------------------------- + % A-Z -> 0 - 25 a-z -> 26 - 51 0-9 -> 52 - 61 + % + -> 62 / -> 63 = -> 64 + % anything else -> NaN -base64chars = NaN(1,256); -base64chars('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=') = 0:64; -x = base64chars(x); + base64chars = NaN(1,256); + base64chars('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=') = 0:64; + x = base64chars(x); -% Remove/ignore any characters not in the base64 characters list or '=' -%-------------------------------------------------------------------------- + % Remove/ignore any characters not in the base64 characters list or '=' + %-------------------------------------------------------------------------- -x = x(~isnan(x)); + x = x(~isnan(x)); -% Replace any incoming padding ('=' -> 64) with a zero pad -%-------------------------------------------------------------------------- + % Replace any incoming padding ('=' -> 64) with a zero pad + %-------------------------------------------------------------------------- -if x(end-1) == 64, p = 2; x(end-1:end) = 0; -elseif x(end) == 64, p = 1; x(end) = 0; -else p = 0; -end + if x(end-1) == 64, p = 2; x(end-1:end) = 0; + elseif x(end) == 64, p = 1; x(end) = 0; + else p = 0; + end -% Allocate decoded data array -%-------------------------------------------------------------------------- + % Allocate decoded data array + %-------------------------------------------------------------------------- -n = length(x) / 4; % number of groups -x = reshape(uint8(x), 4, n); % input data -y = zeros(3, n, 'uint8'); % decoded data + n = length(x) / 4; % number of groups + x = reshape(uint8(x), 4, n); % input data + y = zeros(3, n, 'uint8'); % decoded data -% Rearrange every 4 bytes into 3 bytes -%-------------------------------------------------------------------------- -% 00aaaaaa 00bbbbbb 00cccccc 00dddddd -% -% to form -% -% aaaaaabb bbbbcccc ccdddddd + % Rearrange every 4 bytes into 3 bytes + %-------------------------------------------------------------------------- + % 00aaaaaa 00bbbbbb 00cccccc 00dddddd + % + % to form + % + % aaaaaabb bbbbcccc ccdddddd -y(1,:) = bitshift(x(1,:), 2); % 6 highest bits of y(1,:) -y(1,:) = bitor(y(1,:), bitshift(x(2,:), -4)); % 2 lowest bits of y(1,:) + y(1,:) = bitshift(x(1,:), 2); % 6 highest bits of y(1,:) + y(1,:) = bitor(y(1,:), bitshift(x(2,:), -4)); % 2 lowest bits of y(1,:) -y(2,:) = bitshift(x(2,:), 4); % 4 highest bits of y(2,:) -y(2,:) = bitor(y(2,:), bitshift(x(3,:), -2)); % 4 lowest bits of y(2,:) + y(2,:) = bitshift(x(2,:), 4); % 4 highest bits of y(2,:) + y(2,:) = bitor(y(2,:), bitshift(x(3,:), -2)); % 4 lowest bits of y(2,:) -y(3,:) = bitshift(x(3,:), 6); % 2 highest bits of y(3,:) -y(3,:) = bitor(y(3,:), x(4,:)); % 6 lowest bits of y(3,:) + y(3,:) = bitshift(x(3,:), 6); % 2 highest bits of y(3,:) + y(3,:) = bitor(y(3,:), x(4,:)); % 6 lowest bits of y(3,:) -% Remove any zero pad that was added to make this a multiple of 24 bits -%-------------------------------------------------------------------------- + % Remove any zero pad that was added to make this a multiple of 24 bits + %-------------------------------------------------------------------------- -if p, y(end-p+1:end) = []; end + if p, y(end-p+1:end) = []; end -% Reshape to a row vector -%-------------------------------------------------------------------------- + % Reshape to a row vector + %-------------------------------------------------------------------------- -y = reshape(y, 1, []); + y = reshape(y, 1, []); end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 9440c44..4d00e85 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -19,7 +19,7 @@ from uuid import uuid4 try: - from numpy import ndarray, generic, float64 + from numpy import ndarray, generic, float64, array, frombuffer except ImportError: class ndarray: pass @@ -38,11 +38,12 @@ def default(self, obj): if isinstance(obj, complex): return {'real':obj.real, 'imag':obj.imag} if isinstance(obj, ndarray): - if obj.size > 100: - return {'ndarray': True, 'shape': obj.shape, - 'data': codecs.encode(obj.astype(float64).tobytes(), 'base64').decode('utf-8')} - else: - return obj.tolist() + shape = obj.shape + if len(shape) == 1: + shape = (obj.shape[0], 1) + data = codecs.encode(obj.astype(float64).tobytes(), 'base64') + return {'ndarray': True, 'shape': shape, + 'data': data.decode('utf-8')} if isinstance(obj, generic): return obj.item() # Handle the default case @@ -52,6 +53,12 @@ def default(self, obj): def as_complex(dct): if 'real' in dct and 'imag' in dct: return complex(dct['real'], dct['imag']) + if 'ndarray' in dct and 'data' in dct: + data = dct['data'].encode('utf-8') + shape = dct['shape'].encode('utf-8') + value = frombuffer(codecs.decode(data, 'base64'), float64) + shape = frombuffer(codecs.decode(shape, 'base64'), float64) + return value.reshape(shape) return dct MATLAB_FOLDER = '%s/matlab' % os.path.realpath(os.path.dirname(__file__)) From ebb1c2b8290738cbbf3897885812b4664d5c0c9f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 16 Jan 2015 23:25:36 -0600 Subject: [PATCH 04/12] Add guard for cells --- pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m index a200b4c..77ba01d 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m @@ -101,7 +101,7 @@ elseif ~isscalar(value) obj = javaObject('org.json.JSONArray'); - if strcmp(class(value), 'double') + if strcmp(class(value), 'double') && ~iscell(value) % encode double arrays as a struct double_struct = struct; double_struct.ndarray = 1; From db15e5b3f47884d080897c8ed311ec8d04708178 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 17 Jan 2015 09:33:07 -0600 Subject: [PATCH 05/12] Add handling of Fortran order --- pymatbridge/pymatbridge.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 4d00e85..0a6c0fa 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -19,7 +19,7 @@ from uuid import uuid4 try: - from numpy import ndarray, generic, float64, array, frombuffer + from numpy import ndarray, generic, float64, frombuffer, asfortranarray except ImportError: class ndarray: pass @@ -34,21 +34,29 @@ class spmatrix: # JSON encoder extension to handle complex numbers and numpy arrays class PymatEncoder(json.JSONEncoder): + def default(self, obj): if isinstance(obj, complex): - return {'real':obj.real, 'imag':obj.imag} - if isinstance(obj, ndarray): + return {'real': obj.real, 'imag': obj.imag} + if isinstance(obj, ndarray) and not obj.dtype.kind == 'c': shape = obj.shape if len(shape) == 1: shape = (obj.shape[0], 1) - data = codecs.encode(obj.astype(float64).tobytes(), 'base64') - return {'ndarray': True, 'shape': shape, - 'data': data.decode('utf-8')} + if obj.flags.c_contiguous: + obj = obj.T + elif not obj.flags.f_contiguous: + obj = asfortranarray(obj) + data = obj.astype(float64).tobytes() + data = codecs.encode(data, 'base64').decode('utf-8') + return {'ndarray': True, 'shape': shape, 'data': data} + elif isinstance(obj, ndarray): + return obj.tolist() if isinstance(obj, generic): return obj.item() # Handle the default case return json.JSONEncoder.default(self, obj) + # JSON decoder for complex numbers def as_complex(dct): if 'real' in dct and 'imag' in dct: @@ -58,7 +66,7 @@ def as_complex(dct): shape = dct['shape'].encode('utf-8') value = frombuffer(codecs.decode(data, 'base64'), float64) shape = frombuffer(codecs.decode(shape, 'base64'), float64) - return value.reshape(shape) + return value.reshape(shape, order='F') return dct MATLAB_FOLDER = '%s/matlab' % os.path.realpath(os.path.dirname(__file__)) From 77103402608131e13270e1a102fd2d453d4daa4b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 17 Jan 2015 09:33:36 -0600 Subject: [PATCH 06/12] Handle all arrays on the matlab end --- pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m index 77ba01d..672f3a7 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m @@ -101,10 +101,11 @@ elseif ~isscalar(value) obj = javaObject('org.json.JSONArray'); - if strcmp(class(value), 'double') && ~iscell(value) - % encode double arrays as a struct + if isnumeric(value) && isreal(value) + % encode arrays as a struct double_struct = struct; double_struct.ndarray = 1; + value = double(value); double_struct.data = base64encode(typecast(value(:), 'uint8')); double_struct.shape = base64encode(typecast(size(value), 'uint8')); obj = dump_data_(double_struct, options); From 350519f673b79f8676f9224c7809a36acf3dd366 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 17 Jan 2015 09:34:12 -0600 Subject: [PATCH 07/12] No need to convert to list --- pymatbridge/tests/test_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/tests/test_array.py b/pymatbridge/tests/test_array.py index 4d271e3..8e95080 100644 --- a/pymatbridge/tests/test_array.py +++ b/pymatbridge/tests/test_array.py @@ -19,7 +19,7 @@ def teardown_class(cls): # Pass a 1000*1000 array to Matlab def test_array_size(self): - array = np.random.random_sample((50,50)).tolist() + array = np.random.random_sample((50,50)) res = self.mlab.run_func("array_size.m",{'val':array})['result'] npt.assert_almost_equal(res, array, decimal=8, err_msg = "test_array_size: error") From 7a78e3248c8f94a5c9adef59511d9193b73ac587 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 17 Jan 2015 09:34:23 -0600 Subject: [PATCH 08/12] Update test to reflect floating point --- pymatbridge/tests/test_get_variable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymatbridge/tests/test_get_variable.py b/pymatbridge/tests/test_get_variable.py index b344b0e..be38ea0 100644 --- a/pymatbridge/tests/test_get_variable.py +++ b/pymatbridge/tests/test_get_variable.py @@ -30,8 +30,8 @@ def test_get_array(self): self.mlab.run_code("a = [1 2 3 4]") self.mlab.run_code("b = [1 2; 3 4]") - npt.assert_equal(self.mlab.get_variable('a'), [1,2,3,4]) - npt.assert_equal(self.mlab.get_variable('b'), [[1,2],[3,4]]) + npt.assert_equal(self.mlab.get_variable('a'), [[1.,2.,3.,4.]]) + npt.assert_equal(self.mlab.get_variable('b'), [[1.,2.],[3.,4.]]) # Try to get a non-existent variable From 0bd1ece974b8c6699403e58c135e42b24b644f0d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 17 Jan 2015 09:34:44 -0600 Subject: [PATCH 09/12] Update tests for floating point and explicit shape --- pymatbridge/tests/test_magic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 450b108..3616825 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -62,7 +62,7 @@ def test_line_magic(self): # Get the result back to Python self.ip.run_cell_magic('matlab', '-o actual', 'actual = res') - self.ip.run_cell("expected = np.array([2, 4, 6])") + self.ip.run_cell("expected = np.array([[2., 4., 6.]])") npt.assert_almost_equal(self.ip.user_ns['actual'], self.ip.user_ns['expected'], decimal=7) @@ -82,7 +82,7 @@ def test_matrix(self): # Matlab struct type should be converted to a Python dict def test_struct(self): self.ip.run_cell('num = 2.567') - self.ip.run_cell('num_array = np.array([1.2,3.4,5.6])') + self.ip.run_cell('num_array = np.array([1.2,3.4,5.6]).reshape(3,1)') self.ip.run_cell('str = "Hello World"') self.ip.run_cell_magic('matlab', '-i num,num_array,str -o obj', 'obj.num = num;obj.num_array = num_array;obj.str = str;') From 7324db80dec5c2da87ac2be38d48e04fc447c432 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 17 Jan 2015 10:48:13 -0600 Subject: [PATCH 10/12] Add support for complex arrays --- .../matlab/util/json_v0.2.2/json/json_dump.m | 9 ++- .../matlab/util/json_v0.2.2/json/json_load.m | 18 +++--- pymatbridge/pymatbridge.py | 61 +++++++++++++------ 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m index 672f3a7..ed8ef62 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m @@ -101,12 +101,17 @@ elseif ~isscalar(value) obj = javaObject('org.json.JSONArray'); - if isnumeric(value) && isreal(value) + if isnumeric(value) % encode arrays as a struct double_struct = struct; double_struct.ndarray = 1; value = double(value); - double_struct.data = base64encode(typecast(value(:), 'uint8')); + if isreal(value) + double_struct.data = base64encode(typecast(value(:), 'uint8')); + else + double_struct.real = base64encode(typecast(real(value(:)), 'uint8')); + double_struct.imag = base64encode(typecast(imag(value(:)), 'uint8')); + end double_struct.shape = base64encode(typecast(size(value), 'uint8')); obj = dump_data_(double_struct, options); elseif ndims(value) > 2 diff --git a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m index 428e666..ed2660f 100644 --- a/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m +++ b/pymatbridge/matlab/util/json_v0.2.2/json/json_load.m @@ -122,15 +122,19 @@ value.(safe_field) = parse_data_(node.get(javaObject('java.lang.String', key)), ... options); end - % Check if the struct just decoded represents a complex number - if isfield(value,'real') && isfield(value, 'imag') - complex_value = complex(value.real, value.imag); - value = complex_value; - end - % Check if the struct just decoded represents a numpy array + % Check if the struct just decoded represents an array or complex number if isfield(value,'ndarray') && isfield(value, 'shape') - arr = typecast(base64decode(value.data), 'double'); + if isfield(value, 'data') + arr = typecast(base64decode(value.data), 'double'); + else + r = typecast(base64decode(value.real), 'double'); + im = typecast(base64decode(value.imag), 'double'); + arr = complex(r, im); + end value = reshape(arr, value.shape); + elseif isfield(value,'real') && isfield(value, 'imag') + complex_value = complex(value.real, value.imag); + value = complex_value; end % In MATLAB, nested classes end up with a $ in the name, in Octave it's a . elseif isa(node, 'org.json.JSONObject$Null') || isa(node, 'org.json.JSONObject.Null') diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 0a6c0fa..322b20e 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -32,41 +32,62 @@ class spmatrix: pass +def encode_ndarray(obj): + """Write a numpy array and its shape to base64 buffers""" + shape = obj.shape + if len(shape) == 1: + shape = (obj.shape[0], 1) + if obj.flags.c_contiguous: + obj = obj.T + elif not obj.flags.f_contiguous: + obj = asfortranarray(obj) + data = obj.astype(float64).tobytes() + data = codecs.encode(data, 'base64').decode('utf-8') + return data, shape + + # JSON encoder extension to handle complex numbers and numpy arrays class PymatEncoder(json.JSONEncoder): def default(self, obj): - if isinstance(obj, complex): - return {'real': obj.real, 'imag': obj.imag} - if isinstance(obj, ndarray) and not obj.dtype.kind == 'c': - shape = obj.shape - if len(shape) == 1: - shape = (obj.shape[0], 1) - if obj.flags.c_contiguous: - obj = obj.T - elif not obj.flags.f_contiguous: - obj = asfortranarray(obj) - data = obj.astype(float64).tobytes() - data = codecs.encode(data, 'base64').decode('utf-8') + if isinstance(obj, ndarray) and obj.dtype.kind in 'uif': + data, shape = encode_ndarray(obj) return {'ndarray': True, 'shape': shape, 'data': data} + elif isinstance(obj, ndarray) and obj.dtype.kind == 'c': + real, shape = encode_ndarray(obj.real.copy()) + imag, _ = encode_ndarray(obj.imag.copy()) + return {'ndarray': True, 'shape': shape, + 'real': real, 'imag': imag} elif isinstance(obj, ndarray): return obj.tolist() - if isinstance(obj, generic): + elif isinstance(obj, complex): + return {'real': obj.real, 'imag': obj.imag} + elif isinstance(obj, generic): return obj.item() # Handle the default case return json.JSONEncoder.default(self, obj) -# JSON decoder for complex numbers +def decode_arr(data): + """Extract a numpy array from a base64 buffer""" + data = data.encode('utf-8') + return frombuffer(codecs.decode(data, 'base64'), float64) + + +# JSON decoder for arrays and complex numbers def as_complex(dct): - if 'real' in dct and 'imag' in dct: - return complex(dct['real'], dct['imag']) if 'ndarray' in dct and 'data' in dct: - data = dct['data'].encode('utf-8') - shape = dct['shape'].encode('utf-8') - value = frombuffer(codecs.decode(data, 'base64'), float64) - shape = frombuffer(codecs.decode(shape, 'base64'), float64) + value = decode_arr(dct['data']) + shape = decode_arr(dct['shape']) return value.reshape(shape, order='F') + elif 'ndarray' in dct and 'imag' in dct: + real = decode_arr(dct['real']) + imag = decode_arr(dct['imag']) + shape = decode_arr(dct['shape']) + data = real + 1j * imag + return data.reshape(shape, order='F') + elif 'real' in dct and 'imag' in dct: + return complex(dct['real'], dct['imag']) return dct MATLAB_FOLDER = '%s/matlab' % os.path.realpath(os.path.dirname(__file__)) From e0c83a9c9a255468741a5557e3373db255c420e1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 19 Jan 2015 19:17:49 -0600 Subject: [PATCH 11/12] Rename as_complex->decode_pymat and update struct test --- pymatbridge/pymatbridge.py | 4 ++-- pymatbridge/tests/test_magic.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 322b20e..b592030 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -75,7 +75,7 @@ def decode_arr(data): # JSON decoder for arrays and complex numbers -def as_complex(dct): +def decode_pymat(dct): if 'ndarray' in dct and 'data' in dct: value = decode_arr(dct['data']) shape = decode_arr(dct['shape']) @@ -243,7 +243,7 @@ def is_function_processor_working(self): def _json_response(self, **kwargs): if self.running: time.sleep(0.05) - return json.loads(self._response(**kwargs), object_hook=as_complex) + return json.loads(self._response(**kwargs), object_hook=decode_pymat) # Run a function in Matlab and return the result def run_func(self, func_path, func_args=None): diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 3616825..57e3677 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -82,11 +82,11 @@ def test_matrix(self): # Matlab struct type should be converted to a Python dict def test_struct(self): self.ip.run_cell('num = 2.567') - self.ip.run_cell('num_array = np.array([1.2,3.4,5.6]).reshape(3,1)') + self.ip.run_cell('num_array = np.array([1.2,3.4,5.6])') self.ip.run_cell('str = "Hello World"') self.ip.run_cell_magic('matlab', '-i num,num_array,str -o obj', 'obj.num = num;obj.num_array = num_array;obj.str = str;') npt.assert_equal(isinstance(self.ip.user_ns['obj'], dict), True) npt.assert_equal(self.ip.user_ns['obj']['num'], self.ip.user_ns['num']) - npt.assert_equal(self.ip.user_ns['obj']['num_array'], self.ip.user_ns['num_array']) + npt.assert_equal(self.ip.user_ns['obj']['num_array'].squeeze(), self.ip.user_ns['num_array']) npt.assert_equal(self.ip.user_ns['obj']['str'], self.ip.user_ns['str']) From 2e295d2af09846ec1b7427bf35abe5026ceba6a3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 19 Jan 2015 19:48:40 -0600 Subject: [PATCH 12/12] Use row vector order when passing arrays --- pymatbridge/pymatbridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index b592030..4aad82e 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -36,7 +36,7 @@ def encode_ndarray(obj): """Write a numpy array and its shape to base64 buffers""" shape = obj.shape if len(shape) == 1: - shape = (obj.shape[0], 1) + shape = (1, obj.shape[0]) if obj.flags.c_contiguous: obj = obj.T elif not obj.flags.f_contiguous: