Skip to content

Efficient Array Passing #103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arokem merged 12 commits into from
Jan 20, 2015
Merged

Efficient Array Passing #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
e34c10d
Implement efficient handling of large array passing to Matlab
blink1073 Jan 16, 2015
0af4ccd
Use base64 encoding for large arrays
blink1073 Jan 17, 2015
516b5d8
Finish implementing efficient arrays
blink1073 Jan 17, 2015
ebb1c2b
Add guard for cells
blink1073 Jan 17, 2015
db15e5b
Add handling of Fortran order
blink1073 Jan 17, 2015
7710340
Handle all arrays on the matlab end
blink1073 Jan 17, 2015
350519f
No need to convert to list
blink1073 Jan 17, 2015
7a78e32
Update test to reflect floating point
blink1073 Jan 17, 2015
0bd1ece
Update tests for floating point and explicit shape
blink1073 Jan 17, 2015
7324db8
Add support for complex arrays
blink1073 Jan 17, 2015
e0c83a9
Rename as_complex->decode_pymat and update struct test
blink1073 Jan 20, 2015
2e295d2
Use row vector order when passing arrays
blink1073 Jan 20, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 176 additions & 2 deletions pymatbridge/matlab/util/json_v0.2.2/json/json_dump.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@
% mapped to the same json string '[1,2]'.
%
% See also json.load json.write

json_startup('WarnOnAddpath', true);
options = get_options_(varargin{:});
obj = dump_data_(value, options);
Expand Down Expand Up @@ -102,7 +101,20 @@
elseif ~isscalar(value)
obj = javaObject('org.json.JSONArray');

if ndims(value) > 2
if isnumeric(value)
% encode arrays as a struct
double_struct = struct;
double_struct.ndarray = 1;
value = double(value);
if isreal(value)
double_struct.data = base64encode(typecast(value(:), 'uint8'));
else
double_struct.real = base64encode(typecast(real(value(:)), 'uint8'));
double_struct.imag = base64encode(typecast(imag(value(:)), 'uint8'));
end
double_struct.shape = base64encode(typecast(size(value), 'uint8'));
obj = dump_data_(double_struct, options);
elseif ndims(value) > 2
split_value = num2cell(value, 1:ndims(value)-1);
for i = 1:numel(split_value)
obj.put(dump_data_(split_value{i}, options));
Expand Down Expand Up @@ -151,3 +163,165 @@
error('json:typeError', 'Unsupported data type: %s', class(value));
end
end


function y = base64encode(x, eol)
%BASE64ENCODE Perform base64 encoding on a string.
%
% BASE64ENCODE(STR, EOL) encode the given string STR. EOL is the line ending
% sequence to use; it is optional and defaults to '\n' (ASCII decimal 10).
% The returned encoded string is broken into lines of no more than 76
% characters each, and each line will end with EOL unless it is empty. Let
% EOL be empty if you do not want the encoded string broken into lines.
%
% STR and EOL don't have to be strings (i.e., char arrays). The only
% requirement is that they are vectors containing values in the range 0-255.
%
% This function may be used to encode strings into the Base64 encoding
% specified in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The
% Base64 encoding is designed to represent arbitrary sequences of octets in a
% form that need not be humanly readable. A 65-character subset
% ([A-Za-z0-9+/=]) of US-ASCII is used, enabling 6 bits to be represented per
% printable character.
%
% Examples
% --------
%
% If you want to encode a large file, you should encode it in chunks that are
% a multiple of 57 bytes. This ensures that the base64 lines line up and
% that you do not end up with padding in the middle. 57 bytes of data fills
% one complete base64 line (76 == 57*4/3):
%
% If ifid and ofid are two file identifiers opened for reading and writing,
% respectively, then you can base64 encode the data with
%
% while ~feof(ifid)
% fwrite(ofid, base64encode(fread(ifid, 60*57)));
% end
%
% or, if you have enough memory,
%
% fwrite(ofid, base64encode(fread(ifid)));
%
% See also BASE64DECODE.

% Author: Peter J. Acklam
Copy link
Owner

@arokem arokem Jan 19, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think Peter is OK with us using his code? Did the original have a license attached to it?

Copy link
Collaborator Author

@blink1073 blink1073 Jan 20, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

http://www.fourmilab.ch/webtools/base64/

"This software is in the public domain. Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, without any conditions or restrictions. This software is provided “as is” without express or implied warranty. "

% Time-stamp: 2004-02-03 21:36:56 +0100
% E-mail: pjacklam@online.no
% URL: http://home.online.no/~pjacklam

% check number of input arguments
error(nargchk(1, 2, nargin));

% make sure we have the EOL value
if nargin < 2
eol = ''; %sprintf('\n');
else
if sum(size(eol) > 1) > 1
error('EOL must be a vector.');
end
if any(eol(:) > 255)
error('EOL can not contain values larger than 255.');
end
end

if sum(size(x) > 1) > 1
error('STR must be a vector.');
end

x = uint8(x);
eol = uint8(eol);

ndbytes = length(x); % number of decoded bytes
nchunks = ceil(ndbytes / 3); % number of chunks/groups
nebytes = 4 * nchunks; % number of encoded bytes

% add padding if necessary, to make the length of x a multiple of 3
if rem(ndbytes, 3)
x(end+1 : 3*nchunks) = 0;
end

x = reshape(x, [3, nchunks]); % reshape the data
y = repmat(uint8(0), 4, nchunks); % for the encoded data

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Split up every 3 bytes into 4 pieces
%
% aaaaaabb bbbbcccc ccdddddd
%
% to form
%
% 00aaaaaa 00bbbbbb 00cccccc 00dddddd
%
y(1,:) = bitshift(x(1,:), -2); % 6 highest bits of x(1,:)

y(2,:) = bitshift(bitand(x(1,:), 3), 4); % 2 lowest bits of x(1,:)
y(2,:) = bitor(y(2,:), bitshift(x(2,:), -4)); % 4 highest bits of x(2,:)

y(3,:) = bitshift(bitand(x(2,:), 15), 2); % 4 lowest bits of x(2,:)
y(3,:) = bitor(y(3,:), bitshift(x(3,:), -6)); % 2 highest bits of x(3,:)

y(4,:) = bitand(x(3,:), 63); % 6 lowest bits of x(3,:)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Now perform the following mapping
%
% 0 - 25 -> A-Z
% 26 - 51 -> a-z
% 52 - 61 -> 0-9
% 62 -> +
% 63 -> /
%
% We could use a mapping vector like
%
% ['A':'Z', 'a':'z', '0':'9', '+/']
%
% but that would require an index vector of class double.
%
z = repmat(uint8(0), size(y));
i = y <= 25; z(i) = 'A' + double(y(i));
i = 26 <= y & y <= 51; z(i) = 'a' - 26 + double(y(i));
i = 52 <= y & y <= 61; z(i) = '0' - 52 + double(y(i));
i = y == 62; z(i) = '+';
i = y == 63; z(i) = '/';
y = z;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Add padding if necessary.
%
npbytes = 3 * nchunks - ndbytes; % number of padding bytes
if npbytes
y(end-npbytes+1 : end) = '='; % '=' is used for padding
end

if isempty(eol)

% reshape to a row vector
y = reshape(y, [1, nebytes]);

else

nlines = ceil(nebytes / 76); % number of lines
neolbytes = length(eol); % number of bytes in eol string

% pad data so it becomes a multiple of 76 elements
y(nebytes + 1 : 76 * nlines) = 0;
y = reshape(y, 76, nlines);

% insert eol strings
eol = eol(:);
y(end + 1 : end + neolbytes, :) = eol(:, ones(1, nlines));

% remove padding, but keep the last eol string
m = nebytes + neolbytes * (nlines - 1);
n = (76+neolbytes)*nlines - neolbytes;
y(m+1 : n) = '';

% extract and reshape to row vector
y = reshape(y, 1, m+neolbytes);

end

% output is a character array
y = char(y);
end
102 changes: 100 additions & 2 deletions pymatbridge/matlab/util/json_v0.2.2/json/json_load.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,17 @@
value.(safe_field) = parse_data_(node.get(javaObject('java.lang.String', key)), ...
options);
end
% Check if the struct just decoded represents a complex number
if isfield(value,'real') && isfield(value, 'imag')
% Check if the struct just decoded represents an array or complex number
if isfield(value,'ndarray') && isfield(value, 'shape')
if isfield(value, 'data')
arr = typecast(base64decode(value.data), 'double');
else
r = typecast(base64decode(value.real), 'double');
im = typecast(base64decode(value.imag), 'double');
arr = complex(r, im);
end
value = reshape(arr, value.shape);
elseif isfield(value,'real') && isfield(value, 'imag')
complex_value = complex(value.real, value.imag);
value = complex_value;
end
Expand Down Expand Up @@ -198,3 +207,92 @@
vec = [vec, uint8([fields{:}])];
end
end



function y = base64decode(x)
%BASE64DECODE Perform base64 decoding on a string.
%
% BASE64DECODE(STR) decodes the given base64 string STR.
%
% Any character not part of the 65-character base64 subset set is silently
% ignored.
%
% This function is used to decode strings from the Base64 encoding specified
% in RFC 2045 - MIME (Multipurpose Internet Mail Extensions). The Base64
% encoding is designed to represent arbitrary sequences of octets in a form
% that need not be humanly readable. A 65-character subset ([A-Za-z0-9+/=])
% of US-ASCII is used, enabling 6 bits to be represented per printable
% character.
%
% See also BASE64ENCODE.

% Author: Peter J. Acklam
% Time-stamp: 2004-09-20 08:20:50 +0200
% E-mail: pjacklam@online.no
% URL: http://home.online.no/~pjacklam

% Modified by Guillaume Flandin, May 2008

% check number of input arguments
%--------------------------------------------------------------------------

error(nargchk(1, 1, nargin));

% Perform the following mapping
%--------------------------------------------------------------------------
% A-Z -> 0 - 25 a-z -> 26 - 51 0-9 -> 52 - 61
% + -> 62 / -> 63 = -> 64
% anything else -> NaN

base64chars = NaN(1,256);
base64chars('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=') = 0:64;
x = base64chars(x);

% Remove/ignore any characters not in the base64 characters list or '='
%--------------------------------------------------------------------------

x = x(~isnan(x));

% Replace any incoming padding ('=' -> 64) with a zero pad
%--------------------------------------------------------------------------

if x(end-1) == 64, p = 2; x(end-1:end) = 0;
elseif x(end) == 64, p = 1; x(end) = 0;
else p = 0;
end

% Allocate decoded data array
%--------------------------------------------------------------------------

n = length(x) / 4; % number of groups
x = reshape(uint8(x), 4, n); % input data
y = zeros(3, n, 'uint8'); % decoded data

% Rearrange every 4 bytes into 3 bytes
%--------------------------------------------------------------------------
% 00aaaaaa 00bbbbbb 00cccccc 00dddddd
%
% to form
%
% aaaaaabb bbbbcccc ccdddddd

y(1,:) = bitshift(x(1,:), 2); % 6 highest bits of y(1,:)
y(1,:) = bitor(y(1,:), bitshift(x(2,:), -4)); % 2 lowest bits of y(1,:)

y(2,:) = bitshift(x(2,:), 4); % 4 highest bits of y(2,:)
y(2,:) = bitor(y(2,:), bitshift(x(3,:), -2)); % 4 lowest bits of y(2,:)

y(3,:) = bitshift(x(3,:), 6); % 2 highest bits of y(3,:)
y(3,:) = bitor(y(3,:), x(4,:)); % 6 lowest bits of y(3,:)

% Remove any zero pad that was added to make this a multiple of 24 bits
%--------------------------------------------------------------------------

if p, y(end-p+1:end) = []; end

% Reshape to a row vector
%--------------------------------------------------------------------------

y = reshape(y, 1, []);
end
Loading