From a5d1da927c31f237c5bd555fc4dfe7cd2bdcb585 Mon Sep 17 00:00:00 2001 From: Merlin Date: Thu, 26 May 2016 19:44:07 +0200 Subject: [PATCH 1/6] fix: JSON dump fails on numpy int, float and array types, so we need to convert them to standard types when dumping metadata. --- qcodes/data/gnuplot_format.py | 4 ++-- qcodes/utils/helpers.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/qcodes/data/gnuplot_format.py b/qcodes/data/gnuplot_format.py index decaa7d3574c..5190a005c5ce 100644 --- a/qcodes/data/gnuplot_format.py +++ b/qcodes/data/gnuplot_format.py @@ -3,7 +3,7 @@ import math import json -from qcodes.utils.helpers import deep_update +from qcodes.utils.helpers import deep_update, json_encoder from .data_array import DataArray from .format import Formatter @@ -297,7 +297,7 @@ def write_metadata(self, data_set, read_first=True): fn = io_manager.join(location, self.metadata_file) with io_manager.open(fn, 'w', encoding='utf8') as snap_file: json.dump(data_set.metadata, snap_file, sort_keys=True, - indent=4, ensure_ascii=False) + indent=4, ensure_ascii=False, cls=json_encoder) def read_metadata(self, data_set): io_manager = data_set.io diff --git a/qcodes/utils/helpers.py b/qcodes/utils/helpers.py index b00bc368a99a..d174ed741f2d 100644 --- a/qcodes/utils/helpers.py +++ b/qcodes/utils/helpers.py @@ -6,10 +6,27 @@ import sys import io import numpy as np +import json _tprint_times = {} +class json_encoder(json.JSONEncoder): + """ + Return numpy types as standard types. + http://stackoverflow.com/questions/27050108/convert-numpy-type-to-python + http://stackoverflow.com/questions/9452775/converting-numpy-dtypes-to-native-python-types/11389998#11389998 + """ + def default(self, obj): + if isinstance(obj, np.integer): + return int(obj) + elif isinstance(obj, np.floating): + return float(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + else: + return super(json_encoder, self).default(obj) + def tprint(string, dt=1, tag='default'): """ Print progress of a loop every dt seconds """ ptime = _tprint_times.get(tag, 0) From 1a31c1e77fd7abda593c5efb1968919a815b1389 Mon Sep 17 00:00:00 2001 From: Merlin Date: Thu, 26 May 2016 21:04:04 +0200 Subject: [PATCH 2/6] test: json --- qcodes/tests/test_json.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 qcodes/tests/test_json.py diff --git a/qcodes/tests/test_json.py b/qcodes/tests/test_json.py new file mode 100644 index 000000000000..7b33550d677d --- /dev/null +++ b/qcodes/tests/test_json.py @@ -0,0 +1,35 @@ +from unittest import TestCase +import numpy as np +import json +from copy import deepcopy + +from qcodes.utils.helpers import json_encoder + + +class TestNumpyJson(TestCase): + + metadata = {'name': 'Rapunzel', 'age': 12, 'height': 112.234, + 'scores': np.linspace(0,42)} + + def test_numpy(self): + metadata = deepcopy(self.metadata) + metadata['scores'] = list(range(10)) + data = json.dumps(metadata, sort_keys=True, indent=4, + ensure_ascii=False) + data_dict = json.loads(data) + self.assertEqual(metadata, data_dict) + + def test_numpy_fail(self): + metadata = self.metadata + with self.assertRaises(TypeError): + json.dumps(metadata, sort_keys=True, indent=4, ensure_ascii=False) + + + def test_numpy_good(self): + metadata = self.metadata + data = json.dumps(metadata, sort_keys=True, indent=4, + ensure_ascii=False, cls=json_encoder) + data_dict = json.loads(data) + + # fails + # self.assertEqual(metadata, data_dict) From 104382d0417bde3a9cf4e055bbaba16cd927ce45 Mon Sep 17 00:00:00 2001 From: Merlin Date: Thu, 26 May 2016 21:16:21 +0200 Subject: [PATCH 3/6] fix: more test --- qcodes/tests/test_json.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/qcodes/tests/test_json.py b/qcodes/tests/test_json.py index 7b33550d677d..99bc41baf3ba 100644 --- a/qcodes/tests/test_json.py +++ b/qcodes/tests/test_json.py @@ -1,19 +1,19 @@ from unittest import TestCase import numpy as np import json -from copy import deepcopy from qcodes.utils.helpers import json_encoder class TestNumpyJson(TestCase): - metadata = {'name': 'Rapunzel', 'age': 12, 'height': 112.234, - 'scores': np.linspace(0,42)} + metadata = {'name': 'Rapunzel', 'age': np.int64(12), 'height': np.float64(112.234), + 'scores': np.linspace(0,42,num=3)} def test_numpy(self): - metadata = deepcopy(self.metadata) - metadata['scores'] = list(range(10)) + + metadata = {'name': 'Rapunzel', 'age': 12, 'height': 112.234, + 'scores': list(range(10))} data = json.dumps(metadata, sort_keys=True, indent=4, ensure_ascii=False) data_dict = json.loads(data) @@ -24,12 +24,13 @@ def test_numpy_fail(self): with self.assertRaises(TypeError): json.dumps(metadata, sort_keys=True, indent=4, ensure_ascii=False) - def test_numpy_good(self): metadata = self.metadata data = json.dumps(metadata, sort_keys=True, indent=4, ensure_ascii=False, cls=json_encoder) data_dict = json.loads(data) - # fails - # self.assertEqual(metadata, data_dict) + metadata = {'name': 'Rapunzel', 'age': 12, 'height': 112.234, + 'scores': [0, 21, 42]} + + self.assertEqual(metadata, data_dict) From 456066e3e2c9d18b622b1aaf1a613fa0b14fecc4 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Thu, 26 May 2016 21:59:57 +0200 Subject: [PATCH 4/6] small tweaks to numpy json encoder tests --- qcodes/tests/test_json.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/qcodes/tests/test_json.py b/qcodes/tests/test_json.py index 99bc41baf3ba..b1824138b998 100644 --- a/qcodes/tests/test_json.py +++ b/qcodes/tests/test_json.py @@ -7,17 +7,18 @@ class TestNumpyJson(TestCase): - metadata = {'name': 'Rapunzel', 'age': np.int64(12), 'height': np.float64(112.234), - 'scores': np.linspace(0,42,num=3)} - - def test_numpy(self): - - metadata = {'name': 'Rapunzel', 'age': 12, 'height': 112.234, - 'scores': list(range(10))} - data = json.dumps(metadata, sort_keys=True, indent=4, - ensure_ascii=False) - data_dict = json.loads(data) - self.assertEqual(metadata, data_dict) + def setUp(self): + self.metadata = { + 'name': 'Rapunzel', + 'age': np.int64(12), + 'height': np.float64(112.234), + 'scores': np.linspace(0, 42, num=3), + # include some regular values to ensure they work right + # with our encoder + 'weight': 19, + 'length': 45.23, + 'points': [12, 24, 48] + } def test_numpy_fail(self): metadata = self.metadata @@ -30,7 +31,14 @@ def test_numpy_good(self): ensure_ascii=False, cls=json_encoder) data_dict = json.loads(data) - metadata = {'name': 'Rapunzel', 'age': 12, 'height': 112.234, - 'scores': [0, 21, 42]} + metadata = { + 'name': 'Rapunzel', + 'age': 12, + 'height': 112.234, + 'scores': [0, 21, 42], + 'weight': 19, + 'length': 45.23, + 'points': [12, 24, 48] + } self.assertEqual(metadata, data_dict) From 895c60a3dbb57c0c173c978f77cc9db472d0859a Mon Sep 17 00:00:00 2001 From: Merlin Date: Thu, 26 May 2016 22:01:53 +0200 Subject: [PATCH 5/6] rename: NumpyJSONEncoder --- qcodes/data/gnuplot_format.py | 4 ++-- qcodes/tests/test_json.py | 4 ++-- qcodes/utils/helpers.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qcodes/data/gnuplot_format.py b/qcodes/data/gnuplot_format.py index 5190a005c5ce..28052fd797cd 100644 --- a/qcodes/data/gnuplot_format.py +++ b/qcodes/data/gnuplot_format.py @@ -3,7 +3,7 @@ import math import json -from qcodes.utils.helpers import deep_update, json_encoder +from qcodes.utils.helpers import deep_update, NumpyJSONEncoder from .data_array import DataArray from .format import Formatter @@ -297,7 +297,7 @@ def write_metadata(self, data_set, read_first=True): fn = io_manager.join(location, self.metadata_file) with io_manager.open(fn, 'w', encoding='utf8') as snap_file: json.dump(data_set.metadata, snap_file, sort_keys=True, - indent=4, ensure_ascii=False, cls=json_encoder) + indent=4, ensure_ascii=False, cls=NumpyJSONEncoder) def read_metadata(self, data_set): io_manager = data_set.io diff --git a/qcodes/tests/test_json.py b/qcodes/tests/test_json.py index 99bc41baf3ba..173009f6749d 100644 --- a/qcodes/tests/test_json.py +++ b/qcodes/tests/test_json.py @@ -2,7 +2,7 @@ import numpy as np import json -from qcodes.utils.helpers import json_encoder +from qcodes.utils.helpers import NumpyJSONEncoder class TestNumpyJson(TestCase): @@ -27,7 +27,7 @@ def test_numpy_fail(self): def test_numpy_good(self): metadata = self.metadata data = json.dumps(metadata, sort_keys=True, indent=4, - ensure_ascii=False, cls=json_encoder) + ensure_ascii=False, cls=NumpyJSONEncoder) data_dict = json.loads(data) metadata = {'name': 'Rapunzel', 'age': 12, 'height': 112.234, diff --git a/qcodes/utils/helpers.py b/qcodes/utils/helpers.py index d174ed741f2d..dfc853eb62af 100644 --- a/qcodes/utils/helpers.py +++ b/qcodes/utils/helpers.py @@ -11,7 +11,7 @@ _tprint_times = {} -class json_encoder(json.JSONEncoder): +class NumpyJSONEncoder(json.JSONEncoder): """ Return numpy types as standard types. http://stackoverflow.com/questions/27050108/convert-numpy-type-to-python @@ -25,7 +25,7 @@ def default(self, obj): elif isinstance(obj, np.ndarray): return obj.tolist() else: - return super(json_encoder, self).default(obj) + return super(NumpyJSONEncoder, self).default(obj) def tprint(string, dt=1, tag='default'): """ Print progress of a loop every dt seconds """ From 9317f3d26220ca1334b8d3bed8d54e1dec423275 Mon Sep 17 00:00:00 2001 From: Merlin Date: Thu, 26 May 2016 23:05:53 +0200 Subject: [PATCH 6/6] doc: move links to comments --- qcodes/utils/helpers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/qcodes/utils/helpers.py b/qcodes/utils/helpers.py index dfc853eb62af..343f2eec63e8 100644 --- a/qcodes/utils/helpers.py +++ b/qcodes/utils/helpers.py @@ -12,11 +12,9 @@ class NumpyJSONEncoder(json.JSONEncoder): - """ - Return numpy types as standard types. - http://stackoverflow.com/questions/27050108/convert-numpy-type-to-python - http://stackoverflow.com/questions/9452775/converting-numpy-dtypes-to-native-python-types/11389998#11389998 - """ + """Return numpy types as standard types.""" + # http://stackoverflow.com/questions/27050108/convert-numpy-type-to-python + # http://stackoverflow.com/questions/9452775/converting-numpy-dtypes-to-native-python-types/11389998#11389998 def default(self, obj): if isinstance(obj, np.integer): return int(obj)