From 158cac19ada3682771b31ff261cda7278fe93baf Mon Sep 17 00:00:00 2001
From: Stefano Borini <sborini@enthought.com>
Date: Fri, 8 Apr 2016 16:48:25 +0100
Subject: [PATCH 1/6] Handle numpy pure scalar types

spawned from https://github.com/enthought/mayavi/issues/61
The state pickler/unpickler could not handle numpy scalar types,
which were forced to None. This meant that trait classes having
Float() trait hosting a numpy.float64 could not be pickled
(to be precise they were pickled, with the content being None, silently).
---
 apptools/persistence/state_pickler.py         | 46 +++++++++++++++++--
 .../persistence/tests/test_state_pickler.py   |  5 ++
 2 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/apptools/persistence/state_pickler.py b/apptools/persistence/state_pickler.py
index aed0bdcb2..fa6211b1c 100644
--- a/apptools/persistence/state_pickler.py
+++ b/apptools/persistence/state_pickler.py
@@ -100,10 +100,10 @@
 # Standard library imports.
 import base64
 import sys
-import types
 import pickle
 import gzip
 from io import BytesIO, StringIO
+import warnings
 
 import numpy
 
@@ -139,6 +139,23 @@ def gunzip_string(data):
     writer.close()
     return data
 
+
+def base64_encode(data):
+    if PY_VER > 2:
+        base64.encodebytes(data)
+    else:
+        return base64.encodestring(data)
+
+
+def base64_decode(data):
+    if PY_VER > 2:
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+        return base64.decodebytes(data)
+    else:
+        return data.decode('base64')
+
+
 class StatePicklerError(Exception):
     pass
 
@@ -330,6 +347,8 @@ def _do(self, obj):
             return self._do_reference(obj)
         elif obj_type in self.type_map:
             return self.type_map[obj_type](obj)
+        elif isinstance(obj, numpy.generic):
+            return self._do_numpy_generic_type(obj)
         elif isinstance(obj, tuple):
             # Takes care of StateTuples.
             return self._do_tuple(obj)
@@ -341,8 +360,19 @@ def _do(self, obj):
             return self._do_dict(obj)
         elif hasattr(obj, '__dict__'):
             return self._do_instance(obj)
+        else:
+            warnings.warn("Cannot pickle unrecognized type {}. Returning None"
+                          " for backward compatibility.".format(obj_type))
+            return None
 
     def _get_id(self, value):
+        # We consider a special case for numpy scalar values, because
+        # they hash as native types, but they are special because we
+        # want to recover their true type, and the only way of doing so
+        # is to consider them as objects.
+        if isinstance(value, numpy.generic):
+            return id(value)
+
         try:
             key = hash(value)
         except TypeError:
@@ -445,12 +475,13 @@ def _do_dict(self, value):
 
     def _do_numeric(self, value):
         idx = self._register(value)
-        if PY_VER > 2:
-            data = base64.encodebytes(gzip_string(numpy.ndarray.dumps(value)))
-        else:
-            data = base64.encodestring(gzip_string(numpy.ndarray.dumps(value)))
+        data = base64_encode(gzip_string(numpy.ndarray.dumps(value)))
         return dict(type='numeric', id=idx, data=data)
 
+    def _do_numpy_generic_type(self, value):
+        idx = self._register(value)
+        data = base64_encode(pickle.dumps(value))
+        return dict(type='numpy', id=idx, data=data)
 
 
 ######################################################################
@@ -502,6 +533,7 @@ def __init__(self):
                          'list': self._do_list,
                          'dict': self._do_dict,
                          'numeric': self._do_numeric,
+                         'numpy': self._do_numpy_generic_type,
                          }
 
     def load_state(self, file):
@@ -666,6 +698,10 @@ def _do_numeric(self, value, path):
         self._obj_cache[value['id']] = result
         return result
 
+    def _do_numpy_generic_type(self, value, path):
+        result = pickle.loads(base64_decode(value["data"]))
+        self._obj_cache[value['id']] = result
+        return result
 
 ######################################################################
 # `StateSetter` class
diff --git a/apptools/persistence/tests/test_state_pickler.py b/apptools/persistence/tests/test_state_pickler.py
index 70bf42573..e9ab412b3 100644
--- a/apptools/persistence/tests/test_state_pickler.py
+++ b/apptools/persistence/tests/test_state_pickler.py
@@ -44,6 +44,7 @@ def __init__(self):
         self.i = 7
         self.l = 1234567890123456789
         self.f = math.pi
+        self.fnumpy = numpy.float64(3.0)
         self.c = complex(1.01234, 2.3)
         self.n = None
         self.s = 'String'
@@ -65,6 +66,7 @@ class TestTraits(HasTraits):
     i = Int(7)
     l = Long(12345678901234567890)
     f = Float(math.pi)
+    fnumpy = Float(numpy.float64(3.0))
     c = Complex(complex(1.01234, 2.3))
     n = Any
     s = Str('String')
@@ -168,6 +170,9 @@ def verify_unpickled(self, obj, state):
         self.assertEqual(state.i, obj.i)
         self.assertEqual(state.l, obj.l)
         self.assertEqual(state.f, obj.f)
+        self.assertEqual(state.fnumpy, obj.fnumpy)
+        self.assertIsInstance(obj.fnumpy, numpy.generic)
+        self.assertEqual(type(state.fnumpy), type(obj.fnumpy))
         self.assertEqual(state.c, obj.c)
         self.assertEqual(state.n, obj.n)
         self.assertEqual(state.s, obj.s)

From 70639d81131fefef112dae327d550e70d1805075 Mon Sep 17 00:00:00 2001
From: Stefano Borini <sborini@enthought.com>
Date: Thu, 21 Apr 2016 11:02:13 +0100
Subject: [PATCH 2/6] Fix for tests

---
 apptools/persistence/state_pickler.py  | 2 +-
 apptools/preferences/tests/example.ini | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apptools/persistence/state_pickler.py b/apptools/persistence/state_pickler.py
index fa6211b1c..b63193eef 100644
--- a/apptools/persistence/state_pickler.py
+++ b/apptools/persistence/state_pickler.py
@@ -142,7 +142,7 @@ def gunzip_string(data):
 
 def base64_encode(data):
     if PY_VER > 2:
-        base64.encodebytes(data)
+        return base64.encodebytes(data)
     else:
         return base64.encodestring(data)
 
diff --git a/apptools/preferences/tests/example.ini b/apptools/preferences/tests/example.ini
index 100dc6e16..9b2f1311d 100644
--- a/apptools/preferences/tests/example.ini
+++ b/apptools/preferences/tests/example.ini
@@ -1,10 +1,10 @@
 [acme.ui]
-bgcolor = blue
-width = 50
 ratio = 1.0
-visible = True
-description = 'acme ui'
+description = acme ui
+width = 50
 offsets = "[1, 2, 3, 4]"
+bgcolor = blue
+visible = True
 names = "['joe', 'fred', 'jane']"
 
 [acme.ui.splash_screen]

From 755f6cdf23cbc0db2d32df7b82fca4d6e7bbf528 Mon Sep 17 00:00:00 2001
From: Ioannis Tziakos <ioannist@enthought.com>
Date: Fri, 10 Jun 2016 16:15:38 +0100
Subject: [PATCH 3/6] make sure that the numpy scalar object does not get
 grarbage collected

---
 apptools/persistence/state_pickler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apptools/persistence/state_pickler.py b/apptools/persistence/state_pickler.py
index b63193eef..b642d1152 100644
--- a/apptools/persistence/state_pickler.py
+++ b/apptools/persistence/state_pickler.py
@@ -480,6 +480,7 @@ def _do_numeric(self, value):
 
     def _do_numpy_generic_type(self, value):
         idx = self._register(value)
+        self._misc_cache.append(value)
         data = base64_encode(pickle.dumps(value))
         return dict(type='numpy', id=idx, data=data)
 

From b299f9f8def09782cb3eb1aca75b36546f2205a9 Mon Sep 17 00:00:00 2001
From: Ioannis Tziakos <ioannist@enthought.com>
Date: Fri, 10 Jun 2016 16:15:50 +0100
Subject: [PATCH 4/6] add a small test

---
 .../persistence/tests/test_state_pickler.py   | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/apptools/persistence/tests/test_state_pickler.py b/apptools/persistence/tests/test_state_pickler.py
index e9ab412b3..29baee017 100644
--- a/apptools/persistence/tests/test_state_pickler.py
+++ b/apptools/persistence/tests/test_state_pickler.py
@@ -469,5 +469,28 @@ def test_dump_to_file_str(self):
             os.remove(filepath)
 
 
+class TestStatePickler(unittest.TestCase):
+
+    def setUp(self):
+        self.pickler = state_pickler.StatePickler()
+
+    def tearDown(self):
+        self.pickler = None
+
+    def test_on_base_types(self):
+        state = self.pickler.dump_state(1)
+        self.assertEqual(state, 1)
+
+    def test_on_lists(self):
+        l = [1,2.0, None, [1,2,3]]
+        state = self.pickler.dump_state(l)
+        self.assertEqual(
+            state,
+            {'data': [1, 2.0, None, {'data': [1, 2, 3], 'type': 'list', 'id': 1}],
+             'id': 0,
+             'type': 'list'})
+
+
+
 if __name__ == "__main__":
     unittest.main()

From 44fee2faf0a3c43d0c072fcff7858076062fa2e8 Mon Sep 17 00:00:00 2001
From: Ioannis Tziakos <ioannist@enthought.com>
Date: Fri, 10 Jun 2016 16:45:45 +0100
Subject: [PATCH 5/6] add specific test for numpy_scalars

---
 apptools/persistence/tests/test_state_pickler.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/apptools/persistence/tests/test_state_pickler.py b/apptools/persistence/tests/test_state_pickler.py
index 29baee017..c5393149f 100644
--- a/apptools/persistence/tests/test_state_pickler.py
+++ b/apptools/persistence/tests/test_state_pickler.py
@@ -490,6 +490,11 @@ def test_on_lists(self):
              'id': 0,
              'type': 'list'})
 
+    def test_on_numpy_scalars(self):
+        state = self.pickler.dumps(numpy.int32(78))
+        loaded_state = state_pickler.StateUnpickler().loads_state(state)
+        self.assertEqual(loaded_state, 78)
+        self.assertEqual(loaded_state.dtype, numpy.int32)
 
 
 if __name__ == "__main__":

From daf1b79cb1a212231466e1241473d96050b71fb5 Mon Sep 17 00:00:00 2001
From: Ioannis Tziakos <ioannist@enthought.com>
Date: Fri, 10 Jun 2016 16:46:06 +0100
Subject: [PATCH 6/6] use the numpy functions for serialization

---
 apptools/persistence/state_pickler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apptools/persistence/state_pickler.py b/apptools/persistence/state_pickler.py
index b642d1152..e4f327dc1 100644
--- a/apptools/persistence/state_pickler.py
+++ b/apptools/persistence/state_pickler.py
@@ -481,7 +481,7 @@ def _do_numeric(self, value):
     def _do_numpy_generic_type(self, value):
         idx = self._register(value)
         self._misc_cache.append(value)
-        data = base64_encode(pickle.dumps(value))
+        data = base64_encode(value.dumps())
         return dict(type='numpy', id=idx, data=data)
 
 
@@ -700,7 +700,7 @@ def _do_numeric(self, value, path):
         return result
 
     def _do_numpy_generic_type(self, value, path):
-        result = pickle.loads(base64_decode(value["data"]))
+        result = numpy.loads(base64_decode(value["data"]))
         self._obj_cache[value['id']] = result
         return result