consistent string hashing

Author: jreback

pandas/core/base.py

@@ -16,7 +16,6 @@
 from pandas.util.decorators import (Appender, cache_readonly,
                                     deprecate_kwarg, Substitution)
 from pandas.core.common import AbstractMethodError
-from pandas.tools.hashing import hash_pandas_object
 from pandas.formats.printing import pprint_thing
 
 _shared_docs = dict()
@@ -838,6 +837,7 @@ def hash(self, index=True):
                9751253963311919054], dtype=uint64)
 
         """
+        from pandas.tools.hashing import hash_pandas_object
         return hash_pandas_object(self, index=index)
 
 

pandas/tools/hashing.py

@@ -2,7 +2,9 @@
 data hash pandas / numpy objects
 """
 
+from hashlib import md5
 import numpy as np
+from pandas import Series
 from pandas.types.generic import ABCIndexClass, ABCSeries, ABCDataFrame
 from pandas.types.common import is_categorical_dtype
 
@@ -71,7 +73,13 @@ def hash_array(vals):
 
         vals = vals.view('u{}'.format(vals.dtype.itemsize)).astype('u8')
     else:
-        vals = np.array([hash(x) for x in vals], dtype=np.uint64)
+
+        # we want to stringify
+        # then apply a consistent hashing scheme
+        def f(v):
+            return int(md5(v).hexdigest(), 16) % (10 ** 8)
+        vals = Series(vals).astype(str).str.encode('utf8').values
+        vals = np.array([f(v) for v in vals], dtype='uint64')
 
     # Then, redistribute these 64-bit ints within the space of 64-bit ints
     vals ^= vals >> 30