BUG Decode to UTF-8 the dtype string read from a hdf file (#31756)

Author: pedroreys

doc/source/whatsnew/v1.1.0.rst

@@ -185,6 +185,7 @@ I/O
 - Bug in :meth:`DataFrame.to_parquet` overwriting pyarrow's default for
   ``coerce_timestamps``; following pyarrow's default allows writing nanosecond
   timestamps with ``version="2.0"`` (:issue:`31652`).
+- Bug in :class:`HDFStore` that caused it to set to ``int64`` the dtype of a ``datetime64`` column when reading a DataFrame in Python 3 from fixed format written in Python 2 (:issue:`31750`)
 
 Plotting
 ^^^^^^^^

pandas/io/pytables.py

@@ -2722,7 +2722,7 @@ def read_array(
         if isinstance(node, tables.VLArray):
             ret = node[0][start:stop]
         else:
-            dtype = getattr(attrs, "value_type", None)
+            dtype = _ensure_decoded(getattr(attrs, "value_type", None))
             shape = getattr(attrs, "shape", None)
 
             if shape is not None:

pandas/tests/io/data/legacy_hdf/legacy_table_fixed_datetime_py2.h5

@@ -4074,6 +4074,21 @@ def test_legacy_table_fixed_format_read_py2(self, datapath, setup_path):
             )
             tm.assert_frame_equal(expected, result)
 
+    def test_legacy_table_fixed_format_read_datetime_py2(self, datapath, setup_path):
+        # GH 31750
+        # legacy table with fixed format and datetime64 column written in Python 2
+        with ensure_clean_store(
+            datapath("io", "data", "legacy_hdf", "legacy_table_fixed_datetime_py2.h5"),
+            mode="r",
+        ) as store:
+            result = store.select("df")
+            expected = pd.DataFrame(
+                [[pd.Timestamp("2020-02-06T18:00")]],
+                columns=["A"],
+                index=pd.Index(["date"]),
+            )
+            tm.assert_frame_equal(expected, result)
+
     def test_legacy_table_read_py2(self, datapath, setup_path):
         # issue: 24925
         # legacy table written in Python 2