diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c67e983..5771c046 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,28 +88,3 @@ jobs: - name: Stop the docker run: docker container stop ndts - python2_tests: - runs-on: ubuntu-latest - strategy: - matrix: - os: [debian10] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - - name: Build the docker - env: - OS: ${{ matrix.os }} - run: docker build -t ndts .ci/${OS}_py2 - - - name: Run the docker - run: docker run --name ndts -d -it -v `pwd`:/home/tango ndts - - - name: install python-pninexus - run: .ci/install.sh 2 - - - name: run tests - run: .ci/run.sh 2 - - - name: Stop the docker - run: docker container stop ndts diff --git a/src/pninexus/h5cpp/attribute/__init__.py b/src/pninexus/h5cpp/attribute/__init__.py index bed6ccc9..d49ba455 100644 --- a/src/pninexus/h5cpp/attribute/__init__.py +++ b/src/pninexus/h5cpp/attribute/__init__.py @@ -20,15 +20,28 @@ def attribute_write(self, data): write_data = data if not isinstance(write_data, numpy.ndarray): write_data = numpy.array(write_data) - if write_data.dtype.kind == 'U': - write_data = write_data.astype("S") + try: + write_data = write_data.astype("S") + except Exception: + if isinstance(data, numpy.ndarray) and data.shape: + shape = data.shape + if len(shape) > 1: + data = data.flatten() + write_data = numpy.array( + [bytes(str(dt).encode('utf-8')) for dt in data]) + if len(shape) > 1: + write_data = write_data.reshape(shape) + else: + write_data = numpy.array(str(data).encode('utf-8')) elif write_data.dtype == 'bool': write_data = write_data.astype("int8") + # print("DATA", data, write_data) try: self._write(write_data) - except RuntimeError: + except RuntimeError as e: + print(str(e)) print(write_data, write_data.dtype) diff --git a/src/pninexus/h5cpp/node/__init__.py b/src/pninexus/h5cpp/node/__init__.py index 24ef341d..e59278a8 100644 --- a/src/pninexus/h5cpp/node/__init__.py +++ b/src/pninexus/h5cpp/node/__init__.py @@ -275,8 +275,19 @@ def dataset_write(self, data, selection=None): # if the data is a unicode numpy array we have to convert it to a # simple string array if data.dtype.kind == 'U': - data = data.astype('S') - + try: + data = data.astype('S') + except Exception: + if isinstance(data, numpy.ndarray) and data.shape: + shape = data.shape + if len(shape) > 1: + data = data.flatten() + data = numpy.array( + [bytes(str(dt).encode('utf-8')) for dt in data]) + if len(shape) > 1: + data = data.reshape(shape) + else: + data = numpy.array(str(data).encode('utf-8')) # # determine memory datatype and dataspace # - if the file type is a variable length string we have to adjust the diff --git a/test/h5cpp_tests/attribute_tests/attribute_io_test.py b/test/h5cpp_tests/attribute_tests/attribute_io_test.py index fc619b94..6eb03ccc 100644 --- a/test/h5cpp_tests/attribute_tests/attribute_io_test.py +++ b/test/h5cpp_tests/attribute_tests/attribute_io_test.py @@ -118,7 +118,7 @@ def testStringScalarFixedLength(self): a = self.root.attributes.create("StringScalar", dtype) a.write("hello world") r = a.read() - self.assertEqual(r, "hello world") + self.assertEqual(r, data) def testStringScalarVariableLength(self): @@ -128,6 +128,16 @@ def testStringScalarVariableLength(self): r = a.read() self.assertEqual(r, data) + def testStringUTF8ScalarVariableLength(self): + + data = u"µm" + dtype = String.variable() + dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8 + a = self.root.attributes.create("StringUTF8ScalarVLength", dtype) + a.write(data) + r = a.read() + self.assertEqual(r, data) + def testStringArray(self): data = numpy.array([["hello", "world", "this"], ["is", "a", "test"]]) @@ -147,6 +157,17 @@ def testStringArrayVariableLength(self): r = a.read() npt.assert_array_equal(r, data) + def testStringUTF8ArrayVariableLength(self): + + data = numpy.array([u"µm", u"µA"]) + dtype = String.variable() + dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8 + a = self.root.attributes.create( + "StringUTF8ArrayVLength", dtype, (2,)) + a.write(data) + r = a.read() + npt.assert_array_equal(r, data) + def testIntArray(self): data = numpy.array([1, 2, 3]) diff --git a/test/h5cpp_tests/node_tests/dataset_io_test.py b/test/h5cpp_tests/node_tests/dataset_io_test.py index cf7cfeb3..adc155a8 100644 --- a/test/h5cpp_tests/node_tests/dataset_io_test.py +++ b/test/h5cpp_tests/node_tests/dataset_io_test.py @@ -115,6 +115,17 @@ def testWriteVariableLengthScalar(self): read = dataset.read() self.assertEqual(read, "hello world") + def testWriteVariableLengthUTF8Scalar(self): + data = u"µm" + dtype = h5cpp.datatype.String.variable() + dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8 + dataset = Dataset( + self.root, h5cpp.Path("VariableLengthStringUTF8Scalar"), + dtype, Scalar()) + dataset.write(data) + read = dataset.read() + self.assertEqual(read, data) + def testWriteIntegerArray(self): data = numpy.array([[1, 2, 3, 4], [5, 6, 7, 8]]) @@ -222,3 +233,17 @@ def testWriteVariableLengthStringArray(self): dataset.write(data) read = dataset.read() npt.assert_array_equal(read, data) + + def testWriteVariableLengthStringUTF8Array(self): + + data = numpy.array([u"µm", u"µA"]) + dtype = h5cpp.datatype.String.variable() + dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8 + dataset = Dataset( + self.root, + h5cpp.Path("VariableLengthStringUTF8Array"), + dtype, + Simple((2,))) + dataset.write(data) + read = dataset.read() + npt.assert_array_equal(read, data)