From 3a21f0f6277f1e28b35260beb426b977ac19f757 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 24 Jul 2018 16:03:35 +0200 Subject: [PATCH 1/7] [tools/versionConv] Fix encoding issue Closes #288 --- odml/tools/version_converter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/odml/tools/version_converter.py b/odml/tools/version_converter.py index 47539cb9..f6b33049 100644 --- a/odml/tools/version_converter.py +++ b/odml/tools/version_converter.py @@ -52,6 +52,9 @@ def _parse_xml(self): if elem in doc: doc = doc.replace(elem, val) + # Make sure encoding is present for the xml parser + doc = doc.encode('utf-8') + # Make pretty print available by resetting format parser = ET.XMLParser(remove_blank_text=True) tree = ET.ElementTree(ET.fromstring(doc, parser)) From eed09dfedb23fdca5849c10f4adfaef524f6dc97 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 25 Jul 2018 14:46:28 +0200 Subject: [PATCH 2/7] [tools/xmlparser] Fix from_csv single str val read Closes #295 When reading a single string value from csv which is containing commata, make sure, that it still remains a single string value and is not split up into a multi value at the comma positions. --- odml/tools/xmlparser.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index 1a4f2404..ba323348 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -40,8 +40,14 @@ def to_csv(val): def from_csv(value_string): if not value_string: return [] - if value_string[0] == "[": + if value_string[0] == "[" and value_string[-1] == "]": value_string = value_string[1:-1] + else: + # This is a single string entry, any comma contained + # is part of the value and must not be used to + # split up the string. + return [value_string] + if not value_string: return [] stream = StringIO(value_string) From 9fa62d82b6c0aa4adb7c8e76176e9eade246b4bd Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 25 Jul 2018 14:49:02 +0200 Subject: [PATCH 3/7] [tools/xmlparser] Strip ws and quotes on to_csv When writing multiple values to csv make sure that whitespaces and carriage return linefeeds as well as quotation marks that have been added by the csv writer are removed before returning the csv string. --- odml/tools/xmlparser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/odml/tools/xmlparser.py b/odml/tools/xmlparser.py index ba323348..4a04e239 100644 --- a/odml/tools/xmlparser.py +++ b/odml/tools/xmlparser.py @@ -27,11 +27,15 @@ def to_csv(val): - unicode_values = list(map(unicode, val)) + # Make sure all individual values do not contain + # leading or trailing whitespaces. + unicode_values = list(map(unicode.strip, map(unicode, val))) stream = StringIO() writer = csv.writer(stream, dialect="excel") writer.writerow(unicode_values) - csv_string = stream.getvalue().strip() + # Strip any csv.writer added carriage return line feeds + # and double quotes before saving. + csv_string = stream.getvalue().strip().strip('"') if len(unicode_values) > 1: csv_string = "[" + csv_string + "]" return csv_string From 4e134aa066c11a037c79bcf68d9a1a1877a584cc Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 25 Jul 2018 14:51:24 +0200 Subject: [PATCH 4/7] [tools/versionConv] Save multivals w/o whitespace --- odml/tools/version_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odml/tools/version_converter.py b/odml/tools/version_converter.py index f6b33049..86e40115 100644 --- a/odml/tools/version_converter.py +++ b/odml/tools/version_converter.py @@ -303,7 +303,7 @@ def _handle_properties(self, root): if value.text: if main_val.text: - main_val.text += ", " + value.text.strip() + main_val.text += "," + value.text.strip() multiple_values = True else: main_val.text = value.text.strip() From 40bc1562e33aa700c6402cd225112ceecfb9eec7 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 25 Jul 2018 14:51:55 +0200 Subject: [PATCH 5/7] [test/versionConv] Fix changed multival save --- test/test_version_converter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_version_converter.py b/test/test_version_converter.py index 48b248e1..efa427d2 100644 --- a/test/test_version_converter.py +++ b/test/test_version_converter.py @@ -130,7 +130,7 @@ def test_convert_odml_file(self): self.assertEqual(val_elems[0].find("unit"), None) self.assertEqual(val_elems[0].find("type"), None) self.assertEqual(val_elems[0].find("uncertainty"), None) - self.assertEqual(val_elems[0].text, "[0, 45]") + self.assertEqual(val_elems[0].text, "[0,45]") self.assertEqual(prop.find("unit").text, "deg") self.assertEqual(len(prop.findall("unit")), 1) self.assertEqual(prop.find("type").text, "int") @@ -500,7 +500,7 @@ def test_convert_odml_file_value(self): # Test multiple value export prop = sec.findall("property")[1] self.assertEqual(len(prop), 2) - self.assertEqual(prop.find("value").text, "[1, 2, 3]") + self.assertEqual(prop.find("value").text, "[1,2,3]") # Test empty value export prop = sec.findall("property")[2] @@ -521,7 +521,7 @@ def test_convert_odml_file_value(self): # Test valid multiple Value tag export prop = sec.findall("property")[4] self.assertEqual(len(prop), 7) - self.assertEqual(prop.find("value").text, "[0.1, 0.2, 3]") + self.assertEqual(prop.find("value").text, "[0.1,0.2,3]") self.assertEqual(prop.find("type").text, "float") self.assertEqual(prop.find("uncertainty").text, "0.05") self.assertEqual(prop.find("unit").text, "mV") From 35161bc7164243919127e1eb82613b6a501f23b0 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 25 Jul 2018 14:52:29 +0200 Subject: [PATCH 6/7] [travis] Unsupport py3.5 on macOS --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6c8e3474..9b80df52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,6 @@ matrix: - os: linux python: "3.6" - - os: osx - language: generic - env: - - OSXENV=3.5.0 - os: osx language: generic env: From 46a4fbb5c5ca06e58d8999c644da2852c6923ebb Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 25 Jul 2018 15:03:49 +0200 Subject: [PATCH 7/7] [test/versionConv] Add strng value tests --- test/test_version_converter.py | 62 +++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/test/test_version_converter.py b/test/test_version_converter.py index efa427d2..29c28ab3 100644 --- a/test/test_version_converter.py +++ b/test/test_version_converter.py @@ -481,6 +481,37 @@ def test_convert_odml_file_value(self): + + Single, string, value, with, many, commata.string + testSingleString + + + + Astring + Bstring + Cstring + testStringList + + + + Single string value with wrapping whitespace string + testStringWhiteSpace + + + + Multiple Strings string + with wrapping string + Whitespace string + testStringListWhiteSpace + + + + 1 int + 2 int + 3 int + testIntListWhiteSpace + + """ @@ -490,7 +521,7 @@ def test_convert_odml_file_value(self): conv_doc = vc._convert(vc._parse_xml()) root = conv_doc.getroot() sec = root.find("section") - self.assertEqual(len(sec), 9) + self.assertEqual(len(sec), 14) # Test single value export prop = sec.findall("property")[0] @@ -541,6 +572,35 @@ def test_convert_odml_file_value(self): self.assertEqual(prop.find("name").text, "Unsupported binary value dtype replace") self.assertEqual(prop.find("type").text, "text") + # Test single string value with commata + prop = sec.findall("property")[8] + self.assertEqual(prop.find("name").text, "testSingleString") + self.assertEqual(prop.find("value").text, + "Single, string, value, with, many, commata.") + + # Test string list import + prop = sec.findall("property")[9] + self.assertEqual(prop.find("name").text, "testStringList") + self.assertEqual(prop.find("value").text, "[A,B,C]") + + # Test single string values wrapping whitespace removal + prop = sec.findall("property")[10] + self.assertEqual(prop.find("name").text, "testStringWhiteSpace") + self.assertEqual(prop.find("value").text, + "Single string value with wrapping whitespace") + + # Test multiple string values with wrapping whitespace removal + prop = sec.findall("property")[11] + self.assertEqual(prop.find("name").text, "testStringListWhiteSpace") + self.assertEqual(prop.find("value").text, + "[Multiple Strings,with wrapping,Whitespace]") + + # Test multiple int values with wrapping whitespaces + prop = sec.findall("property")[12] + self.assertEqual(prop.find("name").text, "testIntListWhiteSpace") + self.assertEqual(prop.find("type").text, "int") + self.assertEqual(prop.find("value").text, "[1,2,3]") + def test_parse_dict_document(self): # Test appending tags; not appending empty sections doc_dict = {'Document': {'author': 'HPL', 'sections': []}}