From 5531241326e06f0f71e50c71b9c055f8c3409f40 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 4 Oct 2018 17:00:28 +0200 Subject: [PATCH 1/7] [scripts] Add plain version conv script --- odml/scripts/odml_conversion.py | 123 ++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 odml/scripts/odml_conversion.py diff --git a/odml/scripts/odml_conversion.py b/odml/scripts/odml_conversion.py new file mode 100644 index 00000000..e516b9c8 --- /dev/null +++ b/odml/scripts/odml_conversion.py @@ -0,0 +1,123 @@ +"""odmlConversion + +odmlConversion searches for odML files within a provided SEARCHDIR +and converts them to the newest odML format version. +Original files will never be overwritten. + +Usage: odmlconversion [-r] [-o OUT] SEARCHDIR + +Arguments: + SEARCHDIR Directory to search for odML files. + +Options: + -o OUT Output directory. Must exist if specified. + If not specified, output files will be + written to the current directory. + -r Search recursively. + -h --help Show this screen. + --version Show version. +""" + +import os +import pathlib +import sys +import tempfile + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +import odml + +from docopt import docopt +from odml.tools.version_converter import VersionConverter as VerConf + +try: + unicode = unicode +except NameError: + unicode = str + + +def run_conversion(file_list, output_dir, report, source_format="XML"): + """ + Convert a list of odML files to the latest odML version. + :param file_list: list of files to be converted. + :param output_dir: Directory where odML files converted to + the latest odML version will be saved. + :param report: Reporting StringIO. + :param source_format: Original file format of the odML source files. + XML, JSON and YAML are supported, default is XML. + """ + # Exceptions are kept as broad as possible to ignore any non-odML or + # invalid odML files and ensuring everything that can be will be converted. + for curr_file in file_list: + file_path = unicode(curr_file.absolute()) + report.write("[Info] Handling file '%s'\n" % file_path) + # When loading the current file succeeds, it is + # a recent odML format file and can be ignored. + try: + odml.load(file_path, source_format) + report.write("[Info] Skip recent version file '%s'" % file_path) + except Exception as exc: + out_name = os.path.splitext(os.path.basename(file_path))[0] + outfile = os.path.join(output_dir, "%s_conv.xml" % out_name) + try: + VerConf(file_path).write_to_file(outfile, source_format) + except Exception as exc: + # Ignore files we cannot parse or convert + report.write("[Error] version converting file '%s': '%s'\n" % + (file_path, exc)) + + +def main(args=None): + """ + Convenience script to automatically convert odML files + within a directory (tree) to the newest file version. + Check the cli help for details. + :param args: Command line arguments + """ + parser = docopt(__doc__, argv=args, version="0.1.0") + + root = parser['SEARCHDIR'] + if not os.path.isdir(root): + print(docopt(__doc__, "-h")) + exit(1) + + # Handle all supported odML file formats. + if parser['-r']: + xfiles = list(pathlib.Path(root).rglob('*.odml')) + xfiles.extend(list(pathlib.Path(root).rglob('*.xml'))) + jfiles = list(pathlib.Path(root).rglob('*.json')) + yfiles = list(pathlib.Path(root).rglob('*.yaml')) + else: + xfiles = list(pathlib.Path(root).glob('*.odml')) + xfiles.extend(list(pathlib.Path(root).glob('*.xml'))) + jfiles = list(pathlib.Path(root).glob('*.json')) + yfiles = list(pathlib.Path(root).glob('*.yaml')) + + out_root = os.getcwd() + if parser["-o"]: + if not os.path.isdir(parser["-o"]): + print("[Error] Could not find output directory '%s'" % parser["-o"]) + exit(1) + + out_root = parser["-o"] + + out_dir = tempfile.mkdtemp(prefix="odmlconv_", dir=out_root) + + # Use this monkeypatch reporter until there is a way + # to run the converters silently. + report = StringIO() + report.write("[Info] Files will be saved to '%s'\n" % out_dir) + + run_conversion(xfiles, out_dir, report) + run_conversion(jfiles, out_dir, report, "JSON") + run_conversion(yfiles, out_dir, report, "YAML") + + print(report.getvalue()) + report.close() + + +if __name__ == "__main__": + main(sys.argv[1:]) From c030a62e20783a4e9bbe992ab5377a7272e379a7 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Thu, 4 Oct 2018 17:02:01 +0200 Subject: [PATCH 2/7] [setup] Add version conversion cli script --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bade83c1..467fcb12 100644 --- a/setup.py +++ b/setup.py @@ -47,5 +47,6 @@ long_description=description_text, classifiers=CLASSIFIERS, license="BSD", - entry_points={'console_scripts': ['odmltordf=odml.scripts.odml_to_rdf:main']} + entry_points={'console_scripts': ['odmltordf=odml.scripts.odml_to_rdf:main', + 'odmlconversion=odml.scripts.odml_conversion:main']} ) From 47675b4b73bad3cbd3ae4d8609fd724e9df45e60 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Mon, 8 Oct 2018 15:43:57 +0200 Subject: [PATCH 3/7] [CHANGELOG] Update latest changes --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c49d8268..9e7f4ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ until the next release. # Latest changes in master +## Uncertainty changes + +Uncertainty is now limited to float only. See #294. + +## Version converter changes + +The VersionConverter dealt with an edge case of XML test files with opening tags +that were missing their closing tag rendering them broken. Catching this one edge case +circumvented opening XML files via lxml, leaving the resulting document open to various +encoding problems. + +Support to resolve the specific tag edge cases is dropped in favour of properly opening +XML files via lxml. See #301. + +## Additional console script + +The `odmlconversion` convenience console script has been added to convert multiple +previous odML version files to the latest odML version. + # Version 1.4.1 From 923cea8b8820f8c55b138578bbc020991a802861 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 9 Oct 2018 13:41:44 +0200 Subject: [PATCH 4/7] [info] Update version number --- odml/info.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odml/info.json b/odml/info.json index 4ef36c13..b548a032 100644 --- a/odml/info.json +++ b/odml/info.json @@ -1,5 +1,5 @@ { - "VERSION": "1.4.1", + "VERSION": "1.4.2", "FORMAT_VERSION": "1.1", "AUTHOR": "Hagen Fritsch, Jan Grewe, Christian Kellner, Achilleas Koutsou, Michael Sonntag, Lyuba Zehl", "COPYRIGHT": "(c) 2011-2018, German Neuroinformatics Node", From 284b28b1d4dff44f25c7b241c2d8159eeded64d1 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 9 Oct 2018 13:42:13 +0200 Subject: [PATCH 5/7] [doc/conf] Import proper pkg version number --- doc/conf.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 4f86691c..2f2a4512 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,9 +11,22 @@ # All configuration values have a default; values that are commented out # serve to show the default. +import json import os +import pathlib import sys +from distutils.version import LooseVersion as CheckVer + +currdir = os.path.dirname(os.path.abspath(__file__)) +parent = pathlib.Path(currdir).parent +path = os.path.join(parent, "odml", "info.json") + +with open(path) as infofile: + infodict = json.load(infofile) + +version_str = infodict["VERSION"] + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -69,9 +82,9 @@ def __new__(meta, name, bases, clsdict): # built documents. # # The short X.Y version. -version = '1.4' +version = "%s.%s" % (CheckVer(version_str).version[0], CheckVer(version_str).version[1]) # The full version, including alpha/beta/rc tags. -release = '1.4.1' +release = version_str # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 786bcdd7f4ec754c19e03360dcda7f7723654bfd Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 9 Oct 2018 13:42:39 +0200 Subject: [PATCH 6/7] [setup.py] Add pathlib to pkg dep list --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 467fcb12..d424aa16 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ with open('README.rst') as f: description_text = f.read() -install_req = ["lxml", "pyyaml==3.13", "rdflib", "docopt"] +install_req = ["lxml", "pyyaml==3.13", "rdflib", "docopt", "pathlib"] if sys.version_info < (3, 4): install_req += ["enum34"] From 2ecfaf2edf06f19b4144ed3be0f2bff3954a37db Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 9 Oct 2018 13:57:16 +0200 Subject: [PATCH 7/7] [scripts] Update cli description --- odml/scripts/odml_conversion.py | 10 +++++++--- odml/scripts/odml_to_rdf.py | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/odml/scripts/odml_conversion.py b/odml/scripts/odml_conversion.py index e516b9c8..1222091c 100644 --- a/odml/scripts/odml_conversion.py +++ b/odml/scripts/odml_conversion.py @@ -2,7 +2,9 @@ odmlConversion searches for odML files within a provided SEARCHDIR and converts them to the newest odML format version. -Original files will never be overwritten. +Original files will never be overwritten. New files will be +written either to a new directory at the current or a specified +location. Usage: odmlconversion [-r] [-o OUT] SEARCHDIR @@ -13,7 +15,8 @@ -o OUT Output directory. Must exist if specified. If not specified, output files will be written to the current directory. - -r Search recursively. + -r Search recursively. Directory structures + will not be retained. -h --help Show this screen. --version Show version. """ @@ -23,6 +26,8 @@ import sys import tempfile +from docopt import docopt + try: from StringIO import StringIO except ImportError: @@ -30,7 +35,6 @@ import odml -from docopt import docopt from odml.tools.version_converter import VersionConverter as VerConf try: diff --git a/odml/scripts/odml_to_rdf.py b/odml/scripts/odml_to_rdf.py index 61bc53b6..4eae0bea 100644 --- a/odml/scripts/odml_to_rdf.py +++ b/odml/scripts/odml_to_rdf.py @@ -3,7 +3,9 @@ odmlToRDF searches for odML files within a provided SEARCHDIR and converts them to the newest odML format version and exports all found and resulting odML files to XML formatted RDF. -Original files will never be overwritten. +Original files will never be overwritten. New files will be +written either to a new directory at the current or a specified +location. Usage: odmltordf [-r] [-o OUT] SEARCHDIR @@ -14,7 +16,8 @@ -o OUT Output directory. Must exist if specified. If not specified, output files will be written to the current directory. - -r Search recursively. + -r Search recursively. Directory structures + will not be retained. -h --help Show this screen. --version Show version. """ @@ -24,6 +27,8 @@ import sys import tempfile +from docopt import docopt + try: from StringIO import StringIO except ImportError: @@ -31,7 +36,6 @@ import odml -from docopt import docopt from odml.tools.odmlparser import ODMLReader, ODMLWriter from odml.tools.version_converter import VersionConverter as VerConf