Skip to content

Updates to schema loading and comparison #727

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
VisLab merged 2 commits into from
Aug 2, 2023
Merged

Updates to schema loading and comparison #727

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
56b45d8
Updates to schema loading and comparison
IanCa Aug 1, 2023
ee6f4b2
Add score 1.1.0 to hardcoded schemas
IanCa Aug 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 13 additions & 30 deletions hed/schema/hed_schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
import os
import shutil
import json

from hed.schema.hed_schema_constants import HedKey, HedSectionKey
from hed.schema import hed_schema_constants as constants
from hed.schema.schema_io import schema_util
from hed.schema.schema_io.schema2xml import HedSchema2XML
from hed.schema.schema_io.schema2wiki import HedSchema2Wiki
from hed.schema.schema_io.schema2xml import Schema2XML
from hed.schema.schema_io.schema2wiki import Schema2Wiki
from hed.schema.hed_schema_section import HedSchemaSection, HedSchemaTagSection, HedSchemaUnitClassSection
from hed.errors import ErrorHandler
from hed.errors.error_types import ValidationErrors
Expand Down Expand Up @@ -212,8 +210,7 @@ def get_as_mediawiki_string(self, save_merged=False):
str: The schema as a string in mediawiki format.

"""
schema2wiki = HedSchema2Wiki()
output_strings = schema2wiki.process_schema(self, save_merged)
output_strings = Schema2Wiki.process_schema(self, save_merged)
return '\n'.join(output_strings)

def get_as_xml_string(self, save_merged=True):
Expand All @@ -227,12 +224,11 @@ def get_as_xml_string(self, save_merged=True):
str: Return the schema as an XML string.

"""
schema2xml = HedSchema2XML()
xml_tree = schema2xml.process_schema(self, save_merged)
xml_tree = Schema2XML.process_schema(self, save_merged)
return schema_util._xml_element_2_str(xml_tree)

def save_as_mediawiki(self, filename=None, save_merged=False):
""" Save as mediawiki to a temporary file.
""" Save as mediawiki to a file.

filename: str
If present, move the resulting file to this location.
Expand All @@ -243,19 +239,12 @@ def save_as_mediawiki(self, filename=None, save_merged=False):
Returns:
str: The newly created schema filename.
"""
schema2wiki = HedSchema2Wiki()
output_strings = schema2wiki.process_schema(self, save_merged)
output_strings = Schema2Wiki.process_schema(self, save_merged)
local_wiki_file = schema_util.write_strings_to_file(output_strings, ".mediawiki")
if filename:
directory = os.path.dirname(filename)
if directory and not os.path.exists(directory):
os.makedirs(directory)
shutil.move(local_wiki_file, filename)
return filename
return local_wiki_file
return schema_util.move_file(local_wiki_file, filename)

def save_as_xml(self, filename=None, save_merged=True):
""" Save as XML to a temporary file.
""" Save as XML to a file.

filename: str
If present, move the resulting file to this location.
Expand All @@ -266,16 +255,9 @@ def save_as_xml(self, filename=None, save_merged=True):
Returns:
str: The name of the newly created schema file.
"""
schema2xml = HedSchema2XML()
xml_tree = schema2xml.process_schema(self, save_merged)
xml_tree = Schema2XML.process_schema(self, save_merged)
local_xml_file = schema_util.write_xml_tree_2_xml_file(xml_tree, ".xml")
if filename:
directory = os.path.dirname(filename)
if directory and not os.path.exists(directory):
os.makedirs(directory)
shutil.move(local_xml_file, filename)
return filename
return local_xml_file
return schema_util.move_file(local_xml_file, filename)

def set_schema_prefix(self, schema_namespace):
""" Set library namespace associated for this schema.
Expand Down Expand Up @@ -746,8 +728,9 @@ def _get_attributes_for_section(self, key_class):
# Semi private function used to create a schema in memory(usually from a source file)
# ===============================================
def _add_tag_to_dict(self, long_tag_name, new_entry, key_class):
# No reason we can't add this here always
if self.library and not self.merged and self.with_standard:
# Add the InLibrary attribute to any library schemas as they are loaded
# These are later removed when they are saved out, if saving unmerged
if self.library and (not self.with_standard or (not self.merged and self.with_standard)):
new_entry._set_attribute_value(HedKey.InLibrary, self.library)

section = self._sections[key_class]
Expand Down
15 changes: 14 additions & 1 deletion hed/schema/hed_schema_constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from enum import Enum


class HedSectionKey(Enum):
""" Kegs designating specific sections in a HedSchema object.
"""
Expand Down Expand Up @@ -69,4 +70,16 @@ class HedKey:
VERSION_ATTRIBUTE = 'version'
LIBRARY_ATTRIBUTE = 'library'
WITH_STANDARD_ATTRIBUTE = "withStandard"
UNMERGED_ATTRIBUTE = "unmerged"
UNMERGED_ATTRIBUTE = "unmerged"
NS_ATTRIB = "xmlns:xsi"
NO_LOC_ATTRIB = "xsi:noNamespaceSchemaLocation"

# A list of all attributes that can appear in the header line
valid_header_attributes = {
VERSION_ATTRIBUTE,
LIBRARY_ATTRIBUTE,
WITH_STANDARD_ATTRIBUTE,
NS_ATTRIB,
NO_LOC_ATTRIB,
UNMERGED_ATTRIBUTE
}
24 changes: 13 additions & 11 deletions hed/schema/hed_schema_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
import os
import json
import functools
from hed.schema.schema_io.xml2schema import HedSchemaXMLParser
from hed.schema.schema_io.wiki2schema import HedSchemaWikiParser
from hed.schema import hed_schema_constants, hed_cache
from hed.schema.schema_io.xml2schema import SchemaLoaderXML
from hed.schema.schema_io.wiki2schema import SchemaLoaderWiki
from hed.schema import hed_cache

from hed.errors.exceptions import HedFileError, HedExceptions
from hed.schema.schema_io import schema_util
Expand Down Expand Up @@ -39,9 +39,9 @@ def from_string(schema_string, file_type=".xml", schema_namespace=None):
filename=schema_string)

if file_type.endswith(".xml"):
hed_schema = HedSchemaXMLParser.load_xml(schema_as_string=schema_string)
hed_schema = SchemaLoaderXML.load(schema_as_string=schema_string)
elif file_type.endswith(".mediawiki"):
hed_schema = HedSchemaWikiParser.load_wiki(schema_as_string=schema_string)
hed_schema = SchemaLoaderWiki.load(schema_as_string=schema_string)
else:
raise HedFileError(HedExceptions.INVALID_EXTENSION, "Unknown schema extension", filename=file_type)

Expand Down Expand Up @@ -77,9 +77,9 @@ def load_schema(hed_path=None, schema_namespace=None):
file_as_string = schema_util.url_to_string(hed_path)
hed_schema = from_string(file_as_string, file_type=os.path.splitext(hed_path.lower())[1])
elif hed_path.lower().endswith(".xml"):
hed_schema = HedSchemaXMLParser.load_xml(hed_path)
hed_schema = SchemaLoaderXML.load(hed_path)
elif hed_path.lower().endswith(".mediawiki"):
hed_schema = HedSchemaWikiParser.load_wiki(hed_path)
hed_schema = SchemaLoaderWiki.load(hed_path)
else:
raise HedFileError(HedExceptions.INVALID_EXTENSION, "Unknown schema extension", filename=hed_path)

Expand All @@ -89,7 +89,7 @@ def load_schema(hed_path=None, schema_namespace=None):
return hed_schema


# todo: this could be updated to also support .mediawiki format.
# If this is actually used, we could easily add other versions/update this one
def get_hed_xml_version(xml_file_path):
""" Get the version number from a HED XML file.

Expand All @@ -102,8 +102,8 @@ def get_hed_xml_version(xml_file_path):
:raises HedFileError:
- There is an issue loading the schema
"""
root_node = HedSchemaXMLParser._parse_hed_xml(xml_file_path)
return root_node.attrib[hed_schema_constants.VERSION_ATTRIBUTE]
parser = SchemaLoaderXML(xml_file_path)
return parser.schema.version


@functools.lru_cache(maxsize=MAX_MEMORY_CACHE)
Expand Down Expand Up @@ -143,7 +143,9 @@ def _load_schema_version(xml_version=None, xml_folder=None):
hed_cache.cache_xml_versions(cache_folder=xml_folder)
final_hed_xml_file = hed_cache.get_hed_version_path(xml_version, library_name, xml_folder)
if not final_hed_xml_file:
raise HedFileError(HedExceptions.FILE_NOT_FOUND, f"HED version '{xml_version}' not found in cache: {hed_cache.get_cache_directory()}", filename=xml_folder)
raise HedFileError(HedExceptions.FILE_NOT_FOUND,
f"HED version '{xml_version}' not found in cache: {hed_cache.get_cache_directory()}",
filename=xml_folder)
hed_schema = load_schema(final_hed_xml_file)
else:
raise e
Expand Down
Loading