Schema memory cache/inherited attribute support

hed/__init__.py

@@ -12,7 +12,7 @@
 
 from hed.schema.hed_schema import HedSchema
 from hed.schema.hed_schema_group import HedSchemaGroup
-from hed.schema.hed_schema_io import get_schema, load_schema, load_schema_version
+from hed.schema.hed_schema_io import load_schema, load_schema_version
 
 
 # from hed import errors, models, schema, tools, validator

hed/models/hed_string.py

@@ -332,20 +332,19 @@ def split_hed_string(hed_string):
 
         return result_positions
 
-    def validate(self, hed_schema, allow_placeholders=True, error_handler=None):
+    def validate(self, allow_placeholders=True, error_handler=None):
         """
         Validate the string using the schema
 
         Parameters:
-            hed_schema(HedSchema): The schema to use to validate
             allow_placeholders(bool): allow placeholders in the string
             error_handler(ErrorHandler or None): the error handler to use, creates a default one if none passed
         Returns:
             issues (list of dict): A list of issues for hed string
         """
         from hed.validator import HedValidator
 
-        validator = HedValidator(hed_schema)
+        validator = HedValidator(self._schema)
         return validator.validate(self, allow_placeholders=allow_placeholders, error_handler=error_handler)
 
     def find_top_level_tags(self, anchor_tags, include_groups=2):

hed/models/hed_tag.py

@@ -462,22 +462,6 @@ def has_attribute(self, attribute):
             return self._schema_entry.has_attribute(attribute)
         return False
 
-    def is_extension_allowed_tag(self):
-        """ Check if tag has 'extensionAllowed' attribute.
-
-            Recursively checks parent tag entries for the attribute as well.
-
-        Returns:
-            bool: True if the tag has the 'extensionAllowed' attribute. False, if otherwise.
-
-        """
-        if self.is_takes_value_tag():
-            return False
-
-        if self._schema_entry:
-            return self._schema_entry.any_parent_has_attribute(HedKey.ExtensionAllowed)
-        return False
-
     def get_tag_unit_class_units(self):
         """ Get the unit class units associated with a particular tag.
 
@@ -510,6 +494,8 @@ def get_unit_class_default_unit(self):
     def base_tag_has_attribute(self, tag_attribute):
         """ Check to see if the tag has a specific attribute.
 
+            This is primarily used to check for things like TopLevelTag on Definitions and similar.
+
         Parameters:
             tag_attribute (str): A tag attribute.
 
@@ -522,19 +508,6 @@ def base_tag_has_attribute(self, tag_attribute):
 
         return self._schema_entry.base_tag_has_attribute(tag_attribute)
 
-    def any_parent_has_attribute(self, attribute):
-        """ Check if the tag or any of its parents has the attribute.
-
-        Parameters:
-            attribute (str): The name of the attribute to check for.
-
-        Returns:
-            bool: True if the tag has the given attribute. False, if otherwise.
-
-        """
-        if self._schema_entry:
-            return self._schema_entry.any_parent_has_attribute(attribute=attribute)
-
     @staticmethod
     def _get_schema_namespace(org_tag):
         """ Finds the library namespace for the tag.

hed/schema/__init__.py

@@ -3,7 +3,7 @@
 from .hed_schema_entry import HedSchemaEntry, UnitClassEntry, UnitEntry, HedTagEntry
 from .hed_schema_group import HedSchemaGroup
 from .hed_schema_section import HedSchemaSection
-from .hed_schema_io import load_schema, load_schema_version, from_string, get_hed_xml_version, get_schema
+from .hed_schema_io import load_schema, load_schema_version, from_string, get_hed_xml_version
 from .hed_schema_constants import HedKey, HedSectionKey
 from .hed_cache import cache_xml_versions, get_hed_versions, \
     get_path_from_hed_version, set_cache_directory, get_cache_directory

hed/schema/hed_schema.py

@@ -748,7 +748,7 @@ def _get_attributes_for_section(self, key_class):
     def _add_tag_to_dict(self, long_tag_name, new_entry, key_class):
         # No reason we can't add this here always
         if self.library and not self.merged and self.with_standard:
-            new_entry.set_attribute_value(HedKey.InLibrary, self.library)
+            new_entry._set_attribute_value(HedKey.InLibrary, self.library)
 
         section = self._sections[key_class]
         return section._add_to_dict(long_tag_name, new_entry)

hed/schema/hed_schema_constants.py

@@ -48,6 +48,7 @@ class HedKey:
     UnitModifierProperty = 'unitModifierProperty'
     ValueClassProperty = 'valueClassProperty'
     ElementProperty = 'elementProperty'
+    IsInheritedProperty = 'isInheritedProperty'
 
     SIUnit = 'SIUnit'
     UnitSymbol = 'unitSymbol'

hed/schema/hed_schema_entry.py

@@ -48,63 +48,64 @@ def finalize_entry(self, schema):
             for item in to_remove:
                 self._unknown_attributes.pop(item)
 
-    def set_attribute_value(self, attribute_name, attribute_value):
-        """ Add attribute and set its value.
-
-        Parameters:
-            attribute_name (str): The name of the schema entry attribute.
-            attribute_value (bool or str):  The value of the attribute.
-
-        Notes:
-            - If this an invalid attribute name, it will be also added as an unknown attribute.
-
-        """
-        if not attribute_value:
-            return
-
-        # todo: remove this patch and redo the code
-        # This check doesn't need to be done if the schema is valid.
-        if attribute_name not in self._section.valid_attributes:
-            # print(f"Unknown attribute {attribute_name}")
-            if self._unknown_attributes is None:
-                self._unknown_attributes = {}
-            self._unknown_attributes[attribute_name] = attribute_value
-        self.attributes[attribute_name] = attribute_value
-
-    def has_attribute(self, attribute_name, return_value=False):
-        """ Return True if this entry has the attribute.
+    def has_attribute(self, attribute, return_value=False):
+        """ Checks for the existence of an attribute in this entry.
 
         Parameters:
-            attribute_name (str): The attribute to check for.
-            return_value (bool):  If True return the actual attribute value rather than just indicate presence.
+            attribute (str): The attribute to check for.
+            return_value (bool): If True, returns the actual value of the attribute.
+                                 If False, returns a boolean indicating the presence of the attribute.
 
         Returns:
-            bool or str:  If return_value is false, a boolean is returned rather than the actual value.
+            bool or any: If return_value is False, returns True if the attribute exists and False otherwise.
+            If return_value is True, returns the value of the attribute if it exists, else returns None.
 
         Notes:
-            - A return value of True does not indicate whether or not this attribute is valid
-
+            - The existence of an attribute does not guarantee its validity.
         """
         if return_value:
-            return self.attributes.get(attribute_name, None)
+            return self.attributes.get(attribute, None)
         else:
-            return attribute_name in self.attributes
+            return attribute in self.attributes
 
-    def attribute_has_property(self, attribute_name, property_name):
+    def attribute_has_property(self, attribute, property_name):
         """ Return True if attribute has property.
 
         Parameters:
-            attribute_name (str): Attribute name to check for property_name.
+            attribute (str): Attribute name to check for property_name.
             property_name (str): The property value to return.
 
         Returns:
             bool: Returns True if this entry has the property.
 
         """
-        attr_entry = self._section.valid_attributes.get(attribute_name)
+        attr_entry = self._section.valid_attributes.get(attribute)
         if attr_entry and attr_entry.has_attribute(property_name):
             return True
 
+    def _set_attribute_value(self, attribute, attribute_value):
+        """ Add attribute and set its value.
+
+        Parameters:
+            attribute (str): The name of the schema entry attribute.
+            attribute_value (bool or str):  The value of the attribute.
+
+        Notes:
+            - If this an invalid attribute name, it will be also added as an unknown attribute.
+
+        """
+        if not attribute_value:
+            return
+
+        # todo: remove this patch and redo the code
+        # This check doesn't need to be done if the schema is valid.
+        if attribute not in self._section.valid_attributes:
+            # print(f"Unknown attribute {attribute}")
+            if self._unknown_attributes is None:
+                self._unknown_attributes = {}
+            self._unknown_attributes[attribute] = attribute_value
+        self.attributes[attribute] = attribute_value
+
     @property
     def section_key(self):
         return self._section.section_key
@@ -207,26 +208,72 @@ def __init__(self, *args, **kwargs):
         self.takes_value_child_entry = None  # this is a child takes value tag, if one exists
         self._parent_tag = None
         self.tag_terms = tuple()
+        # During setup, it's better to have attributes shadow inherited before getting its own copy later.
+        self.inherited_attributes = self.attributes
 
-    def any_parent_has_attribute(self, attribute):
-        """ Check if tag (or parents) has the attribute.
+    def has_attribute(self, attribute, return_value=False):
+        """ Returns th existence or value of an attribute in this entry.
+
+            This also checks parent tags for inheritable attributes like ExtensionAllowed.
 
         Parameters:
-            attribute (str): The name of the attribute to check for.
+            attribute (str): The attribute to check for.
+            return_value (bool): If True, returns the actual value of the attribute.
+                                 If False, returns a boolean indicating the presence of the attribute.
 
         Returns:
-            bool: True if the tag has the given attribute. False, if otherwise.
+            bool or any: If return_value is False, returns True if the attribute exists and False otherwise.
+            If return_value is True, returns the value of the attribute if it exists, else returns None.
 
         Notes:
-            - This is mostly used to check extension allowed.  Could be cached.
+            - The existence of an attribute does not guarantee its validity.
+        """
+        val = self.inherited_attributes.get(attribute)
+        if not return_value:
+            val = val is not None
+        return val
+
+    def _check_inherited_attribute(self, attribute, return_value=False, return_union=False):
+        """
+        Checks for the existence of an attribute in this entry and its parents.
+
+        Parameters:
+            attribute (str): The attribute to check for.
+            return_value (bool): If True, returns the actual value of the attribute.
+                                 If False, returns a boolean indicating the presence of the attribute.
+            return_union(bool): If true, return a union of all parent values
+
+        Returns:
+            bool or any: Depending on the flag return_value,
+            returns either the presence of the attribute, or its value.
 
+        Notes:
+            - The existence of an attribute does not guarantee its validity.
+            - For string attributes, the values are joined with a comma as a delimiter from all ancestors.
         """
+        if return_value:
+            attribute_values = []
+
         iter_entry = self
         while iter_entry is not None:
-            if iter_entry.has_attribute(attribute):
-                return True
+            if iter_entry.takes_value_child_entry:
+                break
+            if attribute in iter_entry.attributes:
+                if return_value:
+                    attribute_values.append(iter_entry.attributes[attribute])
+                    if not return_union:
+                        return attribute_values[0]
+                else:
+                    return True
             iter_entry = iter_entry._parent_tag
-        return False
+
+        if return_value:
+            if not attribute_values:
+                return None
+            # Always return the union as a non-union would've been returned above
+            return ",".join(attribute_values)
+        else:
+            return False
 
     def base_tag_has_attribute(self, tag_attribute):
         """ Check if the base tag has a specific attribute.
@@ -260,6 +307,30 @@ def parent_name(self):
         parent_name, _, child_name = self.name.rpartition("/")
         return parent_name
 
+    def _finalize_takes_value_tag(self, schema):
+        if self.name.endswith("/#"):
+            if HedKey.UnitClass in self.attributes:
+                self.unit_classes = {}
+                for unit_class_name in self.attributes[HedKey.UnitClass].split(","):
+                    entry = schema._get_tag_entry(unit_class_name, HedSectionKey.UnitClasses)
+                    if entry:
+                        self.unit_classes[unit_class_name] = entry
+
+            if HedKey.ValueClass in self.attributes:
+                self.value_classes = {}
+                for value_class_name in self.attributes[HedKey.ValueClass].split(","):
+                    entry = schema._get_tag_entry(value_class_name, HedSectionKey.ValueClasses)
+                    if entry:
+                        self.value_classes[value_class_name] = entry
+
+    def _finalize_inherited_attributes(self):
+        # Replace the list with a copy we can modify.
+        self.inherited_attributes = self.attributes.copy()
+        for attribute in self._section.inheritable_attributes:
+            if self._check_inherited_attribute(attribute):
+                treat_as_string = not self.attribute_has_property(attribute, HedKey.BoolProperty)
+                self.inherited_attributes[attribute] = self._check_inherited_attribute(attribute, True, treat_as_string)
+
     def finalize_entry(self, schema):
         """ Called once after schema loading to set state.
 
@@ -276,17 +347,5 @@ def finalize_entry(self, schema):
         self.takes_value_child_entry = schema._get_tag_entry(self.name + "/#")
         self.tag_terms = tuple(self.long_tag_name.lower().split("/"))
 
-        if self.name.endswith("/#"):
-            if HedKey.UnitClass in self.attributes:
-                self.unit_classes = {}
-                for unit_class_name in self.attributes[HedKey.UnitClass].split(","):
-                    entry = schema._get_tag_entry(unit_class_name, HedSectionKey.UnitClasses)
-                    if entry:
-                        self.unit_classes[unit_class_name] = entry
-
-            if HedKey.ValueClass in self.attributes:
-                self.value_classes = {}
-                for value_class_name in self.attributes[HedKey.ValueClass].split(","):
-                    entry = schema._get_tag_entry(value_class_name, HedSectionKey.ValueClasses)
-                    if entry:
-                        self.value_classes[value_class_name] = entry
+        self._finalize_inherited_attributes()
+        self._finalize_takes_value_tag(schema)

hed/schema/hed_schema_io.py

@@ -1,17 +1,20 @@
 """ Utilities for loading and outputting HED schema. """
 import os
 import json
+import functools
 from hed.schema.schema_io.xml2schema import HedSchemaXMLParser
 from hed.schema.schema_io.wiki2schema import HedSchemaWikiParser
 from hed.schema import hed_schema_constants, hed_cache
 
 from hed.errors.exceptions import HedFileError, HedExceptions
-from hed.schema.hed_schema import HedSchema
 from hed.schema.schema_io import schema_util
 from hed.schema.hed_schema_group import HedSchemaGroup
 from hed.schema.schema_validation_util import validate_version_string
 
 
+MAX_MEMORY_CACHE = 20
+
+
 def from_string(schema_string, file_type=".xml", schema_namespace=None):
     """ Create a schema from the given string.
 
@@ -48,17 +51,6 @@ def from_string(schema_string, file_type=".xml", schema_namespace=None):
     return hed_schema
 
 
-def get_schema(hed_versions):
-    if not hed_versions:
-        return None
-    elif isinstance(hed_versions, str) or isinstance(hed_versions, list):
-        return load_schema_version(hed_versions)
-    elif isinstance(hed_versions, HedSchema) or isinstance(hed_versions, HedSchemaGroup):
-        return hed_versions
-    else:
-        raise ValueError("InvalidHedSchemaOrSchemaVersion", "Expected schema or schema version")
-
-
 def load_schema(hed_path=None, schema_namespace=None):
     """ Load a schema from the given file or URL path.
 
@@ -114,12 +106,13 @@ def get_hed_xml_version(xml_file_path):
     return root_node.attrib[hed_schema_constants.VERSION_ATTRIBUTE]
 
 
+@functools.lru_cache(maxsize=MAX_MEMORY_CACHE)
 def _load_schema_version(xml_version=None, xml_folder=None):
     """ Return specified version or latest if not specified.
 
     Parameters:
         xml_folder (str): Path to a folder containing schema.
-        xml_version (str or list): HED version format string. Expected format: '[schema_namespace:][library_name_]X.Y.Z'.
+        xml_version (str): HED version format string. Expected format: '[schema_namespace:][library_name_]X.Y.Z'.
 
     Returns:
         HedSchema or HedSchemaGroup: The requested HedSchema object.