Add support for validating deprecatedFrom. Improve other attribute validation

hed/errors/error_messages.py

@@ -5,8 +5,7 @@
 """
 
 from hed.errors.error_reporter import hed_error, hed_tag_error
-from hed.errors.error_types import ValidationErrors, SchemaErrors, \
-    SidecarErrors, SchemaWarnings, ErrorSeverity, DefinitionErrors, OnsetErrors, ColumnErrors
+from hed.errors.error_types import ValidationErrors, SidecarErrors, ErrorSeverity, DefinitionErrors, OnsetErrors, ColumnErrors
 
 
 @hed_tag_error(ValidationErrors.UNITS_INVALID)
@@ -231,57 +230,6 @@ def val_warning_default_units_used(tag, default_unit):
     return f"No unit specified. Using '{default_unit}' as the default - '{tag}'"
 
 
-@hed_error(SchemaErrors.HED_SCHEMA_DUPLICATE_NODE)
-def schema_error_hed_duplicate_node(tag, duplicate_tag_list, section):
-    tag_join_delimiter = "\n\t"
-    return f"Duplicate term '{str(tag)}' used {len(duplicate_tag_list)} places in '{section}' section schema as:" + \
-           f"{tag_join_delimiter}{tag_join_delimiter.join(duplicate_tag_list)}"
-
-
-@hed_error(SchemaErrors.HED_SCHEMA_DUPLICATE_FROM_LIBRARY)
-def schema_error_hed_duplicate_node(tag, duplicate_tag_list, section):
-    tag_join_delimiter = "\n\t"
-    return f"Duplicate term '{str(tag)}' was found in the library and in the standard schema in '{section}' section schema as:" + \
-           f"{tag_join_delimiter}{tag_join_delimiter.join(duplicate_tag_list)}"
-
-
-@hed_error(SchemaErrors.SCHEMA_ATTRIBUTE_INVALID)
-def schema_error_unknown_attribute(attribute_name, source_tag):
-    return f"Attribute '{attribute_name}' used by '{source_tag}' was not defined in the schema, " \
-           f"or was used outside of it's defined class."
-
-
-@hed_error(SchemaWarnings.INVALID_CHARACTERS_IN_DESC, default_severity=ErrorSeverity.WARNING,
-           actual_code=SchemaWarnings.HED_SCHEMA_CHARACTER_INVALID)
-def schema_warning_invalid_chars_desc(desc_string, tag_name, problem_char, char_index):
-    return f"Invalid character '{problem_char}' in desc for '{tag_name}' at position {char_index}.  '{desc_string}"
-
-
-@hed_error(SchemaWarnings.INVALID_CHARACTERS_IN_TAG, default_severity=ErrorSeverity.WARNING,
-           actual_code=SchemaWarnings.HED_SCHEMA_CHARACTER_INVALID)
-def schema_warning_invalid_chars_tag(tag_name, problem_char, char_index):
-    return f"Invalid character '{problem_char}' in tag '{tag_name}' at position {char_index}."
-
-
-@hed_error(SchemaWarnings.INVALID_CAPITALIZATION, default_severity=ErrorSeverity.WARNING)
-def schema_warning_invalid_capitalization(tag_name, problem_char, char_index):
-    return "First character must be a capital letter or number.  " + \
-           f"Found character '{problem_char}' in tag '{tag_name}' at position {char_index}."
-
-
-@hed_error(SchemaWarnings.NON_PLACEHOLDER_HAS_CLASS, default_severity=ErrorSeverity.WARNING)
-def schema_warning_non_placeholder_class(tag_name, invalid_attribute_name):
-    return "Only placeholder nodes('#') can have a unit or value class." + \
-           f"Found {invalid_attribute_name} on {tag_name}"
-
-
-@hed_error(SchemaWarnings.INVALID_ATTRIBUTE, default_severity=ErrorSeverity.ERROR)
-def schema_error_invalid_attribute(tag_name, invalid_attribute_name):
-    return f"'{invalid_attribute_name}' should not be present in a loaded schema, found on '{tag_name}'." \
-           f"Something went very wrong."
-
-
-
 @hed_error(SidecarErrors.BLANK_HED_STRING)
 def sidecar_error_blank_hed_string():
     return "No HED string found for Value or Category column."

hed/errors/error_reporter.py

@@ -167,8 +167,11 @@ def wrapper(tag, *args, severity=default_severity, **kwargs):
 
 # Import after hed_error decorators are defined.
 from hed.errors import error_messages
+from hed.errors import schema_error_messages
+
 # Intentional to make sure tools don't think the import is unused
 error_messages.mark_as_used = True
+schema_error_messages.mark_as_used = True
 
 
 class ErrorHandler:
@@ -545,6 +548,8 @@ def _add_single_error_to_dict(items, root=None, issue_to_add=None):
 
 def _error_dict_to_string(print_dict, add_link=True, level=0):
     output = ""
+    if print_dict is None:
+        return output
     for context, value in print_dict.items():
         if context == "children":
             for child in value:

hed/errors/error_types.py

@@ -106,19 +106,32 @@ class SidecarErrors:
 
 
 class SchemaErrors:
-    HED_SCHEMA_DUPLICATE_NODE = 'HED_SCHEMA_DUPLICATE_NODE'
+    SCHEMA_DUPLICATE_NODE = 'SCHEMA_DUPLICATE_NODE'
     SCHEMA_ATTRIBUTE_INVALID = 'SCHEMA_ATTRIBUTE_INVALID'
-    HED_SCHEMA_DUPLICATE_FROM_LIBRARY = "SCHEMA_LIBRARY_INVALID"
+    SCHEMA_DUPLICATE_FROM_LIBRARY = "SCHEMA_LIBRARY_INVALID"
 
 
 class SchemaWarnings:
-    INVALID_CHARACTERS_IN_DESC = "INVALID_CHARACTERS_IN_DESC"
-    INVALID_CHARACTERS_IN_TAG = "INVALID_CHARACTERS_IN_TAG"
+    SCHEMA_INVALID_CHARACTERS_IN_DESC = "SCHEMA_INVALID_CHARACTERS_IN_DESC"
+    SCHEMA_INVALID_CHARACTERS_IN_TAG = "SCHEMA_INVALID_CHARACTERS_IN_TAG"
+
     # The actual reported error for the above two
-    HED_SCHEMA_CHARACTER_INVALID = "HED_SCHEMA_CHARACTER_INVALID"
-    INVALID_CAPITALIZATION = 'invalidCaps'
-    NON_PLACEHOLDER_HAS_CLASS = 'NON_PLACEHOLDER_HAS_CLASS'
-    INVALID_ATTRIBUTE = "INVALID_ATTRIBUTE"
+    SCHEMA_CHARACTER_INVALID = "SCHEMA_CHARACTER_INVALID"
+    SCHEMA_INVALID_CAPITALIZATION = 'invalidCaps'
+    SCHEMA_NON_PLACEHOLDER_HAS_CLASS = 'SCHEMA_NON_PLACEHOLDER_HAS_CLASS'
+    SCHEMA_INVALID_ATTRIBUTE = "SCHEMA_INVALID_ATTRIBUTE"
+
+
+class SchemaAttributeErrors:
+    SCHEMA_DEPRECATED_INVALID = "SCHEMA_DEPRECATED_INVALID"
+    SCHEMA_SUGGESTED_TAG_INVALID = "SCHEMA_SUGGESTED_TAG_INVALID"
+    SCHEMA_RELATED_TAG_INVALID = "SCHEMA_RELATED_TAG_INVALID"
+
+    SCHEMA_UNIT_CLASS_INVALID = "SCHEMA_UNIT_CLASS_INVALID"
+    SCHEMA_VALUE_CLASS_INVALID = "SCHEMA_VALUE_CLASS_INVALID"
+
+    SCHEMA_DEFAULT_UNITS_INVALID = "SCHEMA_DEFAULT_UNITS_INVALID"
+    SCHEMA_CHILD_OF_DEPRECATED = "SCHEMA_CHILD_OF_DEPRECATED" # Reported as SCHEMA_DEPRECATED_INVALID
 
 
 class DefinitionErrors:

hed/errors/schema_error_messages.py

@@ -0,0 +1,84 @@
+from hed.errors.error_types import SchemaErrors, SchemaWarnings, ErrorSeverity, SchemaAttributeErrors
+from hed.errors.error_reporter import hed_error
+
+
+@hed_error(SchemaErrors.SCHEMA_DUPLICATE_NODE)
+def schema_error_hed_duplicate_node(tag, duplicate_tag_list, section):
+    tag_join_delimiter = "\n\t"
+    return f"Duplicate term '{str(tag)}' used {len(duplicate_tag_list)} places in '{section}' section schema as:" + \
+           f"{tag_join_delimiter}{tag_join_delimiter.join(duplicate_tag_list)}"
+
+
+@hed_error(SchemaErrors.SCHEMA_DUPLICATE_FROM_LIBRARY)
+def schema_error_hed_duplicate_from_library(tag, duplicate_tag_list, section):
+    tag_join_delimiter = "\n\t"
+    return f"Duplicate term '{str(tag)}' was found in the library and in the standard schema in '{section}' section schema as:" + \
+           f"{tag_join_delimiter}{tag_join_delimiter.join(duplicate_tag_list)}"
+
+
+@hed_error(SchemaErrors.SCHEMA_ATTRIBUTE_INVALID)
+def schema_error_unknown_attribute(attribute_name, source_tag):
+    return f"Attribute '{attribute_name}' used by '{source_tag}' was not defined in the schema, " \
+           f"or was used outside of it's defined class."
+
+
+@hed_error(SchemaWarnings.SCHEMA_INVALID_CHARACTERS_IN_DESC, default_severity=ErrorSeverity.WARNING,
+           actual_code=SchemaWarnings.SCHEMA_CHARACTER_INVALID)
+def schema_warning_invalid_chars_desc(desc_string, tag_name, problem_char, char_index):
+    return f"Invalid character '{problem_char}' in desc for '{tag_name}' at position {char_index}.  '{desc_string}"
+
+
+@hed_error(SchemaWarnings.SCHEMA_INVALID_CHARACTERS_IN_TAG, default_severity=ErrorSeverity.WARNING,
+           actual_code=SchemaWarnings.SCHEMA_CHARACTER_INVALID)
+def schema_warning_invalid_chars_tag(tag_name, problem_char, char_index):
+    return f"Invalid character '{problem_char}' in tag '{tag_name}' at position {char_index}."
+
+
+@hed_error(SchemaWarnings.SCHEMA_INVALID_CAPITALIZATION, default_severity=ErrorSeverity.WARNING)
+def schema_warning_SCHEMA_INVALID_CAPITALIZATION(tag_name, problem_char, char_index):
+    return "First character must be a capital letter or number.  " + \
+           f"Found character '{problem_char}' in tag '{tag_name}' at position {char_index}."
+
+
+@hed_error(SchemaWarnings.SCHEMA_NON_PLACEHOLDER_HAS_CLASS, default_severity=ErrorSeverity.WARNING)
+def schema_warning_non_placeholder_class(tag_name, invalid_attribute_name):
+    return "Only placeholder nodes('#') can have a unit class, value class, or takes value." + \
+           f"Found {invalid_attribute_name} on {tag_name}"
+
+
+@hed_error(SchemaWarnings.SCHEMA_INVALID_ATTRIBUTE, default_severity=ErrorSeverity.ERROR)
+def schema_error_SCHEMA_INVALID_ATTRIBUTE(tag_name, invalid_attribute_name):
+    return f"'{invalid_attribute_name}' should not be present in a loaded schema, found on '{tag_name}'." \
+           f"Something went very wrong."
+
+
+@hed_error(SchemaAttributeErrors.SCHEMA_DEPRECATED_INVALID)
+def schema_error_SCHEMA_DEPRECATED_INVALID(tag_name, invalid_deprecated_version):
+    return f"'{tag_name}' has invalid or unknown value in attribute deprecatedFrom: '{invalid_deprecated_version}'."
+
+
+@hed_error(SchemaAttributeErrors.SCHEMA_CHILD_OF_DEPRECATED,
+           actual_code=SchemaAttributeErrors.SCHEMA_DEPRECATED_INVALID)
+def schema_error_SCHEMA_CHILD_OF_DEPRECATED(deprecated_tag, non_deprecated_child):
+    return f"Deprecated tag '{deprecated_tag}' has a child that is not deprecated: '{non_deprecated_child}'."
+
+
+@hed_error(SchemaAttributeErrors.SCHEMA_SUGGESTED_TAG_INVALID)
+def schema_error_SCHEMA_SUGGESTED_TAG_INVALID(suggestedTag, invalidSuggestedTag, attribute_name):
+    return f"Tag '{suggestedTag}' has an invalid {attribute_name}: '{invalidSuggestedTag}'."
+
+
+@hed_error(SchemaAttributeErrors.SCHEMA_UNIT_CLASS_INVALID)
+def schema_error_SCHEMA_UNIT_CLASS_INVALID(tag, unit_class, attribute_name):
+    return f"Tag '{tag}' has an invalid {attribute_name}: '{unit_class}'."
+
+
+@hed_error(SchemaAttributeErrors.SCHEMA_VALUE_CLASS_INVALID)
+def schema_error_SCHEMA_VALUE_CLASS_INVALID(tag, unit_class, attribute_name):
+    return f"Tag '{tag}' has an invalid {attribute_name}: '{unit_class}'."
+
+
+@hed_error(SchemaAttributeErrors.SCHEMA_DEFAULT_UNITS_INVALID)
+def schema_error_SCHEMA_DEFAULT_UNITS_INVALID(tag, bad_unit, valid_units):
+    valid_units = ",".join(valid_units)
+    return f"Tag '{tag}' has an invalid defaultUnit '{bad_unit}'.  Valid units are: '{valid_units}'."

hed/models/hed_tag.py

@@ -270,7 +270,7 @@ def expanded(self):
 
     @property
     def expandable(self):
-        """Returns if this is expandable
+        """Returns what this expands to
 
            This is primarily used for Def/Def-expand tags at present.
 

hed/schema/hed_cache.py

@@ -60,13 +60,14 @@ def get_cache_directory():
     return HED_CACHE_DIRECTORY
 
 
-def get_hed_versions(local_hed_directory=None, library_name=None, get_libraries=False):
+def get_hed_versions(local_hed_directory=None, library_name=None):
     """ Get the HED versions in the hed directory.
 
     Parameters:
         local_hed_directory (str): Directory to check for versions which defaults to hed_cache.
         library_name (str or None): An optional schema library name.
-        get_libraries (bool): If true, return a dictionary of version numbers, with an entry for each library name.
+                                    None retrieves the standard schema only.
+                                    Pass "all" to retrieve all standard and library schemas as a dict.
 
     Returns:
         list or dict: List of version numbers or dictionary {library_name: [versions]}.
@@ -89,18 +90,16 @@ def get_hed_versions(local_hed_directory=None, library_name=None, get_libraries=
         if expression_match is not None:
             version = expression_match.group(3)
             found_library_name = expression_match.group(2)
-            if not get_libraries and found_library_name != library_name:
+            if library_name != "all" and found_library_name != library_name:
                 continue
             if found_library_name not in all_hed_versions:
                 all_hed_versions[found_library_name] = []
             all_hed_versions[found_library_name].append(version)
     for name, hed_versions in all_hed_versions.items():
         all_hed_versions[name] = _sort_version_list(hed_versions)
-    if get_libraries:
-        return all_hed_versions
     if library_name in all_hed_versions:
         return all_hed_versions[library_name]
-    return []
+    return all_hed_versions
 
 
 def cache_specific_url(hed_xml_url, xml_version=None, library_name=None, cache_folder=None):

hed/schema/hed_schema_constants.py

@@ -41,6 +41,7 @@ class HedKey:
     RelatedTag = "relatedTag"
     SuggestedTag = "suggestedTag"
     Rooted = "rooted"
+    DeprecatedFrom = "deprecatedFrom"
 
     # All known properties
     BoolProperty = 'boolProperty'

hed/schema/schema_attribute_validators.py

@@ -9,9 +9,11 @@
     bool
 """
 
-from hed.errors.error_types import SchemaWarnings, ValidationErrors
+from hed.errors.error_types import SchemaWarnings, ValidationErrors, SchemaAttributeErrors
 from hed.errors.error_reporter import ErrorHandler
 from hed.schema.hed_schema import HedSchema
+from hed.schema.hed_cache import get_hed_versions
+from hed.schema.hed_schema_constants import HedKey
 
 
 def tag_is_placeholder_check(hed_schema, tag_entry, attribute_name):
@@ -28,12 +30,13 @@ def tag_is_placeholder_check(hed_schema, tag_entry, attribute_name):
     """
     issues = []
     if not tag_entry.name.endswith("/#"):
-        issues += ErrorHandler.format_error(SchemaWarnings.NON_PLACEHOLDER_HAS_CLASS, tag_entry.name,
+        issues += ErrorHandler.format_error(SchemaWarnings.SCHEMA_NON_PLACEHOLDER_HAS_CLASS, tag_entry.name,
                                             attribute_name)
 
     return issues
 
 
+# todo: This needs to be refactored, these next several functions are near identical
 def tag_exists_check(hed_schema, tag_entry, attribute_name):
     """ Check if the list of possible tags exists in the schema.
 
@@ -51,14 +54,55 @@ def tag_exists_check(hed_schema, tag_entry, attribute_name):
     split_tags = possible_tags.split(",")
     for org_tag in split_tags:
         if org_tag and org_tag not in hed_schema.tags:
-            issues += ErrorHandler.format_error(ValidationErrors.NO_VALID_TAG_FOUND,
+            issues += ErrorHandler.format_error(SchemaAttributeErrors.SCHEMA_SUGGESTED_TAG_INVALID,
+                                                tag_entry.name,
                                                 org_tag,
-                                                index_in_tag=0,
-                                                index_in_tag_end=len(org_tag))
+                                                attribute_name)
 
     return issues
 
 
+def unit_class_exists(hed_schema, tag_entry, attribute_name):
+    issues = []
+    possible_unit_classes = tag_entry.attributes.get(attribute_name, "")
+    split_tags = possible_unit_classes.split(",")
+    for org_tag in split_tags:
+        if org_tag and org_tag not in hed_schema.unit_classes:
+            issues += ErrorHandler.format_error(SchemaAttributeErrors.SCHEMA_UNIT_CLASS_INVALID,
+                                                tag_entry.name,
+                                                org_tag,
+                                                attribute_name)
+
+    return issues
+
+
+def value_class_exists(hed_schema, tag_entry, attribute_name):
+    issues = []
+    possible_value_classes = tag_entry.attributes.get(attribute_name, "")
+    split_tags = possible_value_classes.split(",")
+    for org_tag in split_tags:
+        if org_tag and org_tag not in hed_schema.value_classes:
+            issues += ErrorHandler.format_error(SchemaAttributeErrors.SCHEMA_VALUE_CLASS_INVALID,
+                                                tag_entry.name,
+                                                org_tag,
+                                                attribute_name)
+
+    return issues
+
+
+def unit_exists(hed_schema, tag_entry, attribute_name):
+    issues = []
+    default_unit = tag_entry.attributes.get(attribute_name, "")
+    if default_unit and default_unit not in tag_entry.derivative_units:
+        issues += ErrorHandler.format_error(SchemaAttributeErrors.SCHEMA_DEFAULT_UNITS_INVALID,
+                                            tag_entry.name,
+                                            default_unit,
+                                            tag_entry.units)
+
+    return issues
+
+
+# This is effectively unused and can never fail - The schema would catch these errors and refuse to load
 def tag_exists_base_schema_check(hed_schema, tag_entry, attribute_name):
     """ Check if the single tag is a partnered schema tag
 
@@ -78,4 +122,32 @@ def tag_exists_base_schema_check(hed_schema, tag_entry, attribute_name):
                                             index_in_tag=0,
                                             index_in_tag_end=len(rooted_tag))
 
+    return issues
+
+
+def tag_is_deprecated_check(hed_schema, tag_entry, attribute_name):
+    """ Check if the tag has a valid deprecatedFrom attribute, and that any children have it
+
+    Parameters:
+        hed_schema (HedSchema): The schema to use for validation
+        tag_entry (HedSchemaEntry): The schema entry for this tag.
+        attribute_name (str): The name of this attribute
+
+    Returns:
+        list: A list of issues. Each issue is a dictionary.
+    """
+    issues = []
+    deprecated_version = tag_entry.attributes.get(attribute_name, "")
+    library_name = tag_entry.has_attribute(HedKey.InLibrary, return_value=True)
+    all_versions = get_hed_versions(library_name=library_name)
+    if deprecated_version and deprecated_version not in all_versions:
+        issues += ErrorHandler.format_error(SchemaAttributeErrors.SCHEMA_DEPRECATED_INVALID,
+                                            tag_entry.name,
+                                            deprecated_version)
+
+    for child in tag_entry.children.values():
+        if not child.has_attribute(attribute_name):
+            issues += ErrorHandler.format_error(SchemaAttributeErrors.SCHEMA_CHILD_OF_DEPRECATED,
+                                                tag_entry.name,
+                                                child.name)
     return issues