Improve def-gatherer. Very much improve spec tests/related validator…

.github/workflows/spec_tests.yaml

@@ -33,8 +33,7 @@ jobs:
 
       - name: Test with unittest
         run: |
-           python -m unittest spec_tests/* > test_results.txt
-        continue-on-error: true
+           python -m unittest spec_tests/test_errors.py > test_results.txt
 
       - name: Upload spec test results
         uses: actions/upload-artifact@v3

hed/errors/error_reporter.py

@@ -30,8 +30,7 @@
 
 # ErrorContext which is expected to be int based.
 int_sort_list = [
-    ErrorContext.ROW,
-    ErrorContext.COLUMN,
+    ErrorContext.ROW
 ]
 
 hed_string_sort_list = [

hed/errors/error_types.py

@@ -27,8 +27,30 @@ class ValidationErrors:
     DEF_EXPAND_INVALID = "DEF_EXPAND_INVALID"
     DEF_INVALID = "DEF_INVALID"
     DEFINITION_INVALID = "DEFINITION_INVALID"
-
-    # NOT OFFICIAL
+    NODE_NAME_EMPTY = 'NODE_NAME_EMPTY'
+    ONSET_OFFSET_ERROR = 'ONSET_OFFSET_ERROR'
+    PARENTHESES_MISMATCH = 'PARENTHESES_MISMATCH'
+    PLACEHOLDER_INVALID = 'PLACEHOLDER_INVALID'
+    REQUIRED_TAG_MISSING = 'REQUIRED_TAG_MISSING'
+    SIDECAR_INVALID = 'SIDECAR_INVALID'
+    SIDECAR_KEY_MISSING = 'SIDECAR_KEY_MISSING'
+    STYLE_WARNING = "STYLE_WARNING"
+    TAG_EMPTY = 'TAG_EMPTY'
+    TAG_EXPRESSION_REPEATED = 'TAG_EXPRESSION_REPEATED'
+    TAG_EXTENDED = 'TAG_EXTENDED'
+    TAG_EXTENSION_INVALID = 'TAG_EXTENSION_INVALID'
+    TAG_GROUP_ERROR = "TAG_GROUP_ERROR"
+    TAG_INVALID = "TAG_INVALID"
+    TAG_NOT_UNIQUE = 'TAG_NOT_UNIQUE'
+    TAG_PREFIX_INVALID = 'TAG_PREFIX_INVALID'
+    TAG_REQUIRES_CHILD = 'TAG_REQUIRES_CHILD'
+    TILDES_UNSUPPORTED = 'TILDES_UNSUPPORTED'
+    UNITS_INVALID = 'UNITS_INVALID'
+    UNITS_MISSING = 'UNITS_MISSING'
+    VERSION_DEPRECATED = 'VERSION_DEPRECATED'
+    VALUE_INVALID = 'VALUE_INVALID'
+
+    # Internal codes
     HED_DEF_UNMATCHED = "HED_DEF_UNMATCHED"
     HED_DEF_VALUE_MISSING = "HED_DEF_VALUE_MISSING"
     HED_DEF_VALUE_EXTRA = "HED_DEF_VALUE_EXTRA"
@@ -37,57 +59,38 @@ class ValidationErrors:
     HED_DEF_EXPAND_UNMATCHED = "HED_DEF_EXPAND_UNMATCHED"
     HED_DEF_EXPAND_VALUE_MISSING = "HED_DEF_EXPAND_VALUE_MISSING"
     HED_DEF_EXPAND_VALUE_EXTRA = "HED_DEF_EXPAND_VALUE_EXTRA"
-    # END NOT OFFICIAL
-
-    HED_NODE_NAME_EMPTY = 'HED_NODE_NAME_EMPTY'
-    HED_ONSET_OFFSET_ERROR = 'HED_ONSET_OFFSET_ERROR'
-    HED_PARENTHESES_MISMATCH = 'HED_PARENTHESES_MISMATCH'
-    HED_PLACEHOLDER_INVALID = 'HED_PLACEHOLDER_INVALID'
-    HED_REQUIRED_TAG_MISSING = 'HED_REQUIRED_TAG_MISSING'
-    HED_SIDECAR_KEY_MISSING = 'HED_SIDECAR_KEY_MISSING'
-    HED_STYLE_WARNING = 'HED_STYLE_WARNING'
-    HED_TAG_EMPTY = 'HED_TAG_EMPTY'
-    HED_TAG_EXTENDED = 'HED_TAG_EXTENDED'
-    HED_TAG_GROUP_ERROR = "HED_TAG_GROUP_ERROR"
-    HED_TAG_INVALID = "HED_TAG_INVALID"
-    HED_TAG_NOT_UNIQUE = 'HED_TAG_NOT_UNIQUE'
+
     HED_TAG_REPEATED = 'HED_TAG_REPEATED'
     HED_TAG_REPEATED_GROUP = 'HED_TAG_REPEATED_GROUP'
-    HED_TAG_REQUIRES_CHILD = 'HED_TAG_REQUIRES_CHILD'
-    HED_TILDES_UNSUPPORTED = 'HED_TILDES_UNSUPPORTED'
-    HED_UNITS_INVALID = 'HED_UNITS_INVALID'
-    HED_UNITS_DEFAULT_USED = 'HED_UNITS_DEFAULT_USED'
-    HED_VALUE_INVALID = 'HED_VALUE_INVALID'
+
+    INVALID_PARENT_NODE = "invalidParent"
+    NO_VALID_TAG_FOUND = "invalidTag"
+
     HED_LIBRARY_UNMATCHED = "HED_LIBRARY_UNMATCHED"
-    TAG_PREFIX_INVALID = "TAG_PREFIX_INVALID"
-    # HED_VERSION_WARNING
+
+    HED_TOP_LEVEL_TAG = "HED_TOP_LEVEL_TAG"
+    HED_MULTIPLE_TOP_TAGS = "HED_MULTIPLE_TOP_TAGS"
+    HED_TAG_GROUP_TAG = "HED_TAG_GROUP_TAG"
+
+    HED_GROUP_EMPTY = 'HED_GROUP_EMPTY'
+    # end internal codes
+
+
+    # Still being worked on below this line
 
     HED_MISSING_REQUIRED_COLUMN = "HED_MISSING_REQUIRED_COLUMN"
     HED_UNKNOWN_COLUMN = "HED_UNKNOWN_COLUMN"
     HED_DUPLICATE_COLUMN = "HED_DUPLICATE_COLUMN"
     HED_BLANK_COLUMN = "HED_BLANK_COLUMN"
 
-
     # Below here shows what the given error maps to
-    HED_GROUP_EMPTY = 'emptyHedGroup'
-    INVALID_TAG_CHARACTER = 'invalidTagCharacter'
-
-    # These are all HED_TAG_INVALID
-    INVALID_EXTENSION = 'invalidExtension'
-    INVALID_PARENT_NODE = "invalidParent"
-    NO_VALID_TAG_FOUND = "invalidTag"
-
-    # These are misc errors that need categorization.
-    HED_TOP_LEVEL_TAG = "HED_TOP_LEVEL_TAG"
-    HED_MULTIPLE_TOP_TAGS = "HED_MULTIPLE_TOP_TAGS"
-    HED_TAG_GROUP_TAG = "HED_TAG_GROUP_TAG"
 
+    INVALID_TAG_CHARACTER = 'invalidTagCharacter'
 
 
 
 class SidecarErrors:
     # These are for json sidecar validation errors(sidecars can also produce most normal validation errors)
-    SIDECAR_INVALID = "SIDECAR_INVALID" # this is the generic error reported for several later ones
     BLANK_HED_STRING = 'blankValueString'
     WRONG_HED_DATA_TYPE = 'wrongHedDataType'
     INVALID_POUND_SIGNS_VALUE = 'invalidNumberPoundSigns'
@@ -97,6 +100,7 @@ class SidecarErrors:
     SIDECAR_NA_USED = 'SIDECAR_NA_USED'
     SIDECAR_HED_USED = 'SIDECAR_HED_USED'
 
+
 class SchemaErrors:
     HED_SCHEMA_DUPLICATE_NODE = 'HED_SCHEMA_DUPLICATE_NODE'
     HED_SCHEMA_ATTRIBUTE_INVALID = 'HED_SCHEMA_ATTRIBUTE_INVALID'
@@ -111,17 +115,24 @@ class SchemaWarnings:
     NON_PLACEHOLDER_HAS_CLASS = 'NON_PLACEHOLDER_HAS_CLASS'
 
 
-# These are all DEFINITION_INVALID errors
 class DefinitionErrors:
-    DEF_TAG_IN_DEFINITION = 'DEF_TAG_IN_DEFINITION'
-    WRONG_NUMBER_GROUP_TAGS = 'wrongNumberGroupTags'
+    # These are all DEFINITION_INVALID errors
     WRONG_NUMBER_PLACEHOLDER_TAGS = 'wrongNumberPlaceholderTags'
     DUPLICATE_DEFINITION = 'duplicateDefinition'
-    TAG_IN_SCHEMA = 'defAlreadyInSchema'
     INVALID_DEFINITION_EXTENSION = 'invalidDefExtension'
+    DEF_TAG_IN_DEFINITION = 'DEF_TAG_IN_DEFINITION'
+    NO_DEFINITION_CONTENTS = "NO_DEFINITION_CONTENTS"
+    PLACEHOLDER_NO_TAKES_VALUE = 'PLACEHOLDER_NO_TAKES_VALUE'
+
+    WRONG_NUMBER_TAGS = 'WRONG_NUMBER_TAGS'
+    WRONG_NUMBER_GROUPS = 'WRONG_NUMBER_GROUPS'
+    BAD_PROP_IN_DEFINITION = 'BAD_PROP_IN_DEFINITION'
+
+    BAD_DEFINITION_LOCATION = 'BAD_DEFINITION_LOCATION'
 
 
 class OnsetErrors:
+    # These are all ONSET_OFFSET_ERROR
     OFFSET_BEFORE_ONSET = "OFFSET_BEFORE_ONSET"
     ONSET_DEF_UNMATCHED = "ONSET_DEF_UNMATCHED"
     ONSET_WRONG_NUMBER_GROUPS = "ONSET_WRONG_NUMBER_GROUPS"

hed/models/base_input.py

@@ -447,4 +447,19 @@ def combine_dataframe(dataframe):
             lambda x: ', '.join(filter(lambda e: bool(e) and e != "n/a", map(str, x))),
             axis=1
         )
-        return dataframe
+        return dataframe
+
+    def get_def_dict(self, hed_schema=None, extra_def_dicts=None):
+        """ Returns the definition dict for this file
+
+        Note: Baseclass implementation returns just extra_def_dicts.
+
+        Parameters:
+            hed_schema(HedSchema): used to identify tags to find definitions(if needed)
+            extra_def_dicts (list, DefinitionDict, or None): Extra dicts to add to the list.
+
+        Returns:
+            DefinitionDict:   A single definition dict representing all the data(and extra def dicts)
+        """
+        from hed.models.definition_dict import DefinitionDict
+        return DefinitionDict(extra_def_dicts, hed_schema)