Update dependabot to also update submodules

.github/dependabot.yml

@@ -2,9 +2,18 @@ version: 2
 updates:
   - package-ecosystem: "github-actions"
     directory: "/"
+    target-branch: "develop"
     schedule:
       interval: "weekly"
   - package-ecosystem: "pip"
     directory: "/"
+    target-branch: "develop"
     schedule:
       interval: "weekly"
+
+  - package-ecosystem: gitsubmodule
+    schedule:
+        interval: "daily"
+    target-branch: "develop"
+    directory: /
+    automerge: true

hed/errors/__init__.py

@@ -1,4 +1,4 @@
-from .error_reporter import ErrorHandler, get_exception_issue_string, get_printable_issue_string, sort_issues
+from .error_reporter import ErrorHandler, get_printable_issue_string, sort_issues
 from .error_types import DefinitionErrors, OnsetErrors, SchemaErrors, SchemaWarnings,  SidecarErrors, ValidationErrors
 from .error_types import ErrorContext, ErrorSeverity
 from .exceptions import HedExceptions, HedFileError

hed/errors/error_messages.py

@@ -291,36 +291,36 @@ def sidecar_na_used(column_name):
     return f"Invalid category key 'n/a' found in column {column_name}."
 
 
-@hed_tag_error(DefinitionErrors.DEF_TAG_IN_DEFINITION, actual_code=ValidationErrors.HED_DEFINITION_INVALID)
+@hed_tag_error(DefinitionErrors.DEF_TAG_IN_DEFINITION, actual_code=ValidationErrors.DEFINITION_INVALID)
 def def_error_def_tag_in_definition(tag, def_name):
     return f"Invalid tag {tag} found in definition for {def_name}. " +\
            f"Def and Def-expand tags cannot be in definitions."
 
 
-@hed_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, actual_code=ValidationErrors.HED_DEFINITION_INVALID)
+@hed_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, actual_code=ValidationErrors.DEFINITION_INVALID)
 def def_error_wrong_group_tags(def_name, tag_list):
     tag_list_strings = [str(tag) for tag in tag_list]
     return f"Too many group tags found in definition for {def_name}.  Expected 1, found: {tag_list_strings}"
 
 
-@hed_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, actual_code=ValidationErrors.HED_DEFINITION_INVALID)
+@hed_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, actual_code=ValidationErrors.DEFINITION_INVALID)
 def def_error_wrong_placeholder_count(def_name, expected_count, tag_list):
     tag_list_strings = [str(tag) for tag in tag_list]
     return f"Incorrect number placeholder tags found in definition for {def_name}.  " + \
            f"Expected {expected_count}, found: {tag_list_strings}"
 
 
-@hed_error(DefinitionErrors.DUPLICATE_DEFINITION, actual_code=ValidationErrors.HED_DEFINITION_INVALID)
+@hed_error(DefinitionErrors.DUPLICATE_DEFINITION, actual_code=ValidationErrors.DEFINITION_INVALID)
 def def_error_duplicate_definition(def_name):
     return f"Duplicate definition found for '{def_name}'."
 
 
-@hed_error(DefinitionErrors.TAG_IN_SCHEMA, actual_code=ValidationErrors.HED_DEFINITION_INVALID)
+@hed_error(DefinitionErrors.TAG_IN_SCHEMA, actual_code=ValidationErrors.DEFINITION_INVALID)
 def def_error_tag_already_in_schema(def_name):
     return f"Term '{def_name}' already used as term in schema and cannot be re-used as a definition."
 
 
-@hed_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, actual_code=ValidationErrors.HED_DEFINITION_INVALID)
+@hed_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, actual_code=ValidationErrors.DEFINITION_INVALID)
 def def_error_invalid_def_extension(def_name):
     return f"Term '{def_name}' has an invalid extension.  Definitions can only have one term."
 

hed/errors/error_reporter.py

@@ -21,6 +21,8 @@
     ErrorContext.SIDECAR_KEY_NAME,
     ErrorContext.ROW,
     ErrorContext.COLUMN,
+    ErrorContext.LINE,
+    ErrorContext.HED_STRING,
     ErrorContext.SCHEMA_SECTION,
     ErrorContext.SCHEMA_TAG,
     ErrorContext.SCHEMA_ATTRIBUTE,
@@ -29,6 +31,7 @@
 # ErrorContext which is expected to be int based.
 int_sort_list = [
     ErrorContext.ROW,
+    ErrorContext.COLUMN,
 ]
 
 hed_string_sort_list = [
@@ -414,34 +417,6 @@ def filter_issues_by_severity(issues_list, severity):
         return [issue for issue in issues_list if issue['severity'] <= severity]
 
 
-def get_exception_issue_string(issues, title=None):
-    """ Return a string with issues list flatted into single string, one issue per line.
-        Possibly being deprecated.
-
-    Parameters:
-        issues (list):  A list of strings containing issues to print.
-        title (str or None): An optional title that will always show up first if present.
-
-    Returns:
-        str: A str containing printable version of the issues or ''.
-
-    """
-    issue_str = ''
-    if issues:
-        issue_list = []
-        for issue in issues:
-            this_str = f"{issue['message']}"
-            if 'code' in issue:
-                this_str = f"{issue['code']}:" + this_str
-            if 'line_number' in issue:
-                this_str = this_str + f"\n\tLine number {issue['line_number']}: {issue.get('line', '')} "
-            issue_list.append(this_str)
-        issue_str += '\n' + '\n'.join(issue_list)
-    if title:
-        issue_str = title + '\n' + issue_str
-    return issue_str
-
-
 def sort_issues(issues, reverse=False):
     """Sorts a list of issues by the error context values.
 
@@ -556,6 +531,7 @@ def _format_single_context_string(context_type, context, tab_count=0):
         ErrorContext.ROW: f'Issues in row {context}:',
         ErrorContext.COLUMN: f'Issues in column {context}:',
         ErrorContext.CUSTOM_TITLE: context,
+        ErrorContext.LINE: f"Line: {context}",
         ErrorContext.HED_STRING: f"hed string: {context}",
         ErrorContext.SCHEMA_SECTION: f"Schema Section: {context}",
         ErrorContext.SCHEMA_TAG: f"Source tag: {context}",

hed/errors/error_types.py

@@ -13,6 +13,7 @@ class ErrorContext:
     SIDECAR_KEY_NAME = 'ec_sidecarKeyName'
     ROW = 'ec_row'
     COLUMN = 'ec_column'
+    LINE = "ec_line"
     HED_STRING = 'ec_HedString'
     SCHEMA_SECTION = 'ec_section'
     SCHEMA_TAG = 'ec_schema_tag'
@@ -25,6 +26,7 @@ class ValidationErrors:
     COMMA_MISSING = 'COMMA_MISSING'
     DEF_EXPAND_INVALID = "DEF_EXPAND_INVALID"
     DEF_INVALID = "DEF_INVALID"
+    DEFINITION_INVALID = "DEFINITION_INVALID"
 
     # NOT OFFICIAL
     HED_DEF_UNMATCHED = "HED_DEF_UNMATCHED"
@@ -37,7 +39,6 @@ class ValidationErrors:
     HED_DEF_EXPAND_VALUE_EXTRA = "HED_DEF_EXPAND_VALUE_EXTRA"
     # END NOT OFFICIAL
 
-    HED_DEFINITION_INVALID = "HED_DEFINITION_INVALID"
     HED_NODE_NAME_EMPTY = 'HED_NODE_NAME_EMPTY'
     HED_ONSET_OFFSET_ERROR = 'HED_ONSET_OFFSET_ERROR'
     HED_PARENTHESES_MISMATCH = 'HED_PARENTHESES_MISMATCH'
@@ -110,7 +111,7 @@ class SchemaWarnings:
     NON_PLACEHOLDER_HAS_CLASS = 'NON_PLACEHOLDER_HAS_CLASS'
 
 
-# These are all HED_DEFINITION_INVALID errors
+# These are all DEFINITION_INVALID errors
 class DefinitionErrors:
     DEF_TAG_IN_DEFINITION = 'DEF_TAG_IN_DEFINITION'
     WRONG_NUMBER_GROUP_TAGS = 'wrongNumberGroupTags'

hed/schema/schema_io/wiki2schema.py

@@ -5,6 +5,7 @@
 
 from hed.schema.hed_schema_constants import HedSectionKey, HedKey
 from hed.errors.exceptions import HedFileError, HedExceptions
+from hed.errors import ErrorContext, error_reporter
 from hed.schema import HedSchema
 from hed.schema import schema_validation_util
 from hed.schema.schema_io import wiki_constants
@@ -92,7 +93,7 @@ def __init__(self, wiki_file_path, schema_as_string):
             raise HedFileError(HedExceptions.FILE_NOT_FOUND, e.strerror, wiki_file_path)
 
         if self.fatal_errors:
-            self.fatal_errors.sort(key = lambda x: x.get("line_number", -1))
+            self.fatal_errors = error_reporter.sort_issues(self.fatal_errors)
             raise HedFileError(HedExceptions.HED_WIKI_DELIMITERS_INVALID,
                                f"{len(self.fatal_errors)} issues found when parsing schema.  See the .issues "
                                f"parameter on this exception for more details.", self.filename,
@@ -596,6 +597,9 @@ def _add_single_line(self, line_number, tag_line, key_class, element_name=None):
 
     def _add_fatal_error(self, line_number, line, warning_message="Schema term is empty or the line is malformed"):
         self.fatal_errors.append(
-            {"line_number": line_number,
-             "line": line,
-             "message": warning_message})
+            {'error_code': HedExceptions.HED_WIKI_DELIMITERS_INVALID,
+             ErrorContext.ROW: line_number,
+             ErrorContext.LINE: line,
+             "message": f"ERROR: {warning_message}"
+             }
+        )

spec_tests/test_errors.py

@@ -17,6 +17,7 @@
     'COMMA_MISSING',
     "DEF_EXPAND_INVALID",
     "DEF_INVALID",
+    "DEFINITION_INVALID"
 ]
 
 skip_tests = ["VERSION_DEPRECATED", "CHARACTER_INVALID", "STYLE_WARNING"]

tests/models/test_sidecar.py

@@ -104,7 +104,7 @@ def test_duplicate_def(self):
         duplicate_dict = sidecar.extract_definitions(hed_schema=self.hed_schema)
         issues = sidecar.validate(self.hed_schema, extra_def_dicts=duplicate_dict, error_handler=ErrorHandler(False))
         self.assertEqual(len(issues), 5)
-        self.assertTrue(issues[0]['code'], ValidationErrors.HED_DEFINITION_INVALID)
+        self.assertTrue(issues[0]['code'], ValidationErrors.DEFINITION_INVALID)
 
     def test_save_load(self):
         sidecar = Sidecar(self.json_def_filename)

tests/schema/test_schema_wiki_fatal_errors.py

@@ -62,12 +62,47 @@ def test_invalid_schema(self):
             with self.assertRaises(HedFileError) as context:
                 schema.load_schema(full_filename)
                 # all of these should produce exceptions.
+            from hed.errors import ErrorHandler, ErrorContext, SchemaErrors, get_printable_issue_string
+            # Verify basic properties of exception
+            expected_line_numbers = self.expected_line_numbers.get(filename, [])
+            if expected_line_numbers:
+                for issue, expected in zip(context.exception.issues, expected_line_numbers):
+                    self.assertEqual(issue[ErrorContext.ROW], expected)
+
+
+            issues = context.exception.issues
+
+            self.assertIsInstance(get_printable_issue_string(issues), str)
+
+            self.assertTrue(context.exception.args[0] == error)
+            self.assertTrue(context.exception.filename == full_filename)
 
+
+    def test_merging_errors_schema(self):
+        for filename, error in self.files_and_errors.items():
+            full_filename = self.full_base_folder + filename
+            with self.assertRaises(HedFileError) as context:
+                schema.load_schema(full_filename)
+                # all of these should produce exceptions.
+            from hed.errors import ErrorHandler, ErrorContext, SchemaErrors, get_printable_issue_string
             # Verify basic properties of exception
             expected_line_numbers = self.expected_line_numbers.get(filename, [])
             if expected_line_numbers:
                 for issue, expected in zip(context.exception.issues, expected_line_numbers):
-                    self.assertEqual(issue["line_number"], expected)
+                    self.assertEqual(issue[ErrorContext.ROW], expected)
+
+            error_handler = ErrorHandler()
+
+            error_handler.push_error_context(ErrorContext.ROW, 1)
+            error_handler.push_error_context(ErrorContext.COLUMN, 2)
+
+            issues = error_handler.format_error_with_context(SchemaErrors.HED_SCHEMA_ATTRIBUTE_INVALID,
+                                                             "error_attribute", source_tag="error_tag")
+            error_handler.pop_error_context()
+            error_handler.pop_error_context()
+
+            issues += context.exception.issues
+            self.assertIsInstance(get_printable_issue_string(issues), str)
 
             self.assertTrue(context.exception.args[0] == error)
             self.assertTrue(context.exception.filename == full_filename)