Add squre bracket in column validation for spreadsheets. Update erro…

hed/errors/__init__.py

@@ -1,4 +1,4 @@
-from .error_reporter import ErrorHandler, get_exception_issue_string, get_printable_issue_string
+from .error_reporter import ErrorHandler, get_exception_issue_string, get_printable_issue_string, sort_issues
 from .error_types import DefinitionErrors, OnsetErrors, SchemaErrors, SchemaWarnings,  SidecarErrors, ValidationErrors
 from .error_types import ErrorContext, ErrorSeverity
 from .exceptions import HedExceptions, HedFileError

hed/errors/error_messages.py

@@ -6,7 +6,7 @@
 
 from hed.errors.error_reporter import hed_error, hed_tag_error
 from hed.errors.error_types import ValidationErrors, SchemaErrors, \
-    SidecarErrors, SchemaWarnings, ErrorSeverity, DefinitionErrors, OnsetErrors
+    SidecarErrors, SchemaWarnings, ErrorSeverity, DefinitionErrors, OnsetErrors, ColumnErrors
 
 
 @hed_tag_error(ValidationErrors.HED_UNITS_INVALID)
@@ -31,14 +31,14 @@ def val_error_tag_extended(tag, problem_tag):
     return f"Hed tag is extended. '{problem_tag}' in {tag}"
 
 
-@hed_error(ValidationErrors.HED_CHARACTER_INVALID)
+@hed_error(ValidationErrors.CHARACTER_INVALID)
 def val_error_invalid_char(source_string, char_index):
     character = source_string[char_index]
     return f'Invalid character "{character}" at index {char_index}"'
 
 
 @hed_tag_error(ValidationErrors.INVALID_TAG_CHARACTER, has_sub_tag=True,
-               actual_code=ValidationErrors.HED_CHARACTER_INVALID)
+               actual_code=ValidationErrors.CHARACTER_INVALID)
 def val_error_invalid_tag_character(tag, problem_tag):
     return f"Invalid character '{problem_tag}' in {tag}"
 
@@ -49,7 +49,7 @@ def val_error_tildes_not_supported(source_string, char_index):
     return f"Tildes not supported.  Replace (a ~ b ~ c) with (a, (b, c)).  '{character}' at index {char_index}'"
 
 
-@hed_error(ValidationErrors.HED_COMMA_MISSING)
+@hed_error(ValidationErrors.COMMA_MISSING)
 def val_error_comma_missing(tag):
     return f"Comma missing after - '{tag}'"
 
@@ -143,27 +143,44 @@ def val_error_sidecar_key_missing(invalid_key, category_keys):
     return f"Category key '{invalid_key}' does not exist in column.  Valid keys are: {category_keys}"
 
 
-@hed_tag_error(ValidationErrors.HED_DEF_UNMATCHED)
-def val_error_def_unmatched(tag):
-    return f"A data-recording’s Def tag cannot be matched to definition.  Tag: '{tag}'"
 
 
-@hed_tag_error(ValidationErrors.HED_DEF_EXPAND_INVALID)
+@hed_tag_error(ValidationErrors.HED_DEF_EXPAND_INVALID, actual_code=ValidationErrors.DEF_EXPAND_INVALID)
 def val_error_bad_def_expand(tag, actual_def, found_def):
     return f"A data-recording’s Def-expand tag does not match the given definition." + \
            f"Tag: '{tag}'.  Actual Def: {actual_def}.  Found Def: {found_def}"
 
 
-@hed_tag_error(ValidationErrors.HED_DEF_VALUE_MISSING, actual_code=ValidationErrors.HED_DEF_VALUE_INVALID)
+@hed_tag_error(ValidationErrors.HED_DEF_UNMATCHED, actual_code=ValidationErrors.DEF_INVALID)
+def val_error_def_unmatched(tag):
+    return f"A data-recording’s Def tag cannot be matched to definition.  Tag: '{tag}'"
+
+
+@hed_tag_error(ValidationErrors.HED_DEF_VALUE_MISSING, actual_code=ValidationErrors.DEF_INVALID)
 def val_error_def_value_missing(tag):
     return f"A def tag requires a placeholder value, but was not given one.  Definition: '{tag}'"
 
 
-@hed_tag_error(ValidationErrors.HED_DEF_VALUE_EXTRA, actual_code=ValidationErrors.HED_DEF_VALUE_INVALID)
+@hed_tag_error(ValidationErrors.HED_DEF_VALUE_EXTRA, actual_code=ValidationErrors.DEF_INVALID)
 def val_error_def_value_extra(tag):
     return f"A def tag does not take a placeholder value, but was given one.  Definition: '{tag}"
 
 
+@hed_tag_error(ValidationErrors.HED_DEF_EXPAND_UNMATCHED, actual_code=ValidationErrors.DEF_EXPAND_INVALID)
+def val_error_def_expand_unmatched(tag):
+    return f"A data-recording’s Def-expand tag cannot be matched to definition.  Tag: '{tag}'"
+
+
+@hed_tag_error(ValidationErrors.HED_DEF_EXPAND_VALUE_MISSING, actual_code=ValidationErrors.DEF_EXPAND_INVALID)
+def val_error_def_expand_value_missing(tag):
+    return f"A Def-expand tag requires a placeholder value, but was not given one.  Definition: '{tag}'"
+
+
+@hed_tag_error(ValidationErrors.HED_DEF_EXPAND_VALUE_EXTRA, actual_code=ValidationErrors.DEF_EXPAND_INVALID)
+def val_error_def_expand_value_extra(tag):
+    return f"A Def-expand tag does not take a placeholder value, but was given one.  Definition: '{tag}"
+
+
 @hed_tag_error(ValidationErrors.HED_TOP_LEVEL_TAG, actual_code=ValidationErrors.HED_TAG_GROUP_ERROR)
 def val_error_top_level_tag(tag):
     return f"A tag that must be in a top level group was found in another location.  {str(tag)}"
@@ -342,3 +359,26 @@ def onset_wrong_placeholder(tag, has_placeholder):
     if has_placeholder:
         return f"Onset/offset def tag {tag} expects a placeholder value, but does not have one."
     return f"Onset/offset def tag {tag} should not have a placeholder, but has one."
+
+
+@hed_error(ColumnErrors.INVALID_COLUMN_REF)
+def invalid_column_ref(bad_refs):
+    return f"Bad column references found(columns do not exist): {bad_refs}"
+
+
+@hed_error(ColumnErrors.SELF_COLUMN_REF)
+def self_column_ref(self_ref):
+    return f"Column references itself: {self_ref}"
+
+
+@hed_error(ColumnErrors.NESTED_COLUMN_REF)
+def nested_column_ref(column_name, ref_column):
+    return f"Column {column_name} has a nested reference to {ref_column}.  " \
+           f"Column reference columns cannot contain other column references."
+
+
+@hed_error(ColumnErrors.MALFORMED_COLUMN_REF)
+def nested_column_ref(column_name, index, symbol):
+    return f"Column {column_name} has a malformed column reference.  Improper symbol {symbol} found at index {index}."
+
+

hed/errors/error_reporter.py

@@ -10,6 +10,27 @@
 
 error_functions = {}
 
+# Controls if the default issue printing skips adding indentation for this context
+no_tab_context = {ErrorContext.HED_STRING, ErrorContext.SCHEMA_ATTRIBUTE}
+
+# Default sort ordering for issues list
+default_sort_list = [
+    ErrorContext.CUSTOM_TITLE,
+    ErrorContext.FILE_NAME,
+    ErrorContext.SIDECAR_COLUMN_NAME,
+    ErrorContext.SIDECAR_KEY_NAME,
+    ErrorContext.ROW,
+    ErrorContext.COLUMN,
+    ErrorContext.HED_STRING,
+    ErrorContext.SCHEMA_SECTION,
+    ErrorContext.SCHEMA_TAG,
+    ErrorContext.SCHEMA_ATTRIBUTE,
+]
+
+# ErrorContext which is expected to be int based.
+int_sort_list = [
+    ErrorContext.ROW,
+]
 
 def _register_error_function(error_type, wrapper_func):
     if error_type in error_functions:
@@ -153,19 +174,23 @@ def __init__(self, check_for_warnings=True):
         self.error_context = []
         self._check_for_warnings = check_for_warnings
 
-    def push_error_context(self, context_type, context, increment_depth_after=True):
+    def push_error_context(self, context_type, context):
         """ Push a new error context to narrow down error scope.
 
         Parameters:
             context_type (ErrorContext): A value from ErrorContext representing the type of scope.
             context (str, int, or HedString): The main value for the context_type.
-            increment_depth_after (bool): If True, add an extra tab to any subsequent errors in the scope.
 
         Notes:
             The context depends on the context_type. For ErrorContext.FILE_NAME this would be the actual filename.
 
         """
-        self.error_context.append((context_type, context, increment_depth_after))
+        if context is None:
+            if context_type in int_sort_list:
+                context = 0
+            else:
+                context_type = ""
+        self.error_context.append((context_type, context))
 
     def pop_error_context(self):
         """ Remove the last scope from the error context.
@@ -292,8 +317,8 @@ def _add_context_to_errors(error_object, error_context_to_add):
         """
         if error_object is None:
             error_object = {}
-        for (context_type, context, increment_count) in error_context_to_add:
-            error_object[context_type] = (context, increment_count)
+        for (context_type, context) in error_context_to_add:
+            error_object[context_type] = context
 
         return error_object
 
@@ -330,7 +355,7 @@ def _get_tag_span_to_error_object(error_object):
         else:
             return None, None
 
-        hed_string = error_object[ErrorContext.HED_STRING][0]
+        hed_string = error_object[ErrorContext.HED_STRING]
         span = hed_string._get_org_span(source_tag)
         return span
 
@@ -385,6 +410,7 @@ def filter_issues_by_severity(issues_list, severity):
 
 def get_exception_issue_string(issues, title=None):
     """ Return a string with issues list flatted into single string, one issue per line.
+        Possibly being deprecated.
 
     Parameters:
         issues (list):  A list of strings containing issues to print.
@@ -410,6 +436,29 @@ def get_exception_issue_string(issues, title=None):
     return issue_str
 
 
+def sort_issues(issues, reverse=False):
+    """Sorts a list of issues by the error context values.
+
+    Parameters:
+        issues (list): A list of dictionaries representing the issues to be sorted.
+        reverse (bool, optional): If True, sorts the list in descending order. Default is False.
+
+    Returns:
+        list: The sorted list of issues."""
+    def _get_keys(d):
+        result = []
+        for key in default_sort_list:
+            if key in int_sort_list:
+                result.append(d.get(key, -1))
+            else:
+                result.append(d.get(key, ""))
+        return tuple(result)
+
+    issues = sorted(issues, key=_get_keys, reverse=reverse)
+
+    return issues
+
+
 def get_printable_issue_string(issues, title=None, severity=None, skip_filename=True):
     """ Return a string with issues list flatted into single string, one per line.
 
@@ -471,7 +520,7 @@ def _get_context_from_issue(val_issue, skip_filename=True):
         if skip_filename and key == ErrorContext.FILE_NAME:
             continue
         if key.startswith("ec_"):
-            single_issue_context.append((key, *val_issue[key]))
+            single_issue_context.append((key, val_issue[key]))
 
     return single_issue_context
 
@@ -512,7 +561,7 @@ def _get_context_string(single_issue_context, last_used_context):
     """ Convert a single context list into the final human readable output form.
 
     Parameters:
-        single_issue_context (list): A list of tuples containing the context(context_type, context, increment_tab)
+        single_issue_context (list): A list of tuples containing the context(context_type, context)
         last_used_context (list): A list of tuples containing the last drawn context.
 
     Returns:
@@ -528,18 +577,18 @@ def _get_context_string(single_issue_context, last_used_context):
     tab_count = 0
     found_difference = False
     for i, context_tuple in enumerate(single_issue_context):
-        (context_type, context, increment_tab) = context_tuple
+        (context_type, context) = context_tuple
         if len(last_used_context) > i and not found_difference:
             last_drawn = last_used_context[i]
             # Was drawn, and hasn't changed.
             if last_drawn == context_tuple:
-                if increment_tab:
+                if context_type not in no_tab_context:
                     tab_count += 1
                 continue
 
         context_string += _format_single_context_string(context_type, context, tab_count)
         found_difference = True
-        if increment_tab:
+        if context_type not in no_tab_context:
             tab_count += 1
 
     tab_string = '\t' * tab_count

hed/errors/error_types.py

@@ -21,11 +21,22 @@ class ErrorContext:
 
 class ValidationErrors:
     # General validation errors
-    HED_CHARACTER_INVALID = 'HED_CHARACTER_INVALID'
-    HED_COMMA_MISSING = 'HED_COMMA_MISSING'
+    CHARACTER_INVALID = 'CHARACTER_INVALID'
+    COMMA_MISSING = 'COMMA_MISSING'
+    DEF_EXPAND_INVALID = "DEF_EXPAND_INVALID"
+    DEF_INVALID = "DEF_INVALID"
+
+    # NOT OFFICIAL
     HED_DEF_UNMATCHED = "HED_DEF_UNMATCHED"
+    HED_DEF_VALUE_MISSING = "HED_DEF_VALUE_MISSING"
+    HED_DEF_VALUE_EXTRA = "HED_DEF_VALUE_EXTRA"
+
     HED_DEF_EXPAND_INVALID = "HED_DEF_EXPAND_INVALID"
-    HED_DEF_VALUE_INVALID = "HED_DEF_VALUE_INVALID"
+    HED_DEF_EXPAND_UNMATCHED = "HED_DEF_EXPAND_UNMATCHED"
+    HED_DEF_EXPAND_VALUE_MISSING = "HED_DEF_EXPAND_VALUE_MISSING"
+    HED_DEF_EXPAND_VALUE_EXTRA = "HED_DEF_EXPAND_VALUE_EXTRA"
+    # END NOT OFFICIAL
+
     HED_DEFINITION_INVALID = "HED_DEFINITION_INVALID"
     HED_NODE_NAME_EMPTY = 'HED_NODE_NAME_EMPTY'
     HED_ONSET_OFFSET_ERROR = 'HED_ONSET_OFFSET_ERROR'
@@ -70,8 +81,7 @@ class ValidationErrors:
     HED_MULTIPLE_TOP_TAGS = "HED_MULTIPLE_TOP_TAGS"
     HED_TAG_GROUP_TAG = "HED_TAG_GROUP_TAG"
 
-    HED_DEF_VALUE_MISSING = "HED_DEF_VALUE_MISSING"
-    HED_DEF_VALUE_EXTRA = "HED_DEF_VALUE_EXTRA"
+
 
 
 class SidecarErrors:
@@ -117,3 +127,10 @@ class OnsetErrors:
     ONSET_PLACEHOLDER_WRONG = "ONSET_PLACEHOLDER_WRONG"
     ONSET_TOO_MANY_DEFS = "ONSET_TOO_MANY_DEFS"
     ONSET_TAG_OUTSIDE_OF_GROUP = "ONSET_TAG_OUTSIDE_OF_GROUP"
+
+
+class ColumnErrors:
+    INVALID_COLUMN_REF = "INVALID_COLUMN_REF"
+    SELF_COLUMN_REF = "SELF_COLUMN_REF"
+    NESTED_COLUMN_REF = "NESTED_COLUMN_REF"
+    MALFORMED_COLUMN_REF = "MALFORMED_COLUMN_REF"