Merging the refactored model code into develop

hed/__init__.py

@@ -7,12 +7,13 @@
 from hed.models.spreadsheet_input import SpreadsheetInput
 from hed.models.tabular_input import TabularInput
 from hed.models.sidecar import Sidecar
+from hed.models.definition_dict import DefinitionDict
+
 
 from hed.schema.hed_schema import HedSchema
 from hed.schema.hed_schema_group import HedSchemaGroup
 from hed.schema.hed_schema_io import get_schema, get_schema_versions, load_schema, load_schema_version
 
-from hed.validator.hed_validator import HedValidator
 
 # from hed import errors, models, schema, tools, validator
 

hed/errors/__init__.py

@@ -1,4 +1,4 @@
-from .error_reporter import ErrorHandler, get_exception_issue_string, get_printable_issue_string
+from .error_reporter import ErrorHandler, get_exception_issue_string, get_printable_issue_string, sort_issues
 from .error_types import DefinitionErrors, OnsetErrors, SchemaErrors, SchemaWarnings,  SidecarErrors, ValidationErrors
 from .error_types import ErrorContext, ErrorSeverity
 from .exceptions import HedExceptions, HedFileError

hed/errors/error_reporter.py

@@ -10,6 +10,30 @@
 
 error_functions = {}
 
+# Controls if the default issue printing skips adding indentation for this context
+no_tab_context = {ErrorContext.HED_STRING, ErrorContext.SCHEMA_ATTRIBUTE}
+
+# Default sort ordering for issues list
+default_sort_list = [
+    ErrorContext.CUSTOM_TITLE,
+    ErrorContext.FILE_NAME,
+    ErrorContext.SIDECAR_COLUMN_NAME,
+    ErrorContext.SIDECAR_KEY_NAME,
+    ErrorContext.ROW,
+    ErrorContext.COLUMN,
+    ErrorContext.SCHEMA_SECTION,
+    ErrorContext.SCHEMA_TAG,
+    ErrorContext.SCHEMA_ATTRIBUTE,
+]
+
+# ErrorContext which is expected to be int based.
+int_sort_list = [
+    ErrorContext.ROW,
+]
+
+hed_string_sort_list = [
+    ErrorContext.HED_STRING
+]
 
 def _register_error_function(error_type, wrapper_func):
     if error_type in error_functions:
@@ -43,8 +67,8 @@ def wrapper(*args, severity=default_severity, **kwargs):
             Returns:
                 list: A list of dict with the errors.=
             """
-            base_message, error_vars = func(*args, **kwargs)
-            error_object = ErrorHandler._create_error_object(actual_code, base_message, severity, **error_vars)
+            base_message = func(*args, **kwargs)
+            error_object = ErrorHandler._create_error_object(actual_code, base_message, severity)
             return error_object
 
         _register_error_function(error_type, wrapper_func=wrapper)
@@ -97,8 +121,8 @@ def wrapper(tag, index_in_tag, index_in_tag_end, *args, severity=default_severit
                 except AttributeError:
                     org_tag_text = str(tag)
 
-                base_message, error_vars = func(org_tag_text, problem_sub_tag, *args, **kwargs)
-                error_object = ErrorHandler._create_error_object(actual_code, base_message, severity, **error_vars,
+                base_message = func(org_tag_text, problem_sub_tag, *args, **kwargs)
+                error_object = ErrorHandler._create_error_object(actual_code, base_message, severity,
                                                                  index_in_tag=index_in_tag,
                                                                  index_in_tag_end=index_in_tag_end, source_tag=tag)
 
@@ -129,8 +153,8 @@ def wrapper(tag, *args, severity=default_severity, **kwargs):
                     org_tag_text = tag.get_original_hed_string()
                 else:
                     org_tag_text = str(tag)
-                base_message, error_vars = func(org_tag_text, *args, **kwargs)
-                error_object = ErrorHandler._create_error_object(actual_code, base_message, severity, **error_vars,
+                base_message = func(org_tag_text, *args, **kwargs)
+                error_object = ErrorHandler._create_error_object(actual_code, base_message, severity,
                                                                  source_tag=tag)
 
                 return error_object
@@ -148,23 +172,31 @@ def wrapper(tag, *args, severity=default_severity, **kwargs):
 
 
 class ErrorHandler:
-    def __init__(self):
+    def __init__(self, check_for_warnings=True):
         # The current (ordered) dictionary of contexts.
         self.error_context = []
+        self._check_for_warnings = check_for_warnings
 
-    def push_error_context(self, context_type, context, increment_depth_after=True):
+    def push_error_context(self, context_type, context):
         """ Push a new error context to narrow down error scope.
 
         Parameters:
             context_type (ErrorContext): A value from ErrorContext representing the type of scope.
             context (str, int, or HedString): The main value for the context_type.
-            increment_depth_after (bool): If True, add an extra tab to any subsequent errors in the scope.
 
         Notes:
             The context depends on the context_type. For ErrorContext.FILE_NAME this would be the actual filename.
 
         """
-        self.error_context.append((context_type, context, increment_depth_after))
+        if context is None:
+            from hed import HedString
+            if context_type in int_sort_list:
+                context = 0
+            elif context_type in hed_string_sort_list:
+                context = HedString("")
+            else:
+                context = ""
+        self.error_context.append((context_type, context))
 
     def pop_error_context(self):
         """ Remove the last scope from the error context.
@@ -191,8 +223,12 @@ def get_error_context_copy(self):
     def format_error_with_context(self, *args, **kwargs):
         error_object = ErrorHandler.format_error(*args, **kwargs)
         if self is not None:
-            self._add_context_to_errors(error_object[0], self.error_context)
-            self._update_error_with_char_pos(error_object[0])
+            actual_error = error_object[0]
+            # # Filter out warning errors
+            if not self._check_for_warnings and actual_error['severity'] >= ErrorSeverity.WARNING:
+                return []
+            self._add_context_to_errors(actual_error, self.error_context)
+            self._update_error_with_char_pos(actual_error)
 
         return error_object
 
@@ -225,26 +261,19 @@ def format_error(error_type, *args, actual_error=None, **kwargs):
 
         return [error_object]
 
-    def add_context_to_issues(self, issues):
+    def add_context_and_filter(self, issues):
+        """ Filter out warnings if requested, while adding context to issues.
+
+            issues(list):
+                list:   A list containing a single dictionary representing a single error.
+        """
+        if not self._check_for_warnings:
+            issues[:] = self.filter_issues_by_severity(issues, ErrorSeverity.ERROR)
+
         for error_object in issues:
             self._add_context_to_errors(error_object, self.error_context)
             self._update_error_with_char_pos(error_object)
 
-    def format_error_list(self, issue_params):
-        """ Convert an issue params list to an issues list.  This means adding the error context primarily.
-
-        Parameters:
-            issue_params (list):  A list of dict containing the unformatted issues list.
-
-        Returns:
-            list: A list of dict containing unformatted errors.
-
-        """
-        formatted_issues = []
-        for issue in issue_params:
-            formatted_issues += self.format_error(**issue)
-        return formatted_issues
-
     @staticmethod
     def format_error_from_context(error_type, error_context, *args, actual_error=None, **kwargs):
         """ Format an error based on the error type.
@@ -262,6 +291,7 @@ def format_error_from_context(error_type, error_context, *args, actual_error=Non
         Notes:
             - Generally the error_context is returned from _add_context_to_errors.
             - The actual_error is useful for errors that are shared like invalid character.
+            - This can't filter out warnings like the other ones.
 
         """
         error_func = error_functions.get(error_type)
@@ -293,8 +323,8 @@ def _add_context_to_errors(error_object, error_context_to_add):
         """
         if error_object is None:
             error_object = {}
-        for (context_type, context, increment_count) in error_context_to_add:
-            error_object[context_type] = (context, increment_count)
+        for (context_type, context) in error_context_to_add:
+            error_object[context_type] = context
 
         return error_object
 
@@ -331,7 +361,7 @@ def _get_tag_span_to_error_object(error_object):
         else:
             return None, None
 
-        hed_string = error_object[ErrorContext.HED_STRING][0]
+        hed_string = error_object[ErrorContext.HED_STRING]
         span = hed_string._get_org_span(source_tag)
         return span
 
@@ -386,6 +416,7 @@ def filter_issues_by_severity(issues_list, severity):
 
 def get_exception_issue_string(issues, title=None):
     """ Return a string with issues list flatted into single string, one issue per line.
+        Possibly being deprecated.
 
     Parameters:
         issues (list):  A list of strings containing issues to print.
@@ -411,6 +442,32 @@ def get_exception_issue_string(issues, title=None):
     return issue_str
 
 
+def sort_issues(issues, reverse=False):
+    """Sorts a list of issues by the error context values.
+
+    Parameters:
+        issues (list): A list of dictionaries representing the issues to be sorted.
+        reverse (bool, optional): If True, sorts the list in descending order. Default is False.
+
+    Returns:
+        list: The sorted list of issues."""
+    def _get_keys(d):
+        from hed import HedString
+        result = []
+        for key in default_sort_list:
+            if key in int_sort_list:
+                result.append(d.get(key, -1))
+            elif key in hed_string_sort_list:
+                result.append(d.get(key, HedString("")))
+            else:
+                result.append(d.get(key, ""))
+        return tuple(result)
+
+    issues = sorted(issues, key=_get_keys, reverse=reverse)
+
+    return issues
+
+
 def get_printable_issue_string(issues, title=None, severity=None, skip_filename=True):
     """ Return a string with issues list flatted into single string, one per line.
 
@@ -472,7 +529,7 @@ def _get_context_from_issue(val_issue, skip_filename=True):
         if skip_filename and key == ErrorContext.FILE_NAME:
             continue
         if key.startswith("ec_"):
-            single_issue_context.append((key, *val_issue[key]))
+            single_issue_context.append((key, val_issue[key]))
 
     return single_issue_context
 
@@ -513,7 +570,7 @@ def _get_context_string(single_issue_context, last_used_context):
     """ Convert a single context list into the final human readable output form.
 
     Parameters:
-        single_issue_context (list): A list of tuples containing the context(context_type, context, increment_tab)
+        single_issue_context (list): A list of tuples containing the context(context_type, context)
         last_used_context (list): A list of tuples containing the last drawn context.
 
     Returns:
@@ -529,18 +586,18 @@ def _get_context_string(single_issue_context, last_used_context):
     tab_count = 0
     found_difference = False
     for i, context_tuple in enumerate(single_issue_context):
-        (context_type, context, increment_tab) = context_tuple
+        (context_type, context) = context_tuple
         if len(last_used_context) > i and not found_difference:
             last_drawn = last_used_context[i]
             # Was drawn, and hasn't changed.
             if last_drawn == context_tuple:
-                if increment_tab:
+                if context_type not in no_tab_context:
                     tab_count += 1
                 continue
 
         context_string += _format_single_context_string(context_type, context, tab_count)
         found_difference = True
-        if increment_tab:
+        if context_type not in no_tab_context:
             tab_count += 1
 
     tab_string = '\t' * tab_count

hed/errors/error_types.py

@@ -21,11 +21,22 @@ class ErrorContext:
 
 class ValidationErrors:
     # General validation errors
-    HED_CHARACTER_INVALID = 'HED_CHARACTER_INVALID'
-    HED_COMMA_MISSING = 'HED_COMMA_MISSING'
+    CHARACTER_INVALID = 'CHARACTER_INVALID'
+    COMMA_MISSING = 'COMMA_MISSING'
+    DEF_EXPAND_INVALID = "DEF_EXPAND_INVALID"
+    DEF_INVALID = "DEF_INVALID"
+
+    # NOT OFFICIAL
     HED_DEF_UNMATCHED = "HED_DEF_UNMATCHED"
+    HED_DEF_VALUE_MISSING = "HED_DEF_VALUE_MISSING"
+    HED_DEF_VALUE_EXTRA = "HED_DEF_VALUE_EXTRA"
+
     HED_DEF_EXPAND_INVALID = "HED_DEF_EXPAND_INVALID"
-    HED_DEF_VALUE_INVALID = "HED_DEF_VALUE_INVALID"
+    HED_DEF_EXPAND_UNMATCHED = "HED_DEF_EXPAND_UNMATCHED"
+    HED_DEF_EXPAND_VALUE_MISSING = "HED_DEF_EXPAND_VALUE_MISSING"
+    HED_DEF_EXPAND_VALUE_EXTRA = "HED_DEF_EXPAND_VALUE_EXTRA"
+    # END NOT OFFICIAL
+
     HED_DEFINITION_INVALID = "HED_DEFINITION_INVALID"
     HED_NODE_NAME_EMPTY = 'HED_NODE_NAME_EMPTY'
     HED_ONSET_OFFSET_ERROR = 'HED_ONSET_OFFSET_ERROR'
@@ -47,6 +58,7 @@ class ValidationErrors:
     HED_UNITS_DEFAULT_USED = 'HED_UNITS_DEFAULT_USED'
     HED_VALUE_INVALID = 'HED_VALUE_INVALID'
     HED_LIBRARY_UNMATCHED = "HED_LIBRARY_UNMATCHED"
+    TAG_PREFIX_INVALID = "TAG_PREFIX_INVALID"
     # HED_VERSION_WARNING
 
     HED_MISSING_REQUIRED_COLUMN = "HED_MISSING_REQUIRED_COLUMN"
@@ -69,18 +81,20 @@ class ValidationErrors:
     HED_MULTIPLE_TOP_TAGS = "HED_MULTIPLE_TOP_TAGS"
     HED_TAG_GROUP_TAG = "HED_TAG_GROUP_TAG"
 
-    HED_DEF_VALUE_MISSING = "HED_DEF_VALUE_MISSING"
-    HED_DEF_VALUE_EXTRA = "HED_DEF_VALUE_EXTRA"
+
 
 
 class SidecarErrors:
     # These are for json sidecar validation errors(sidecars can also produce most normal validation errors)
+    SIDECAR_INVALID = "SIDECAR_INVALID" # this is the generic error reported for several later ones
     BLANK_HED_STRING = 'blankValueString'
     WRONG_HED_DATA_TYPE = 'wrongHedDataType'
     INVALID_POUND_SIGNS_VALUE = 'invalidNumberPoundSigns'
     INVALID_POUND_SIGNS_CATEGORY = 'tooManyPoundSigns'
     UNKNOWN_COLUMN_TYPE = 'sidecarUnknownColumn'
-
+    SIDECAR_HED_USED_COLUMN = 'SIDECAR_HED_USED_COLUMN'
+    SIDECAR_NA_USED = 'SIDECAR_NA_USED'
+    SIDECAR_HED_USED = 'SIDECAR_HED_USED'
 
 class SchemaErrors:
     HED_SCHEMA_DUPLICATE_NODE = 'HED_SCHEMA_DUPLICATE_NODE'
@@ -114,3 +128,10 @@ class OnsetErrors:
     ONSET_PLACEHOLDER_WRONG = "ONSET_PLACEHOLDER_WRONG"
     ONSET_TOO_MANY_DEFS = "ONSET_TOO_MANY_DEFS"
     ONSET_TAG_OUTSIDE_OF_GROUP = "ONSET_TAG_OUTSIDE_OF_GROUP"
+
+
+class ColumnErrors:
+    INVALID_COLUMN_REF = "INVALID_COLUMN_REF"
+    SELF_COLUMN_REF = "SELF_COLUMN_REF"
+    NESTED_COLUMN_REF = "NESTED_COLUMN_REF"
+    MALFORMED_COLUMN_REF = "MALFORMED_COLUMN_REF"

hed/errors/exceptions.py

@@ -8,6 +8,8 @@ class HedExceptions:
     CANNOT_PARSE_JSON = 'cannotParseJson'
     INVALID_EXTENSION = 'invalidExtension'
 
+    INVALID_DATAFRAME = 'INVALID_DATAFRAME'
+
     # These are actual schema issues, not that the file cannot be found or parsed
     SCHEMA_HEADER_MISSING = 'HED_SCHEMA_HEADER_INVALID'
     HED_SCHEMA_HEADER_INVALID = 'HED_SCHEMA_HEADER_INVALID'

hed/models/__init__.py

@@ -5,15 +5,12 @@
 from .column_metadata import ColumnMetadata, ColumnType
 from .definition_dict import DefinitionDict
 from .definition_entry import DefinitionEntry
-from .def_mapper import DefMapper
 from .expression_parser import QueryParser
 from .hed_group import HedGroup
 from .spreadsheet_input import SpreadsheetInput
-from .hed_ops import HedOps
 from .hed_string import HedString
 from .hed_string_group import HedStringGroup
 from .hed_tag import HedTag
-from .onset_mapper import OnsetMapper
 from .sidecar import Sidecar
 from .tabular_input import TabularInput
 from .timeseries_input import TimeseriesInput