hed-standard · VisLab · Jun 1, 2023 · Apr 12, 2023 · Apr 12, 2023 · Apr 12, 2023
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -16,4 +16,3 @@ updates:
         interval: "daily"
     target-branch: "develop"
     directory: /
-    automerge: true
diff --git a/.github/workflows/spec_tests.yaml b/.github/workflows/spec_tests.yaml
@@ -33,8 +33,7 @@ jobs:
 
       - name: Test with unittest
         run: |
-           python -m unittest spec_tests/* > test_results.txt
-        continue-on-error: true
+           python -m unittest spec_tests/test_errors.py > test_results.txt
 
       - name: Upload spec test results
         uses: actions/upload-artifact@v3

diff --git a/README.md b/README.md
@@ -1,3 +1,7 @@
+[![Maintainability](https://api.codeclimate.com/v1/badges/11bf2329590e7b0164ba/maintainability)](https://codeclimate.com/github/hed-standard/hed-python/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/11bf2329590e7b0164ba/test_coverage)](https://codeclimate.com/github/hed-standard/hed-python/test_coverage)
+![PyPI - Status](https://img.shields.io/pypi/v/hedtools)
+
 # HEDTools - Python
 HED (Hierarchical Event Descriptors) is a framework for systematically describing
 both laboratory and real-world events as well as other experimental metadata.
@@ -95,7 +99,7 @@ Use `hed.schema.set_cache_directory` to change the location.
 The HED cache can be shared across processes.
 
 Starting with `hedtools 0.2.0` local copies of the most recent schema versions
-are stored within the code modules for easy access.
+are stored within the code modules for easy access.  
 
 ### Other links of interest
 

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,12 +1,10 @@
 defusedxml>=0.7.1
-inflect>=5.5.1
+inflect>=6.0.2
 myst-parser>=0.18.1
-openpyxl>=3.0.9
+numpy>=1.21.6
+openpyxl>=3.1.0
 pandas>=1.3.5
-portalocker>=2.4.0
-semantic_version>=2.9.0
+portalocker>=2.7.0
+semantic_version>=2.10.0
 Sphinx>=5.2.2
 sphinx_rtd_theme>=1.0.0
-
-# This is just needed for secure_filename and should probably be removed
-Werkzeug>=2.1.2
diff --git a/hed/errors/__init__.py b/hed/errors/__init__.py
@@ -1,4 +1,5 @@
 from .error_reporter import ErrorHandler, get_printable_issue_string, sort_issues
-from .error_types import DefinitionErrors, OnsetErrors, SchemaErrors, SchemaWarnings,  SidecarErrors, ValidationErrors
+from .error_types import DefinitionErrors, OnsetErrors, SchemaErrors, SchemaWarnings,  SidecarErrors, \
+    ValidationErrors, ColumnErrors
 from .error_types import ErrorContext, ErrorSeverity
 from .exceptions import HedExceptions, HedFileError
diff --git a/hed/errors/error_messages.py b/hed/errors/error_messages.py
diff --git a/hed/errors/error_reporter.py b/hed/errors/error_reporter.py
@@ -30,8 +30,7 @@
 
 # ErrorContext which is expected to be int based.
 int_sort_list = [
-    ErrorContext.ROW,
-    ErrorContext.COLUMN,
+    ErrorContext.ROW
 ]
 
 hed_string_sort_list = [
@@ -397,10 +396,9 @@ def val_error_unknown(*args, **kwargs):
 
         Returns:
             str: The error message.
-            dict: The extra args.
 
         """
-        return f"Unknown error.  Args: {str(args)}", kwargs
+        return f"Unknown error.  Args: {str(args), str(kwargs)}"
 
     @staticmethod
     def filter_issues_by_severity(issues_list, severity):

diff --git a/hed/errors/error_types.py b/hed/errors/error_types.py
@@ -27,8 +27,30 @@ class ValidationErrors:
     DEF_EXPAND_INVALID = "DEF_EXPAND_INVALID"
     DEF_INVALID = "DEF_INVALID"
     DEFINITION_INVALID = "DEFINITION_INVALID"
-
-    # NOT OFFICIAL
+    NODE_NAME_EMPTY = 'NODE_NAME_EMPTY'
+    ONSET_OFFSET_INSET_ERROR = 'ONSET_OFFSET_INSET_ERROR'
+    PARENTHESES_MISMATCH = 'PARENTHESES_MISMATCH'
+    PLACEHOLDER_INVALID = 'PLACEHOLDER_INVALID'
+    REQUIRED_TAG_MISSING = 'REQUIRED_TAG_MISSING'
+    SIDECAR_INVALID = 'SIDECAR_INVALID'
+    SIDECAR_KEY_MISSING = 'SIDECAR_KEY_MISSING'
+    STYLE_WARNING = "STYLE_WARNING"
+    TAG_EMPTY = 'TAG_EMPTY'
+    TAG_EXPRESSION_REPEATED = 'TAG_EXPRESSION_REPEATED'
+    TAG_EXTENDED = 'TAG_EXTENDED'
+    TAG_EXTENSION_INVALID = 'TAG_EXTENSION_INVALID'
+    TAG_GROUP_ERROR = "TAG_GROUP_ERROR"
+    TAG_INVALID = "TAG_INVALID"
+    TAG_NOT_UNIQUE = 'TAG_NOT_UNIQUE'
+    TAG_PREFIX_INVALID = 'TAG_PREFIX_INVALID'
+    TAG_REQUIRES_CHILD = 'TAG_REQUIRES_CHILD'
+    TILDES_UNSUPPORTED = 'TILDES_UNSUPPORTED'
+    UNITS_INVALID = 'UNITS_INVALID'
+    UNITS_MISSING = 'UNITS_MISSING'
+    VERSION_DEPRECATED = 'VERSION_DEPRECATED'
+    VALUE_INVALID = 'VALUE_INVALID'
+
+    # Internal codes
     HED_DEF_UNMATCHED = "HED_DEF_UNMATCHED"
     HED_DEF_VALUE_MISSING = "HED_DEF_VALUE_MISSING"
     HED_DEF_VALUE_EXTRA = "HED_DEF_VALUE_EXTRA"
@@ -37,57 +59,41 @@ class ValidationErrors:
     HED_DEF_EXPAND_UNMATCHED = "HED_DEF_EXPAND_UNMATCHED"
     HED_DEF_EXPAND_VALUE_MISSING = "HED_DEF_EXPAND_VALUE_MISSING"
     HED_DEF_EXPAND_VALUE_EXTRA = "HED_DEF_EXPAND_VALUE_EXTRA"
-    # END NOT OFFICIAL
-
-    HED_NODE_NAME_EMPTY = 'HED_NODE_NAME_EMPTY'
-    HED_ONSET_OFFSET_ERROR = 'HED_ONSET_OFFSET_ERROR'
-    HED_PARENTHESES_MISMATCH = 'HED_PARENTHESES_MISMATCH'
-    HED_PLACEHOLDER_INVALID = 'HED_PLACEHOLDER_INVALID'
-    HED_REQUIRED_TAG_MISSING = 'HED_REQUIRED_TAG_MISSING'
-    HED_SIDECAR_KEY_MISSING = 'HED_SIDECAR_KEY_MISSING'
-    HED_STYLE_WARNING = 'HED_STYLE_WARNING'
-    HED_TAG_EMPTY = 'HED_TAG_EMPTY'
-    HED_TAG_EXTENDED = 'HED_TAG_EXTENDED'
-    HED_TAG_GROUP_ERROR = "HED_TAG_GROUP_ERROR"
-    HED_TAG_INVALID = "HED_TAG_INVALID"
-    HED_TAG_NOT_UNIQUE = 'HED_TAG_NOT_UNIQUE'
+
     HED_TAG_REPEATED = 'HED_TAG_REPEATED'
     HED_TAG_REPEATED_GROUP = 'HED_TAG_REPEATED_GROUP'
-    HED_TAG_REQUIRES_CHILD = 'HED_TAG_REQUIRES_CHILD'
-    HED_TILDES_UNSUPPORTED = 'HED_TILDES_UNSUPPORTED'
-    HED_UNITS_INVALID = 'HED_UNITS_INVALID'
-    HED_UNITS_DEFAULT_USED = 'HED_UNITS_DEFAULT_USED'
-    HED_VALUE_INVALID = 'HED_VALUE_INVALID'
-    HED_LIBRARY_UNMATCHED = "HED_LIBRARY_UNMATCHED"
-    TAG_PREFIX_INVALID = "TAG_PREFIX_INVALID"
-    # HED_VERSION_WARNING
-
-    HED_MISSING_REQUIRED_COLUMN = "HED_MISSING_REQUIRED_COLUMN"
-    HED_UNKNOWN_COLUMN = "HED_UNKNOWN_COLUMN"
-    HED_DUPLICATE_COLUMN = "HED_DUPLICATE_COLUMN"
-    HED_BLANK_COLUMN = "HED_BLANK_COLUMN"
-
 
-    # Below here shows what the given error maps to
-    HED_GROUP_EMPTY = 'emptyHedGroup'
-    INVALID_TAG_CHARACTER = 'invalidTagCharacter'
-
-    # These are all HED_TAG_INVALID
-    INVALID_EXTENSION = 'invalidExtension'
     INVALID_PARENT_NODE = "invalidParent"
     NO_VALID_TAG_FOUND = "invalidTag"
 
-    # These are misc errors that need categorization.
+    HED_LIBRARY_UNMATCHED = "HED_LIBRARY_UNMATCHED"
+
     HED_TOP_LEVEL_TAG = "HED_TOP_LEVEL_TAG"
     HED_MULTIPLE_TOP_TAGS = "HED_MULTIPLE_TOP_TAGS"
     HED_TAG_GROUP_TAG = "HED_TAG_GROUP_TAG"
 
+    HED_GROUP_EMPTY = 'HED_GROUP_EMPTY'
+    # end internal codes
+
+
+    # Still being worked on below this line
+
+    HED_MISSING_REQUIRED_COLUMN = "HED_MISSING_REQUIRED_COLUMN"
+    HED_UNKNOWN_COLUMN = "HED_UNKNOWN_COLUMN"
+    SIDECAR_AND_OTHER_COLUMNS = "SIDECAR_AND_OTHER_COLUMNS"
+
+    DUPLICATE_COLUMN_IN_LIST = "DUPLICATE_COLUMN_IN_LIST"
+    DUPLICATE_COLUMN_BETWEEN_SOURCES = "DUPLICATE_COLUMN_BETWEEN_SOURCES"
+    HED_BLANK_COLUMN = "HED_BLANK_COLUMN"
+
+    # Below here shows what the given error maps to
+
+    INVALID_TAG_CHARACTER = 'invalidTagCharacter'
 
 
 
 class SidecarErrors:
     # These are for json sidecar validation errors(sidecars can also produce most normal validation errors)
-    SIDECAR_INVALID = "SIDECAR_INVALID" # this is the generic error reported for several later ones
     BLANK_HED_STRING = 'blankValueString'
     WRONG_HED_DATA_TYPE = 'wrongHedDataType'
     INVALID_POUND_SIGNS_VALUE = 'invalidNumberPoundSigns'
@@ -96,10 +102,13 @@ class SidecarErrors:
     SIDECAR_HED_USED_COLUMN = 'SIDECAR_HED_USED_COLUMN'
     SIDECAR_NA_USED = 'SIDECAR_NA_USED'
     SIDECAR_HED_USED = 'SIDECAR_HED_USED'
+    SIDECAR_BRACES_INVALID = "SIDECAR_BRACES_INVALID"
+
 
 class SchemaErrors:
     HED_SCHEMA_DUPLICATE_NODE = 'HED_SCHEMA_DUPLICATE_NODE'
     HED_SCHEMA_ATTRIBUTE_INVALID = 'HED_SCHEMA_ATTRIBUTE_INVALID'
+    HED_SCHEMA_DUPLICATE_FROM_LIBRARY = "SCHEMA_LIBRARY_INVALID"
 
 
 class SchemaWarnings:
@@ -109,27 +118,35 @@ class SchemaWarnings:
     HED_SCHEMA_CHARACTER_INVALID = "HED_SCHEMA_CHARACTER_INVALID"
     INVALID_CAPITALIZATION = 'invalidCaps'
     NON_PLACEHOLDER_HAS_CLASS = 'NON_PLACEHOLDER_HAS_CLASS'
+    INVALID_ATTRIBUTE = "INVALID_ATTRIBUTE"
 
 
-# These are all DEFINITION_INVALID errors
 class DefinitionErrors:
-    DEF_TAG_IN_DEFINITION = 'DEF_TAG_IN_DEFINITION'
-    WRONG_NUMBER_GROUP_TAGS = 'wrongNumberGroupTags'
+    # These are all DEFINITION_INVALID errors
     WRONG_NUMBER_PLACEHOLDER_TAGS = 'wrongNumberPlaceholderTags'
     DUPLICATE_DEFINITION = 'duplicateDefinition'
-    TAG_IN_SCHEMA = 'defAlreadyInSchema'
     INVALID_DEFINITION_EXTENSION = 'invalidDefExtension'
+    DEF_TAG_IN_DEFINITION = 'DEF_TAG_IN_DEFINITION'
+    NO_DEFINITION_CONTENTS = "NO_DEFINITION_CONTENTS"
+    PLACEHOLDER_NO_TAKES_VALUE = 'PLACEHOLDER_NO_TAKES_VALUE'
+
+    WRONG_NUMBER_TAGS = 'WRONG_NUMBER_TAGS'
+    WRONG_NUMBER_GROUPS = 'WRONG_NUMBER_GROUPS'
+    BAD_PROP_IN_DEFINITION = 'BAD_PROP_IN_DEFINITION'
+
+    BAD_DEFINITION_LOCATION = 'BAD_DEFINITION_LOCATION'
 
 
 class OnsetErrors:
+    # These are all ONSET_OFFSET_INSET_ERROR
     OFFSET_BEFORE_ONSET = "OFFSET_BEFORE_ONSET"
     ONSET_DEF_UNMATCHED = "ONSET_DEF_UNMATCHED"
     ONSET_WRONG_NUMBER_GROUPS = "ONSET_WRONG_NUMBER_GROUPS"
     ONSET_NO_DEF_TAG_FOUND = "ONSET_NO_DEF_TAG_FOUND"
     ONSET_PLACEHOLDER_WRONG = "ONSET_PLACEHOLDER_WRONG"
     ONSET_TOO_MANY_DEFS = "ONSET_TOO_MANY_DEFS"
     ONSET_TAG_OUTSIDE_OF_GROUP = "ONSET_TAG_OUTSIDE_OF_GROUP"
-
+    INSET_BEFORE_ONSET = "INSET_BEFORE_ONSET"
 
 class ColumnErrors:
     INVALID_COLUMN_REF = "INVALID_COLUMN_REF"

diff --git a/hed/errors/exceptions.py b/hed/errors/exceptions.py
@@ -14,7 +14,16 @@ class HedExceptions:
     # These are actual schema issues, not that the file cannot be found or parsed
     SCHEMA_HEADER_MISSING = 'HED_SCHEMA_HEADER_INVALID'
     HED_SCHEMA_HEADER_INVALID = 'HED_SCHEMA_HEADER_INVALID'
-    BAD_HED_LIBRARY_NAME = 'badHedLibraryName'
+
+    SCHEMA_LIBRARY_INVALID = "SCHEMA_LIBRARY_INVALID"
+    BAD_HED_LIBRARY_NAME = 'SCHEMA_LIBRARY_INVALID'
+    BAD_WITH_STANDARD = "SCHEMA_LIBRARY_INVALID"
+    BAD_WITH_STANDARD_VERSION = "SCHEMA_LIBRARY_INVALID"
+    ROOTED_TAG_INVALID = "SCHEMA_LIBRARY_INVALID"
+    ROOTED_TAG_HAS_PARENT = "SCHEMA_LIBRARY_INVALID"
+    ROOTED_TAG_DOES_NOT_EXIST = "SCHEMA_LIBRARY_INVALID"
+    IN_LIBRARY_IN_UNMERGED = "SCHEMA_LIBRARY_INVALID"
+
     HED_SCHEMA_VERSION_INVALID = 'HED_SCHEMA_VERSION_INVALID'
     SCHEMA_START_MISSING = 'HED_WIKI_SEPARATOR_INVALID'
     SCHEMA_END_INVALID = 'HED_WIKI_SEPARATOR_INVALID'
@@ -31,14 +40,14 @@ class HedExceptions:
 
 class HedFileError(Exception):
     """Exception raised when a file cannot be parsed due to being malformed, file IO, etc."""
-    def __init__(self, error_type, message, filename, issues=None):
-        self.error_type = error_type
+    def __init__(self, code, message, filename, issues=None):
+        self.code = code
         self.message = message
         self.filename = filename
         self.issues = issues
         if self.issues is None:
             self.issues = [
                 {'message': message,
                  ErrorContext.FILE_NAME: filename,
-                 'error_code': error_type}
+                 'code': code}
             ]
diff --git a/hed/models/__init__.py b/hed/models/__init__.py
@@ -14,3 +14,4 @@
 from .sidecar import Sidecar
 from .tabular_input import TabularInput
 from .timeseries_input import TimeseriesInput
+from .df_util import get_assembled, convert_to_form, shrink_defs, expand_defs, process_def_expands