diff --git a/.github/workflows/spec_tests.yaml b/.github/workflows/spec_tests.yaml index e41edd61f..b4e9dbf37 100644 --- a/.github/workflows/spec_tests.yaml +++ b/.github/workflows/spec_tests.yaml @@ -33,8 +33,7 @@ jobs: - name: Test with unittest run: | - python -m unittest spec_tests/* > test_results.txt - continue-on-error: true + python -m unittest spec_tests/test_errors.py > test_results.txt - name: Upload spec test results uses: actions/upload-artifact@v3 diff --git a/hed/errors/error_messages.py b/hed/errors/error_messages.py index 36d1b446f..8f77edaf7 100644 --- a/hed/errors/error_messages.py +++ b/hed/errors/error_messages.py @@ -9,24 +9,24 @@ SidecarErrors, SchemaWarnings, ErrorSeverity, DefinitionErrors, OnsetErrors, ColumnErrors -@hed_tag_error(ValidationErrors.HED_UNITS_INVALID) +@hed_tag_error(ValidationErrors.UNITS_INVALID) def val_error_invalid_unit(tag, units): units_string = ','.join(sorted(units)) return f'Invalid unit - "{tag}" valid units are "{units_string}"' -@hed_error(ValidationErrors.HED_TAG_EMPTY) +@hed_error(ValidationErrors.TAG_EMPTY) def val_error_extra_comma(source_string, char_index): character = source_string[char_index] return f"HED tags cannot be empty. Extra delimiter found: '{character}' at index {char_index}'" -@hed_tag_error(ValidationErrors.HED_GROUP_EMPTY, actual_code=ValidationErrors.HED_TAG_EMPTY) +@hed_tag_error(ValidationErrors.HED_GROUP_EMPTY, actual_code=ValidationErrors.TAG_EMPTY) def val_error_empty_group(tag): return f"HED tags cannot be empty. Extra delimiters found: '{tag}'" -@hed_tag_error(ValidationErrors.HED_TAG_EXTENDED, has_sub_tag=True, default_severity=ErrorSeverity.WARNING) +@hed_tag_error(ValidationErrors.TAG_EXTENDED, has_sub_tag=True, default_severity=ErrorSeverity.WARNING) def val_error_tag_extended(tag, problem_tag): return f"Hed tag is extended. '{problem_tag}' in {tag}" @@ -43,7 +43,7 @@ def val_error_invalid_tag_character(tag, problem_tag): return f"Invalid character '{problem_tag}' in {tag}" -@hed_error(ValidationErrors.HED_TILDES_UNSUPPORTED) +@hed_error(ValidationErrors.TILDES_UNSUPPORTED) def val_error_tildes_not_supported(source_string, char_index): character = source_string[char_index] return f"Tildes not supported. Replace (a ~ b ~ c) with (a, (b, c)). '{character}' at index {char_index}'" @@ -54,29 +54,29 @@ def val_error_comma_missing(tag): return f"Comma missing after - '{tag}'" -@hed_tag_error(ValidationErrors.HED_TAG_REPEATED) +@hed_tag_error(ValidationErrors.HED_TAG_REPEATED, actual_code=ValidationErrors.TAG_EXPRESSION_REPEATED) def val_error_duplicate_tag(tag): return f'Repeated tag - "{tag}"' -@hed_error(ValidationErrors.HED_TAG_REPEATED_GROUP) +@hed_error(ValidationErrors.HED_TAG_REPEATED_GROUP, actual_code=ValidationErrors.TAG_EXPRESSION_REPEATED) def val_error_duplicate_group(group): return f'Repeated group - "{group}"' -@hed_error(ValidationErrors.HED_PARENTHESES_MISMATCH) +@hed_error(ValidationErrors.PARENTHESES_MISMATCH) def val_error_parentheses(opening_parentheses_count, closing_parentheses_count): return f'Number of opening and closing parentheses are unequal. '\ f'{opening_parentheses_count} opening parentheses. {closing_parentheses_count} '\ 'closing parentheses' -@hed_tag_error(ValidationErrors.HED_TAG_REQUIRES_CHILD) +@hed_tag_error(ValidationErrors.TAG_REQUIRES_CHILD) def val_error_require_child(tag): return f"Descendant tag required - '{tag}'" -@hed_error(ValidationErrors.HED_TAG_NOT_UNIQUE) +@hed_error(ValidationErrors.TAG_NOT_UNIQUE) def val_error_multiple_unique(tag_prefix): return f"Multiple unique tags with prefix - '{tag_prefix}'" @@ -86,25 +86,24 @@ def val_error_prefix_invalid(tag, tag_prefix): return f"Prefixes can only contain alpha characters. - '{tag_prefix}'" -@hed_tag_error(ValidationErrors.INVALID_EXTENSION, actual_code=ValidationErrors.HED_TAG_INVALID) +@hed_tag_error(ValidationErrors.TAG_EXTENSION_INVALID) def val_error_invalid_extension(tag): return f'Invalid extension on tag - "{tag}"' -@hed_tag_error(ValidationErrors.INVALID_PARENT_NODE, has_sub_tag=True, actual_code=ValidationErrors.HED_TAG_INVALID) +@hed_tag_error(ValidationErrors.INVALID_PARENT_NODE, has_sub_tag=True, actual_code=ValidationErrors.TAG_EXTENSION_INVALID) def val_error_invalid_parent(tag, problem_tag, expected_parent_tag): - return f"In '{tag}', '{problem_tag}' appears as '{str(expected_parent_tag)}' and cannot be used " \ - f"as an extension." + return f"In '{tag}', '{problem_tag}' appears as '{str(expected_parent_tag)}' and cannot be used as an extension." -@hed_tag_error(ValidationErrors.NO_VALID_TAG_FOUND, has_sub_tag=True, actual_code=ValidationErrors.HED_TAG_INVALID) +@hed_tag_error(ValidationErrors.NO_VALID_TAG_FOUND, has_sub_tag=True, actual_code=ValidationErrors.TAG_INVALID) def val_error_no_valid_tag(tag, problem_tag): return f"'{problem_tag}' in {tag} is not a valid base hed tag." -@hed_tag_error(ValidationErrors.HED_VALUE_INVALID) +@hed_tag_error(ValidationErrors.VALUE_INVALID) def val_error_no_value(tag): - return f"''{tag}' has an invalid value portion." + return f"'{tag}' has an invalid value portion." @hed_error(ValidationErrors.HED_MISSING_REQUIRED_COLUMN, default_severity=ErrorSeverity.WARNING) @@ -128,17 +127,17 @@ def val_error_hed_duplicate_column(column_name): return f"Multiple columns have name {column_name}. This is not a fatal error, but discouraged." -@hed_tag_error(ValidationErrors.HED_LIBRARY_UNMATCHED) +@hed_tag_error(ValidationErrors.HED_LIBRARY_UNMATCHED, actual_code=ValidationErrors.TAG_PREFIX_INVALID) def val_error_unknown_prefix(tag, unknown_prefix, known_prefixes): return f"Tag '{tag} has unknown prefix '{unknown_prefix}'. Valid prefixes: {known_prefixes}" -@hed_tag_error(ValidationErrors.HED_NODE_NAME_EMPTY, has_sub_tag=True) +@hed_tag_error(ValidationErrors.NODE_NAME_EMPTY, has_sub_tag=True) def val_error_extra_slashes_spaces(tag, problem_tag): return f"Extra slashes or spaces '{problem_tag}' in tag '{tag}'" -@hed_error(ValidationErrors.HED_SIDECAR_KEY_MISSING, default_severity=ErrorSeverity.WARNING) +@hed_error(ValidationErrors.SIDECAR_KEY_MISSING, default_severity=ErrorSeverity.WARNING) def val_error_sidecar_key_missing(invalid_key, category_keys): return f"Category key '{invalid_key}' does not exist in column. Valid keys are: {category_keys}" @@ -181,34 +180,34 @@ def val_error_def_expand_value_extra(tag): return f"A Def-expand tag does not take a placeholder value, but was given one. Definition: '{tag}" -@hed_tag_error(ValidationErrors.HED_TOP_LEVEL_TAG, actual_code=ValidationErrors.HED_TAG_GROUP_ERROR) +@hed_tag_error(ValidationErrors.HED_TOP_LEVEL_TAG, actual_code=ValidationErrors.TAG_GROUP_ERROR) def val_error_top_level_tag(tag): return f"A tag that must be in a top level group was found in another location. {str(tag)}" -@hed_tag_error(ValidationErrors.HED_TAG_GROUP_TAG, actual_code=ValidationErrors.HED_TAG_GROUP_ERROR) +@hed_tag_error(ValidationErrors.HED_TAG_GROUP_TAG, actual_code=ValidationErrors.TAG_GROUP_ERROR) def val_error_tag_group_tag(tag): return f"A tag that must be in a group was found in another location. {str(tag)}" -@hed_tag_error(ValidationErrors.HED_MULTIPLE_TOP_TAGS, actual_code=ValidationErrors.HED_TAG_GROUP_ERROR) +@hed_tag_error(ValidationErrors.HED_MULTIPLE_TOP_TAGS, actual_code=ValidationErrors.TAG_GROUP_ERROR) def val_error_top_level_tags(tag, multiple_tags): tags_as_string = [str(tag) for tag in multiple_tags] return f"Multiple top level tags found in a single group. First one found: {str(tag)}. " + \ f"Remainder:{str(tags_as_string)}" -@hed_error(ValidationErrors.HED_REQUIRED_TAG_MISSING) +@hed_error(ValidationErrors.REQUIRED_TAG_MISSING) def val_warning_required_prefix_missing(tag_prefix): return f"Tag with prefix '{tag_prefix}' is required" -@hed_tag_error(ValidationErrors.HED_STYLE_WARNING, default_severity=ErrorSeverity.WARNING) +@hed_tag_error(ValidationErrors.STYLE_WARNING, default_severity=ErrorSeverity.WARNING) def val_warning_capitalization(tag): return f"First word not capitalized or camel case - '{tag}'" -@hed_tag_error(ValidationErrors.HED_UNITS_DEFAULT_USED, default_severity=ErrorSeverity.WARNING) +@hed_tag_error(ValidationErrors.UNITS_MISSING, default_severity=ErrorSeverity.WARNING) def val_warning_default_units_used(tag, default_unit): return f"No unit specified. Using '{default_unit}' as the default - '{tag}'" @@ -260,12 +259,12 @@ def sidecar_error_hed_data_type(expected_type, given_type): return f"Invalid HED string datatype sidecar. Should be '{expected_type}', but got '{given_type}'" -@hed_error(SidecarErrors.INVALID_POUND_SIGNS_VALUE, actual_code=ValidationErrors.HED_PLACEHOLDER_INVALID) +@hed_error(SidecarErrors.INVALID_POUND_SIGNS_VALUE, actual_code=ValidationErrors.PLACEHOLDER_INVALID) def sidecar_error_invalid_pound_sign_count(pound_sign_count): return f"There should be exactly one # character in a sidecar string. Found {pound_sign_count}" -@hed_error(SidecarErrors.INVALID_POUND_SIGNS_CATEGORY, actual_code=ValidationErrors.HED_PLACEHOLDER_INVALID) +@hed_error(SidecarErrors.INVALID_POUND_SIGNS_CATEGORY, actual_code=ValidationErrors.PLACEHOLDER_INVALID) def sidecar_error_too_many_pound_signs(pound_sign_count): return f"There should be no # characters in a category sidecar string. Found {pound_sign_count}" @@ -276,17 +275,17 @@ def sidecar_error_unknown_column(column_name): "Most likely the column definition in question needs a # sign to replace a number somewhere." -@hed_error(SidecarErrors.SIDECAR_HED_USED, actual_code=SidecarErrors.SIDECAR_INVALID) +@hed_error(SidecarErrors.SIDECAR_HED_USED, actual_code=ValidationErrors.SIDECAR_INVALID) def SIDECAR_HED_USED(): return "'HED' is a reserved name and cannot be used as a sidecar except in expected places." -@hed_error(SidecarErrors.SIDECAR_HED_USED_COLUMN, actual_code=SidecarErrors.SIDECAR_INVALID) +@hed_error(SidecarErrors.SIDECAR_HED_USED_COLUMN, actual_code=ValidationErrors.SIDECAR_INVALID) def SIDECAR_HED_USED_COLUMN(): return "'HED' is a reserved name and cannot be used as a sidecar column name" -@hed_error(SidecarErrors.SIDECAR_NA_USED, actual_code=SidecarErrors.SIDECAR_INVALID) +@hed_error(SidecarErrors.SIDECAR_NA_USED, actual_code=ValidationErrors.SIDECAR_INVALID) def sidecar_na_used(column_name): return f"Invalid category key 'n/a' found in column {column_name}." @@ -294,19 +293,31 @@ def sidecar_na_used(column_name): @hed_tag_error(DefinitionErrors.DEF_TAG_IN_DEFINITION, actual_code=ValidationErrors.DEFINITION_INVALID) def def_error_def_tag_in_definition(tag, def_name): return f"Invalid tag {tag} found in definition for {def_name}. " +\ - f"Def and Def-expand tags cannot be in definitions." + f"Def, Def-expand, and Definition tags cannot be in definitions." -@hed_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, actual_code=ValidationErrors.DEFINITION_INVALID) +@hed_error(DefinitionErrors.NO_DEFINITION_CONTENTS, actual_code=ValidationErrors.DEFINITION_INVALID) +def def_error_no_group_tags(def_name): + return f"No group tag found in definition for {def_name}." + + +@hed_error(DefinitionErrors.WRONG_NUMBER_GROUPS, actual_code=ValidationErrors.DEFINITION_INVALID) def def_error_wrong_group_tags(def_name, tag_list): tag_list_strings = [str(tag) for tag in tag_list] return f"Too many group tags found in definition for {def_name}. Expected 1, found: {tag_list_strings}" +@hed_error(DefinitionErrors.WRONG_NUMBER_TAGS, actual_code=ValidationErrors.DEFINITION_INVALID) +def def_error_wrong_group_tags(def_name, tag_list): + tag_list_strings = [str(tag) for tag in tag_list] + return f"Too many tags found in definition for {def_name}. Expected 1, found: {tag_list_strings}" + + + @hed_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, actual_code=ValidationErrors.DEFINITION_INVALID) def def_error_wrong_placeholder_count(def_name, expected_count, tag_list): tag_list_strings = [str(tag) for tag in tag_list] - return f"Incorrect number placeholder tags found in definition for {def_name}. " + \ + return f"Incorrect number placeholders or placeholder tags found in definition for {def_name}. " + \ f"Expected {expected_count}, found: {tag_list_strings}" @@ -315,51 +326,61 @@ def def_error_duplicate_definition(def_name): return f"Duplicate definition found for '{def_name}'." -@hed_error(DefinitionErrors.TAG_IN_SCHEMA, actual_code=ValidationErrors.DEFINITION_INVALID) -def def_error_tag_already_in_schema(def_name): - return f"Term '{def_name}' already used as term in schema and cannot be re-used as a definition." +@hed_tag_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, actual_code=ValidationErrors.DEFINITION_INVALID) +def def_error_invalid_def_extension(tag, def_name): + return f"Tag '{str(tag)}' has an invalid placeholder in definition '{def_name}'" + + +@hed_error(DefinitionErrors.PLACEHOLDER_NO_TAKES_VALUE, actual_code=ValidationErrors.DEFINITION_INVALID) +def def_error_no_takes_value(def_name, placeholder_tag): + return f"Definition '{def_name}' has has a placeholder tag {str(placeholder_tag)} that isn't a takes value tag." + + +@hed_tag_error(DefinitionErrors.BAD_PROP_IN_DEFINITION, actual_code=ValidationErrors.DEFINITION_INVALID) +def def_error_no_takes_value(tag, def_name): + return f"Tag '{str(tag)}' in Definition '{def_name}' has has a tag with the unique or required attribute." -@hed_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, actual_code=ValidationErrors.DEFINITION_INVALID) -def def_error_invalid_def_extension(def_name): - return f"Term '{def_name}' has an invalid extension. Definitions can only have one term." +@hed_tag_error(DefinitionErrors.BAD_DEFINITION_LOCATION, actual_code=ValidationErrors.DEFINITION_INVALID) +def def_error_bad_location(tag): + return f"Tag '{str(tag)}' is found in a location it is not allowed to be." -@hed_tag_error(OnsetErrors.ONSET_DEF_UNMATCHED, actual_code=ValidationErrors.HED_ONSET_OFFSET_ERROR) +@hed_tag_error(OnsetErrors.ONSET_DEF_UNMATCHED, actual_code=ValidationErrors.ONSET_OFFSET_ERROR) def onset_error_def_unmatched(tag): return f"The def tag in an onset/offset tag is unmatched. Def tag: '{tag}'" -@hed_tag_error(OnsetErrors.OFFSET_BEFORE_ONSET, actual_code=ValidationErrors.HED_ONSET_OFFSET_ERROR) +@hed_tag_error(OnsetErrors.OFFSET_BEFORE_ONSET, actual_code=ValidationErrors.ONSET_OFFSET_ERROR) def onset_error_offset_before_onset(tag): return f"Offset tag '{tag}' does not have a matching onset." -@hed_tag_error(OnsetErrors.ONSET_NO_DEF_TAG_FOUND, actual_code=ValidationErrors.HED_ONSET_OFFSET_ERROR) +@hed_tag_error(OnsetErrors.ONSET_NO_DEF_TAG_FOUND, actual_code=ValidationErrors.ONSET_OFFSET_ERROR) def onset_no_def_found(tag): return f"'{tag}' tag has no def or def-expand tag in string." -@hed_tag_error(OnsetErrors.ONSET_TOO_MANY_DEFS, actual_code=ValidationErrors.HED_ONSET_OFFSET_ERROR) +@hed_tag_error(OnsetErrors.ONSET_TOO_MANY_DEFS, actual_code=ValidationErrors.ONSET_OFFSET_ERROR) def onset_too_many_defs(tag, tag_list): tag_list_strings = [str(tag) for tag in tag_list] return f"Too many def tags found in onset for {tag}. Expected 1, also found: {tag_list_strings}" -@hed_tag_error(OnsetErrors.ONSET_WRONG_NUMBER_GROUPS, actual_code=ValidationErrors.HED_ONSET_OFFSET_ERROR) +@hed_tag_error(OnsetErrors.ONSET_WRONG_NUMBER_GROUPS, actual_code=ValidationErrors.ONSET_OFFSET_ERROR) def onset_too_many_groups(tag, tag_list): tag_list_strings = [str(a_tag) for a_tag in tag_list] return f"An onset tag should have at most 2 sibling nodes, an offset tag should have 1. " +\ f"Found {len(tag_list_strings)}: {tag_list_strings}" -@hed_tag_error(OnsetErrors.ONSET_TAG_OUTSIDE_OF_GROUP, actual_code=ValidationErrors.HED_ONSET_OFFSET_ERROR) +@hed_tag_error(OnsetErrors.ONSET_TAG_OUTSIDE_OF_GROUP, actual_code=ValidationErrors.ONSET_OFFSET_ERROR) def onset_wrong_type_tag(tag, def_tag): return f"Onset def tag '{def_tag}' has an improper sibling tag '{tag}'. All onset context tags must be " + \ f"in a single group together." -@hed_tag_error(OnsetErrors.ONSET_PLACEHOLDER_WRONG, actual_code=ValidationErrors.HED_ONSET_OFFSET_ERROR) +@hed_tag_error(OnsetErrors.ONSET_PLACEHOLDER_WRONG, actual_code=ValidationErrors.ONSET_OFFSET_ERROR) def onset_wrong_placeholder(tag, has_placeholder): if has_placeholder: return f"Onset/offset def tag {tag} expects a placeholder value, but does not have one." diff --git a/hed/errors/error_reporter.py b/hed/errors/error_reporter.py index 61381e82a..8b4808c16 100644 --- a/hed/errors/error_reporter.py +++ b/hed/errors/error_reporter.py @@ -30,8 +30,7 @@ # ErrorContext which is expected to be int based. int_sort_list = [ - ErrorContext.ROW, - ErrorContext.COLUMN, + ErrorContext.ROW ] hed_string_sort_list = [ diff --git a/hed/errors/error_types.py b/hed/errors/error_types.py index 5da166247..00e2185bb 100644 --- a/hed/errors/error_types.py +++ b/hed/errors/error_types.py @@ -27,8 +27,30 @@ class ValidationErrors: DEF_EXPAND_INVALID = "DEF_EXPAND_INVALID" DEF_INVALID = "DEF_INVALID" DEFINITION_INVALID = "DEFINITION_INVALID" - - # NOT OFFICIAL + NODE_NAME_EMPTY = 'NODE_NAME_EMPTY' + ONSET_OFFSET_ERROR = 'ONSET_OFFSET_ERROR' + PARENTHESES_MISMATCH = 'PARENTHESES_MISMATCH' + PLACEHOLDER_INVALID = 'PLACEHOLDER_INVALID' + REQUIRED_TAG_MISSING = 'REQUIRED_TAG_MISSING' + SIDECAR_INVALID = 'SIDECAR_INVALID' + SIDECAR_KEY_MISSING = 'SIDECAR_KEY_MISSING' + STYLE_WARNING = "STYLE_WARNING" + TAG_EMPTY = 'TAG_EMPTY' + TAG_EXPRESSION_REPEATED = 'TAG_EXPRESSION_REPEATED' + TAG_EXTENDED = 'TAG_EXTENDED' + TAG_EXTENSION_INVALID = 'TAG_EXTENSION_INVALID' + TAG_GROUP_ERROR = "TAG_GROUP_ERROR" + TAG_INVALID = "TAG_INVALID" + TAG_NOT_UNIQUE = 'TAG_NOT_UNIQUE' + TAG_PREFIX_INVALID = 'TAG_PREFIX_INVALID' + TAG_REQUIRES_CHILD = 'TAG_REQUIRES_CHILD' + TILDES_UNSUPPORTED = 'TILDES_UNSUPPORTED' + UNITS_INVALID = 'UNITS_INVALID' + UNITS_MISSING = 'UNITS_MISSING' + VERSION_DEPRECATED = 'VERSION_DEPRECATED' + VALUE_INVALID = 'VALUE_INVALID' + + # Internal codes HED_DEF_UNMATCHED = "HED_DEF_UNMATCHED" HED_DEF_VALUE_MISSING = "HED_DEF_VALUE_MISSING" HED_DEF_VALUE_EXTRA = "HED_DEF_VALUE_EXTRA" @@ -37,57 +59,38 @@ class ValidationErrors: HED_DEF_EXPAND_UNMATCHED = "HED_DEF_EXPAND_UNMATCHED" HED_DEF_EXPAND_VALUE_MISSING = "HED_DEF_EXPAND_VALUE_MISSING" HED_DEF_EXPAND_VALUE_EXTRA = "HED_DEF_EXPAND_VALUE_EXTRA" - # END NOT OFFICIAL - - HED_NODE_NAME_EMPTY = 'HED_NODE_NAME_EMPTY' - HED_ONSET_OFFSET_ERROR = 'HED_ONSET_OFFSET_ERROR' - HED_PARENTHESES_MISMATCH = 'HED_PARENTHESES_MISMATCH' - HED_PLACEHOLDER_INVALID = 'HED_PLACEHOLDER_INVALID' - HED_REQUIRED_TAG_MISSING = 'HED_REQUIRED_TAG_MISSING' - HED_SIDECAR_KEY_MISSING = 'HED_SIDECAR_KEY_MISSING' - HED_STYLE_WARNING = 'HED_STYLE_WARNING' - HED_TAG_EMPTY = 'HED_TAG_EMPTY' - HED_TAG_EXTENDED = 'HED_TAG_EXTENDED' - HED_TAG_GROUP_ERROR = "HED_TAG_GROUP_ERROR" - HED_TAG_INVALID = "HED_TAG_INVALID" - HED_TAG_NOT_UNIQUE = 'HED_TAG_NOT_UNIQUE' + HED_TAG_REPEATED = 'HED_TAG_REPEATED' HED_TAG_REPEATED_GROUP = 'HED_TAG_REPEATED_GROUP' - HED_TAG_REQUIRES_CHILD = 'HED_TAG_REQUIRES_CHILD' - HED_TILDES_UNSUPPORTED = 'HED_TILDES_UNSUPPORTED' - HED_UNITS_INVALID = 'HED_UNITS_INVALID' - HED_UNITS_DEFAULT_USED = 'HED_UNITS_DEFAULT_USED' - HED_VALUE_INVALID = 'HED_VALUE_INVALID' + + INVALID_PARENT_NODE = "invalidParent" + NO_VALID_TAG_FOUND = "invalidTag" + HED_LIBRARY_UNMATCHED = "HED_LIBRARY_UNMATCHED" - TAG_PREFIX_INVALID = "TAG_PREFIX_INVALID" - # HED_VERSION_WARNING + + HED_TOP_LEVEL_TAG = "HED_TOP_LEVEL_TAG" + HED_MULTIPLE_TOP_TAGS = "HED_MULTIPLE_TOP_TAGS" + HED_TAG_GROUP_TAG = "HED_TAG_GROUP_TAG" + + HED_GROUP_EMPTY = 'HED_GROUP_EMPTY' + # end internal codes + + + # Still being worked on below this line HED_MISSING_REQUIRED_COLUMN = "HED_MISSING_REQUIRED_COLUMN" HED_UNKNOWN_COLUMN = "HED_UNKNOWN_COLUMN" HED_DUPLICATE_COLUMN = "HED_DUPLICATE_COLUMN" HED_BLANK_COLUMN = "HED_BLANK_COLUMN" - # Below here shows what the given error maps to - HED_GROUP_EMPTY = 'emptyHedGroup' - INVALID_TAG_CHARACTER = 'invalidTagCharacter' - - # These are all HED_TAG_INVALID - INVALID_EXTENSION = 'invalidExtension' - INVALID_PARENT_NODE = "invalidParent" - NO_VALID_TAG_FOUND = "invalidTag" - - # These are misc errors that need categorization. - HED_TOP_LEVEL_TAG = "HED_TOP_LEVEL_TAG" - HED_MULTIPLE_TOP_TAGS = "HED_MULTIPLE_TOP_TAGS" - HED_TAG_GROUP_TAG = "HED_TAG_GROUP_TAG" + INVALID_TAG_CHARACTER = 'invalidTagCharacter' class SidecarErrors: # These are for json sidecar validation errors(sidecars can also produce most normal validation errors) - SIDECAR_INVALID = "SIDECAR_INVALID" # this is the generic error reported for several later ones BLANK_HED_STRING = 'blankValueString' WRONG_HED_DATA_TYPE = 'wrongHedDataType' INVALID_POUND_SIGNS_VALUE = 'invalidNumberPoundSigns' @@ -97,6 +100,7 @@ class SidecarErrors: SIDECAR_NA_USED = 'SIDECAR_NA_USED' SIDECAR_HED_USED = 'SIDECAR_HED_USED' + class SchemaErrors: HED_SCHEMA_DUPLICATE_NODE = 'HED_SCHEMA_DUPLICATE_NODE' HED_SCHEMA_ATTRIBUTE_INVALID = 'HED_SCHEMA_ATTRIBUTE_INVALID' @@ -111,17 +115,24 @@ class SchemaWarnings: NON_PLACEHOLDER_HAS_CLASS = 'NON_PLACEHOLDER_HAS_CLASS' -# These are all DEFINITION_INVALID errors class DefinitionErrors: - DEF_TAG_IN_DEFINITION = 'DEF_TAG_IN_DEFINITION' - WRONG_NUMBER_GROUP_TAGS = 'wrongNumberGroupTags' + # These are all DEFINITION_INVALID errors WRONG_NUMBER_PLACEHOLDER_TAGS = 'wrongNumberPlaceholderTags' DUPLICATE_DEFINITION = 'duplicateDefinition' - TAG_IN_SCHEMA = 'defAlreadyInSchema' INVALID_DEFINITION_EXTENSION = 'invalidDefExtension' + DEF_TAG_IN_DEFINITION = 'DEF_TAG_IN_DEFINITION' + NO_DEFINITION_CONTENTS = "NO_DEFINITION_CONTENTS" + PLACEHOLDER_NO_TAKES_VALUE = 'PLACEHOLDER_NO_TAKES_VALUE' + + WRONG_NUMBER_TAGS = 'WRONG_NUMBER_TAGS' + WRONG_NUMBER_GROUPS = 'WRONG_NUMBER_GROUPS' + BAD_PROP_IN_DEFINITION = 'BAD_PROP_IN_DEFINITION' + + BAD_DEFINITION_LOCATION = 'BAD_DEFINITION_LOCATION' class OnsetErrors: + # These are all ONSET_OFFSET_ERROR OFFSET_BEFORE_ONSET = "OFFSET_BEFORE_ONSET" ONSET_DEF_UNMATCHED = "ONSET_DEF_UNMATCHED" ONSET_WRONG_NUMBER_GROUPS = "ONSET_WRONG_NUMBER_GROUPS" diff --git a/hed/models/base_input.py b/hed/models/base_input.py index f0e4209c2..09e0875c5 100644 --- a/hed/models/base_input.py +++ b/hed/models/base_input.py @@ -447,4 +447,19 @@ def combine_dataframe(dataframe): lambda x: ', '.join(filter(lambda e: bool(e) and e != "n/a", map(str, x))), axis=1 ) - return dataframe \ No newline at end of file + return dataframe + + def get_def_dict(self, hed_schema=None, extra_def_dicts=None): + """ Returns the definition dict for this file + + Note: Baseclass implementation returns just extra_def_dicts. + + Parameters: + hed_schema(HedSchema): used to identify tags to find definitions(if needed) + extra_def_dicts (list, DefinitionDict, or None): Extra dicts to add to the list. + + Returns: + DefinitionDict: A single definition dict representing all the data(and extra def dicts) + """ + from hed.models.definition_dict import DefinitionDict + return DefinitionDict(extra_def_dicts, hed_schema) \ No newline at end of file diff --git a/hed/models/def_expand_gather.py b/hed/models/def_expand_gather.py index 5d2b5c935..db1a8fd47 100644 --- a/hed/models/def_expand_gather.py +++ b/hed/models/def_expand_gather.py @@ -2,6 +2,82 @@ from hed.models import DefinitionDict, DefinitionEntry, HedString +class AmbiguousDef: + def __init__(self): + self.actual_defs = [] + self.placeholder_defs = [] + + def add_def(self, def_tag, def_expand_group): + group_tag = def_expand_group.get_first_group() + def_extension = def_tag.extension.split('/')[-1] + self.actual_defs.append(group_tag) + group_tag = group_tag.copy() + matching_tags = [tag for tag in group_tag.get_all_tags() if + tag.extension == def_extension] + + for tag in matching_tags: + tag.extension = "#" + self.placeholder_defs.append(group_tag) + + def validate(self): + """Validate the given ambiguous definition + + Returns: + bool: True if this is a valid definition with exactly 1 placeholder. + + raises: + ValueError: Raised if this is an invalid(not ambiguous) definition. + """ + # todo: improve this and get_group + placeholder_group = self.get_group() + if not placeholder_group: + raise ValueError("Invalid Definition") + placeholder_mask = [(tag.extension == "#") for tag in placeholder_group.get_all_tags()] + all_tags_list = [group.get_all_tags() for group in self.actual_defs] + for tags, placeholder in zip(zip(*all_tags_list), placeholder_mask): + if placeholder: + continue + + tag_set = set(tag.extension for tag in tags) + if len(tag_set) > 1: + raise ValueError("Invalid Definition") + + return placeholder_mask.count(True) == 1 + + @staticmethod + def _get_matching_value(tags): + """Get the matching value for a set of HedTag extensions. + + Parameters: + tags (iterator): The list of HedTags to find a matching value for. + + Returns: + str or None: The matching value if found, None otherwise. + """ + extensions = [tag.extension for tag in tags] + unique_extensions = set(extensions) + + if len(unique_extensions) == 1: + return unique_extensions.pop() + elif "#" in unique_extensions: + unique_extensions.remove("#") + if len(unique_extensions) == 1: + return unique_extensions.pop() + return None + + def get_group(self): + new_group = self.placeholder_defs[0].copy() + + all_tags_list = [group.get_all_tags() for group in self.placeholder_defs] + for tags, new_tag in zip(zip(*all_tags_list), new_group.get_all_tags()): + matching_val = self._get_matching_value(tags) + if matching_val is None: + return None + new_tag.extension = matching_val + + return new_group + + class DefExpandGatherer: """Class for gathering definitions from a series of def-expands, including possibly ambiguous ones""" def __init__(self, hed_schema, known_defs=None, ambiguous_defs=None, errors=None): @@ -14,10 +90,10 @@ def __init__(self, hed_schema, known_defs=None, ambiguous_defs=None, errors=None """ self.hed_schema = hed_schema - self.known_defs = known_defs if known_defs else {} self.ambiguous_defs = ambiguous_defs if ambiguous_defs else {} + self.ambiguous_defs_new = ambiguous_defs if ambiguous_defs else {} self.errors = errors if errors else {} - self.def_dict = DefinitionDict(self.known_defs, self.hed_schema) + self.def_dict = DefinitionDict(known_defs, self.hed_schema) def process_def_expands(self, hed_strings, known_defs=None): """Process the HED strings containing def-expand tags. @@ -38,11 +114,11 @@ def process_def_expands(self, hed_strings, known_defs=None): self.def_dict.add_definitions(known_defs, self.hed_schema) for i in hed_strings[def_expand_mask].index: string = hed_strings.loc[i] - self._process_hed_string(string) + self._process_def_expand(string) return self.def_dict, self.ambiguous_defs, self.errors - def _process_hed_string(self, string): + def _process_def_expand(self, string): """Process a single HED string to extract definitions and handle known and ambiguous definitions. Parameters: @@ -74,109 +150,57 @@ def _handle_known_definition(self, def_tag, def_expand_group, def_group): if def_group_contents: if def_group_contents != def_expand_group: - self.errors.setdefault(def_tag_name.lower(), []).append(def_group) - return True - - if def_tag_name.lower() in self.errors: - self.errors.setdefault(def_tag_name.lower(), []).append(def_group) + self.errors.setdefault(def_tag_name.lower(), []).append(def_expand_group.get_first_group()) return True - return False - - def _handle_ambiguous_definition(self, def_tag, def_expand_group): - """Handle ambiguous def-expand tag in a HED string. - - Parameters: - def_tag (HedTag): The def-expand tag. - def_expand_group (HedGroup): The group containing the def-expand tag. - """ - def_tag_name = def_tag.extension.split('/')[0] - has_extension = "/" in def_tag.extension - if not has_extension: group_tag = def_expand_group.get_first_group() self.def_dict.defs[def_tag_name.lower()] = DefinitionEntry(name=def_tag_name, contents=group_tag, takes_value=False, source_context=[]) - else: - self._process_ambiguous_extension(def_tag, def_expand_group) + return True + + # this is needed for the cases where we have a definition with errors, but it's not a known definition. + if def_tag_name.lower() in self.errors: + self.errors.setdefault(f"{def_tag_name.lower()}", []).append(def_expand_group.get_first_group()) + return True + + return False - def _process_ambiguous_extension(self, def_tag, def_expand_group): - """Process ambiguous extensions in a def-expand HED string. + def _handle_ambiguous_definition(self, def_tag, def_expand_group): + """Handle ambiguous def-expand tag in a HED string. Parameters: def_tag (HedTag): The def-expand tag. def_expand_group (HedGroup): The group containing the def-expand tag. """ def_tag_name = def_tag.extension.split('/')[0] - def_extension = def_tag.extension.split('/')[-1] - - matching_tags = [tag for tag in def_expand_group.get_all_tags() if - tag.extension == def_extension and tag != def_tag] - - for tag in matching_tags: - tag.extension = "#" - - group_tag = def_expand_group.get_first_group() - - these_defs = self.ambiguous_defs.setdefault(def_tag_name.lower(), []) - these_defs.append(group_tag) - - value_per_tag = [] - if len(these_defs) >= 1: - all_tags_list = [group.get_all_tags() for group in these_defs] - for tags in zip(*all_tags_list): - matching_val = self._get_matching_value(tags) - value_per_tag.append(matching_val) - - self._handle_value_per_tag(def_tag_name, value_per_tag, group_tag) - - def _handle_value_per_tag(self, def_tag_name, value_per_tag, group_tag): - """Handle the values per tag in ambiguous def-expand tag. - - Parameters: - def_tag_name (str): The name of the def-expand tag. - value_per_tag (list): The list of values per HedTag. - group_tag (HedGroup): The def expand contents - """ - if value_per_tag.count(None): - groups = self.ambiguous_defs.get(def_tag_name.lower(), []) - for group in groups: - self.errors.setdefault(def_tag_name.lower(), []).append(group) - + these_defs = self.ambiguous_defs.setdefault(def_tag_name.lower(), AmbiguousDef()) + these_defs.add_def(def_tag, def_expand_group) + + try: + if these_defs.validate(): + new_contents = these_defs.get_group() + self.def_dict.defs[def_tag_name.lower()] = DefinitionEntry(name=def_tag_name, contents=new_contents, + takes_value=True, + source_context=[]) + del self.ambiguous_defs[def_tag_name.lower()] + except ValueError as e: + for ambiguous_def in these_defs.placeholder_defs: + self.errors.setdefault(def_tag_name.lower(), []).append(ambiguous_def) del self.ambiguous_defs[def_tag_name.lower()] - return - - ambiguous_values = value_per_tag.count("#") - if ambiguous_values == 1: - new_contents = group_tag.copy() - for tag, value in zip(new_contents.get_all_tags(), value_per_tag): - if value is not None: - tag.extension = f"{value}" - self.def_dict.defs[def_tag_name.lower()] = DefinitionEntry(name=def_tag_name, contents=new_contents, - takes_value=True, - source_context=[]) - del self.ambiguous_defs[def_tag_name.lower()] + return @staticmethod - def _get_matching_value(tags): - """Get the matching value for a set of HedTag extensions. - - Parameters: - tags (iterator): The list of HedTags to find a matching value for. + def get_ambiguous_group(ambiguous_def): + """Turns an entry in the ambiguous_defs dict into a single HedGroup Returns: - str or None: The matching value if found, None otherwise. + HedGroup: the ambiguous definition with known placeholders filled in """ - extensions = [tag.extension for tag in tags] - unique_extensions = set(extensions) - - if len(unique_extensions) == 1: - return unique_extensions.pop() - elif "#" in unique_extensions: - unique_extensions.remove("#") - if len(unique_extensions) == 1: - return unique_extensions.pop() - return None + if not ambiguous_def: + # mostly to not crash, this shouldn't happen. + return HedString("") + return ambiguous_def.get_group() diff --git a/hed/models/definition_dict.py b/hed/models/definition_dict.py index fce00e1fa..ebe1af6f8 100644 --- a/hed/models/definition_dict.py +++ b/hed/models/definition_dict.py @@ -3,6 +3,7 @@ from hed.errors.error_types import DefinitionErrors from hed.errors.error_reporter import ErrorHandler from hed.models.model_constants import DefTagNames +from hed.schema.hed_schema_constants import HedKey class DefinitionDict: @@ -98,38 +99,11 @@ def check_for_definitions(self, hed_string_obj, error_handler=None): list: List of issues encountered in checking for definitions. Each issue is a dictionary. """ - new_def_issues = [] + def_issues = [] for definition_tag, group in hed_string_obj.find_top_level_tags(anchor_tags={DefTagNames.DEFINITION_KEY}): + group_tag, new_def_issues = self._find_group(definition_tag, group, error_handler) def_tag_name = definition_tag.extension - # initial validation - groups = group.groups() - if len(groups) > 1: - new_def_issues += \ - ErrorHandler.format_error_with_context(error_handler, - DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, - def_name=def_tag_name, tag_list=groups) - continue - if len(group.tags()) != 1: - new_def_issues += \ - ErrorHandler.format_error_with_context(error_handler, - DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, - def_name=def_tag_name, - tag_list=[tag for tag in group.tags() - if tag is not definition_tag]) - continue - - group_tag = groups[0] if groups else None - - # final validation - # Verify no other def or def expand tags found in group - if group_tag: - for def_tag in group.find_def_tags(recursive=True, include_groups=0): - new_def_issues += ErrorHandler.format_error_with_context(error_handler, - DefinitionErrors.DEF_TAG_IN_DEFINITION, - tag=def_tag, - def_name=def_tag_name) - continue def_takes_value = def_tag_name.lower().endswith("/#") if def_takes_value: def_tag_name = def_tag_name[:-len("/#")] @@ -138,22 +112,18 @@ def check_for_definitions(self, hed_string_obj, error_handler=None): if "/" in def_tag_lower or "#" in def_tag_lower: new_def_issues += ErrorHandler.format_error_with_context(error_handler, DefinitionErrors.INVALID_DEFINITION_EXTENSION, + tag=definition_tag, def_name=def_tag_name) + + if new_def_issues: + def_issues += new_def_issues continue - # # Verify placeholders here. - placeholder_tags = [] - if group_tag: - for tag in group_tag.get_all_tags(): - if "#" in str(tag): - placeholder_tags.append(tag) + new_def_issues += self._validate_contents(definition_tag, group_tag, error_handler) + new_def_issues += self._validate_placeholders(def_tag_name, group_tag, def_takes_value, error_handler) - if (len(placeholder_tags) == 1) != def_takes_value: - new_def_issues += ErrorHandler.format_error_with_context(error_handler, - DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, - def_name=def_tag_name, - tag_list=placeholder_tags, - expected_count=1 if def_takes_value else 0) + if new_def_issues: + def_issues += new_def_issues continue if error_handler: @@ -164,12 +134,95 @@ def check_for_definitions(self, hed_string_obj, error_handler=None): new_def_issues += ErrorHandler.format_error_with_context(error_handler, DefinitionErrors.DUPLICATE_DEFINITION, def_name=def_tag_name) + def_issues += new_def_issues continue self.defs[def_tag_lower] = DefinitionEntry(name=def_tag_name, contents=group_tag, takes_value=def_takes_value, source_context=context) - return new_def_issues + return def_issues + + def _validate_placeholders(self, def_tag_name, group, def_takes_value, error_handler): + new_issues = [] + placeholder_tags = [] + tags_with_issues = [] + if group: + for tag in group.get_all_tags(): + count = str(tag).count("#") + if count: + placeholder_tags.append(tag) + if count > 1: + tags_with_issues.append(tag) + + if tags_with_issues: + new_issues += ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, + def_name=def_tag_name, + tag_list=tags_with_issues, + expected_count=1 if def_takes_value else 0) + + if (len(placeholder_tags) == 1) != def_takes_value: + new_issues += ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, + def_name=def_tag_name, + tag_list=placeholder_tags, + expected_count=1 if def_takes_value else 0) + return new_issues + + if def_takes_value: + placeholder_tag = placeholder_tags[0] + if not placeholder_tag.is_takes_value_tag(): + new_issues += ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.PLACEHOLDER_NO_TAKES_VALUE, + def_name=def_tag_name, + placeholder_tag=placeholder_tag) + + return new_issues + + def _find_group(self, definition_tag, group, error_handler): + # initial validation + groups = group.groups() + issues = [] + if len(groups) > 1: + issues += \ + ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.WRONG_NUMBER_GROUPS, + def_name=definition_tag.extension, tag_list=groups) + elif len(groups) == 0: + issues += \ + ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.NO_DEFINITION_CONTENTS, + def_name=definition_tag.extension) + if len(group.tags()) != 1: + issues += \ + ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.WRONG_NUMBER_TAGS, + def_name=definition_tag.extension, + tag_list=[tag for tag in group.tags() + if tag is not definition_tag]) + + group_tag = groups[0] if groups else None + + return group_tag, issues + + def _validate_contents(self, definition_tag, group, error_handler): + issues = [] + if group: + for def_tag in group.find_tags({DefTagNames.DEF_KEY, DefTagNames.DEF_EXPAND_KEY, DefTagNames.DEFINITION_KEY}, recursive=True, + include_groups=0): + issues += ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.DEF_TAG_IN_DEFINITION, + tag=def_tag, + def_name=definition_tag.extension) + + for tag in group.get_all_tags(): + if tag.has_attribute(HedKey.Unique) or tag.has_attribute(HedKey.Required): + issues += ErrorHandler.format_error_with_context(error_handler, + DefinitionErrors.BAD_PROP_IN_DEFINITION, + tag=tag, + def_name=definition_tag.extension) + + return issues def construct_def_tags(self, hed_string_obj): """ Identify def/def-expand tag contents in the given string. diff --git a/hed/models/definition_entry.py b/hed/models/definition_entry.py index ba9f69b6a..5d3f09edd 100644 --- a/hed/models/definition_entry.py +++ b/hed/models/definition_entry.py @@ -22,9 +22,6 @@ def __init__(self, name, contents, takes_value, source_context): self.contents = contents self.takes_value = takes_value self.source_context = source_context - # self.tag_dict = {} - # if contents: - # add_group_to_dict(contents, self.tag_dict) def get_definition(self, replace_tag, placeholder_value=None, return_copy_of_tag=False): """ Return a copy of the definition with the tag expanded and the placeholder plugged in. @@ -66,28 +63,5 @@ def get_definition(self, replace_tag, placeholder_value=None, return_copy_of_tag startpos=replace_tag.span[0], endpos=replace_tag.span[1], contents=output_contents) return f"{DefTagNames.DEF_EXPAND_ORG_KEY}/{name}", output_contents -# -# def add_group_to_dict(group, tag_dict=None): -# """ Add the tags and their values from a HED group to a tag dictionary. -# -# Parameters: -# group (HedGroup): Contents to add to the tag dict. -# tag_dict (dict): The starting tag dictionary to add to. -# -# Returns: -# dict: The updated tag_dict containing the tags with a list of values. -# -# Notes: -# - Expects tags to have forms calculated already. -# -# """ -# if tag_dict is None: -# tag_dict = {} -# for tag in group.get_all_tags(): -# short_base_tag = tag.short_base_tag -# value = tag.extension -# value_dict = tag_dict.get(short_base_tag, {}) -# if value: -# value_dict[value] = '' -# tag_dict[short_base_tag] = value_dict -# return tag_dict + def __str__(self): + return str(self.contents) \ No newline at end of file diff --git a/hed/models/hed_group.py b/hed/models/hed_group.py index 64b789ec4..37947a89d 100644 --- a/hed/models/hed_group.py +++ b/hed/models/hed_group.py @@ -517,11 +517,11 @@ def find_def_tags(self, recursive=False, include_groups=3): for group in groups: for child in group.children: if isinstance(child, HedTag): - if child.short_base_tag.lower() == DefTagNames.DEF_KEY: + if child.short_base_tag == DefTagNames.DEF_ORG_KEY: def_tags.append((child, child, group)) else: for tag in child.tags(): - if tag.short_base_tag.lower() == DefTagNames.DEF_EXPAND_KEY: + if tag.short_base_tag == DefTagNames.DEF_EXPAND_ORG_KEY: def_tags.append((tag, child, group)) if include_groups == 0 or include_groups == 1 or include_groups == 2: diff --git a/hed/models/hed_string_group.py b/hed/models/hed_string_group.py index 5fb5a1fd5..14e639f51 100644 --- a/hed/models/hed_string_group.py +++ b/hed/models/hed_string_group.py @@ -30,7 +30,7 @@ def __init__(self, hed_string_obj_list): self._original_children = self._children def get_original_hed_string(self): - return "".join([group._hed_string for group in self._children]) + return ",".join([group._hed_string for group in self._children]) def sort(self): combined_string = HedString.from_hed_strings(self._children) diff --git a/hed/models/tabular_input.py b/hed/models/tabular_input.py index 388718fb9..d504cb8a4 100644 --- a/hed/models/tabular_input.py +++ b/hed/models/tabular_input.py @@ -42,3 +42,18 @@ def reset_column_mapper(self, sidecar=None): new_mapper = ColumnMapper(sidecar=sidecar, optional_tag_columns=[self.HED_COLUMN_NAME]) self.reset_mapper(new_mapper) + + def get_def_dict(self, hed_schema=None, extra_def_dicts=None): + """ Returns the definition dict for this sidecar. + + Parameters: + hed_schema(HedSchema): used to identify tags to find definitions + extra_def_dicts (list, DefinitionDict, or None): Extra dicts to add to the list. + + Returns: + DefinitionDict: A single definition dict representing all the data(and extra def dicts) + """ + if self._sidecar: + return self._sidecar.get_def_dict(hed_schema, extra_def_dicts) + else: + super().get_def_dict(hed_schema, extra_def_dicts) \ No newline at end of file diff --git a/hed/tools/remodeling/operations/summarize_definitions_op.py b/hed/tools/remodeling/operations/summarize_definitions_op.py index a49c5d5dd..c6f6f7001 100644 --- a/hed/tools/remodeling/operations/summarize_definitions_op.py +++ b/hed/tools/remodeling/operations/summarize_definitions_op.py @@ -1,10 +1,9 @@ """ Summarize the values in the columns of a tabular file. """ -from hed import DefinitionDict, TabularInput, Sidecar -from hed.models.df_util import process_def_expands -from hed.tools.analysis.analysis_util import assemble_hed +from hed import TabularInput from hed.tools.remodeling.operations.base_op import BaseOp from hed.tools.remodeling.operations.base_context import BaseContext +from hed.models.def_expand_gather import DefExpandGatherer class SummarizeDefinitionsOp(BaseOp): @@ -67,23 +66,17 @@ def do_op(self, dispatcher, df, name, sidecar=None): Updates the context. """ - - summary = dispatcher.context_dict.get(self.summary_name, None) - if not summary: - summary = DefinitionSummaryContext(self) - dispatcher.context_dict[self.summary_name] = summary + summary = dispatcher.context_dict.setdefault(self.summary_name, + DefinitionSummaryContext(self, dispatcher.hed_schema)) summary.update_context({'df': dispatcher.post_proc_data(df), 'name': name, 'sidecar': sidecar, 'schema': dispatcher.hed_schema}) return df class DefinitionSummaryContext(BaseContext): - - def __init__(self, sum_op): + def __init__(self, sum_op, hed_schema, known_defs=None): super().__init__(sum_op.SUMMARY_TYPE, sum_op.summary_name, sum_op.summary_filename) - self.defs = DefinitionDict() - self.unresolved = {} - self.errors = {} + self.def_gatherer = DefExpandGatherer(hed_schema, known_defs=known_defs) def update_context(self, new_context): """ Update the summary for a given tabular input file. @@ -96,15 +89,10 @@ def update_context(self, new_context): """ data_input = TabularInput(new_context['df'], sidecar=new_context['sidecar'], name=new_context['name']) - sidecar = Sidecar(new_context['sidecar']) - df, _ = assemble_hed(data_input, sidecar, new_context['schema'], - columns_included=None, expand_defs=True) - hed_strings = df['HED_assembled'] - self.defs, self.unresolved, errors = process_def_expands(hed_strings, new_context['schema'], - known_defs=self.defs, ambiguous_defs=self.unresolved) - self.errors.update(errors) - - def _get_details_dict(self, summary): + series, def_dict = data_input.series_a, data_input.get_def_dict(new_context['schema']) + self.def_gatherer.process_def_expands(series, def_dict) + + def _get_details_dict(self, def_gatherer): """ Return the summary-specific information in a dictionary. Parameters: @@ -114,7 +102,34 @@ def _get_details_dict(self, summary): dict: dictionary with the summary results. """ - return None + def build_summary_dict(items_dict, title, process_func, display_description=False): + summary_dict = {} + items = {} + for key, value in items_dict.items(): + if process_func: + value = process_func(value) + if "#" in str(value): + key = key + "/#" + if display_description: + description, value = DefinitionSummaryContext.remove_description(value) + items[key] = {"description": description, "contents": str(value)} + else: + if isinstance(value, list): + items[key] = [str(x) for x in value] + else: + items[key] = str(value) + summary_dict[title] = items + return summary_dict + + known_defs_summary = build_summary_dict(def_gatherer.def_dict, "Known Definitions", None, + display_description=True) + ambiguous_defs_summary = build_summary_dict(def_gatherer.ambiguous_defs, "Ambiguous Definitions", + def_gatherer.get_ambiguous_group) + errors_summary = build_summary_dict(def_gatherer.errors, "Errors", None) + + known_defs_summary.update(ambiguous_defs_summary) + known_defs_summary.update(errors_summary) + return known_defs_summary def _merge_all(self): """ Create an Object containing the definition summary. @@ -123,8 +138,7 @@ def _merge_all(self): Object - the overall summary object for definitions. """ - - return None + return self.def_gatherer def _get_result_string(self, name, result, indent=BaseContext.DISPLAY_INDENT): """ Return a formatted string with the summary for the indicated name. @@ -147,18 +161,33 @@ def _get_result_string(self, name, result, indent=BaseContext.DISPLAY_INDENT): return self._get_individual_string(result, indent=indent) @staticmethod - def _get_dataset_string(result, indent=BaseContext.DISPLAY_INDENT): - """ Return a string with the overall summary for all of the tabular files. - - Parameters: - result (dict): Dictionary of merged summary information. - indent (str): String of blanks used as the amount to indent for readability. - - Returns: - str: Formatted string suitable for saving in a file or printing. - - """ - return "" + def _get_dataset_string(summary_dict, indent=BaseContext.DISPLAY_INDENT): + def nested_dict_to_string(data, level=1): + result = [] + for key, value in data.items(): + if isinstance(value, dict): + result.append(f"{indent * level}{key}: {len(value)} items") + result.append(nested_dict_to_string(value, level + 1)) + elif isinstance(value, list): + result.append(f"{indent * level}{key}:") + for item in value: + result.append(f"{indent * (level + 1)}{item}") + else: + result.append(f"{indent * level}{key}: {value}") + return "\n".join(result) + + return nested_dict_to_string(summary_dict) + + def remove_description(def_entry): + def_group = def_entry.contents.copy() + description = "" + desc_tag = def_group.find_tags({"description"}, include_groups=False) + if desc_tag: + def_group.remove(desc_tag) + desc_tag = desc_tag[0] + description = desc_tag.extension + + return description, def_group @staticmethod def _get_individual_string(result, indent=BaseContext.DISPLAY_INDENT): diff --git a/hed/validator/def_validator.py b/hed/validator/def_validator.py index 2a362af18..f3f083933 100644 --- a/hed/validator/def_validator.py +++ b/hed/validator/def_validator.py @@ -9,7 +9,6 @@ class DefValidator(DefinitionDict): """ Handles validating Def/ and Def-expand/. """ - def __init__(self, def_dicts=None, hed_schema=None): """ Initialize for definitions in hed strings. @@ -75,7 +74,7 @@ def _validate_def_contents(self, def_tag, def_expand_group, tag_validator): tag=def_tag, actual_def=def_contents, found_def=def_expand_group) if def_entry.takes_value and tag_validator: - placeholder_tag = def_contents.find_placeholder_tag() + placeholder_tag = def_contents.get_first_group().find_placeholder_tag() error_code = ValidationErrors.DEF_INVALID if is_def_expand_tag: error_code = ValidationErrors.DEF_EXPAND_INVALID diff --git a/hed/validator/hed_validator.py b/hed/validator/hed_validator.py index ac9746fc4..9f692bdb3 100644 --- a/hed/validator/hed_validator.py +++ b/hed/validator/hed_validator.py @@ -5,7 +5,7 @@ """ -from hed.errors.error_types import ValidationErrors +from hed.errors.error_types import ValidationErrors, DefinitionErrors from hed.errors.error_reporter import ErrorHandler, check_for_any_errors from hed.models.hed_string import HedString @@ -18,11 +18,14 @@ class HedValidator: """ Top level validation of HED strings. """ - def __init__(self, hed_schema=None, def_dicts=None, run_full_onset_checks=True): + def __init__(self, hed_schema=None, def_dicts=None, run_full_onset_checks=True, definitions_allowed=False): """ Constructor for the HedValidator class. Parameters: hed_schema (HedSchema or HedSchemaGroup): HedSchema object to use for validation. + def_dicts(DefinitionDict or list or dict): the def dicts to use for validation + run_full_onset_checks(bool): If True, check for matching onset/offset tags + definitions_allowed(bool): If False, flag definitions found as errors """ super().__init__() self._tag_validator = None @@ -32,6 +35,7 @@ def __init__(self, hed_schema=None, def_dicts=None, run_full_onset_checks=True): self._def_validator = DefValidator(def_dicts, hed_schema) self._onset_validator = OnsetValidator(def_dict=self._def_validator, run_full_onset_checks=run_full_onset_checks) + self._definitions_allowed = definitions_allowed def validate(self, hed_string, allow_placeholders, error_handler=None): """ @@ -69,24 +73,14 @@ def run_basic_checks(self, hed_string, allow_placeholders): # e.g. checking units when a definition placeholder has units self._def_validator.construct_def_tags(hed_string) issues += self._validate_individual_tags_in_hed_string(hed_string, allow_placeholders=allow_placeholders) - # todo: maybe restore this. Issue is bad def-expand values are caught above, - # so the actual def-expand tag won't be checked if we bail early. - # if check_for_any_errors(issues): - # return issues issues += self._def_validator.validate_def_tags(hed_string, self._tag_validator) - if check_for_any_errors(issues): - return issues - issues += self._onset_validator.validate_onset_offset(hed_string) - if check_for_any_errors(issues): - return issues return issues def run_full_string_checks(self, hed_string): issues = [] issues += self._validate_tags_in_hed_string(hed_string) - if check_for_any_errors(issues): - return issues issues += self._validate_groups_in_hed_string(hed_string) + issues += self._onset_validator.validate_onset_offset(hed_string) return issues def _validate_groups_in_hed_string(self, hed_string_obj): @@ -172,6 +166,9 @@ def _validate_individual_tags_in_hed_string(self, hed_string_obj, allow_placehol for group in hed_string_obj.get_all_groups(): is_definition = group in all_definition_groups for hed_tag in group.tags(): + if not self._definitions_allowed and hed_tag.short_base_tag == DefTagNames.DEFINITION_ORG_KEY: + validation_issues += ErrorHandler.format_error(DefinitionErrors.BAD_DEFINITION_LOCATION, hed_tag) + # todo: unclear if this should be restored at some point # if hed_tag.expandable and not hed_tag.expanded: # for tag in hed_tag.expandable.get_all_tags(): # validation_issues += self._tag_validator. \ diff --git a/hed/validator/onset_validator.py b/hed/validator/onset_validator.py index af5db6bec..74efef12d 100644 --- a/hed/validator/onset_validator.py +++ b/hed/validator/onset_validator.py @@ -44,7 +44,7 @@ def validate_onset_offset(self, hed_string_obj): children = [child for child in found_group.children if def_group is not child and found_onset is not child] max_children = 1 - if found_onset.short_base_tag.lower() == DefTagNames.OFFSET_KEY: + if found_onset.short_base_tag == DefTagNames.OFFSET_ORG_KEY: max_children = 0 if len(children) > max_children: onset_issues += ErrorHandler.format_error(OnsetErrors.ONSET_WRONG_NUMBER_GROUPS, @@ -69,7 +69,7 @@ def _find_onset_tags(self, hed_string_obj): return hed_string_obj.find_top_level_tags(anchor_tags={DefTagNames.ONSET_KEY, DefTagNames.OFFSET_KEY}) def _handle_onset_or_offset(self, def_tag, onset_offset_tag): - is_onset = onset_offset_tag.short_base_tag.lower() == DefTagNames.ONSET_KEY + is_onset = onset_offset_tag.short_base_tag == DefTagNames.ONSET_ORG_KEY full_def_name = def_name = def_tag.extension placeholder = None found_slash = def_name.find("/") diff --git a/hed/validator/sidecar_validator.py b/hed/validator/sidecar_validator.py index 8c68808e8..450446371 100644 --- a/hed/validator/sidecar_validator.py +++ b/hed/validator/sidecar_validator.py @@ -1,10 +1,11 @@ import copy -from hed.errors import ErrorHandler, ErrorContext, SidecarErrors +from hed.errors import ErrorHandler, ErrorContext, SidecarErrors, DefinitionErrors from hed.models import ColumnType from hed import HedString from hed import Sidecar from hed.models.column_metadata import ColumnMetadata from hed.errors.error_reporter import sort_issues +from hed.models.model_constants import DefTagNames class SidecarValidator: @@ -40,26 +41,39 @@ def validate(self, sidecar, extra_def_dicts=None, name=None, error_handler=None) sidecar_def_dict = sidecar.get_def_dict(hed_schema=self._schema, extra_def_dicts=extra_def_dicts) hed_validator = HedValidator(self._schema, def_dicts=sidecar_def_dict, - run_full_onset_checks=False) + run_full_onset_checks=False, + definitions_allowed=True) issues += self.validate_structure(sidecar, error_handler=error_handler) issues += sidecar._extract_definition_issues issues += sidecar_def_dict.issues - # todo: Add the definition validation. + definition_checks = {} for hed_string, column_data, position in sidecar.hed_string_iter(error_handler): hed_string_obj = HedString(hed_string, hed_schema=self._schema, def_dict=sidecar_def_dict) error_handler.push_error_context(ErrorContext.HED_STRING, hed_string_obj) new_issues = hed_validator.run_basic_checks(hed_string_obj, allow_placeholders=True) - if not new_issues: - new_issues = hed_validator.run_full_string_checks(hed_string_obj) - if not new_issues: - new_issues = self._validate_pound_sign_count(hed_string_obj, column_type=column_data.column_type) + new_issues += hed_validator.run_full_string_checks(hed_string_obj) + + def_check_list = definition_checks.setdefault(column_data.column_name, []) + def_check_list.append(hed_string_obj.find_tags({DefTagNames.DEFINITION_KEY}, recursive=True, include_groups=0)) + # Might refine this later - for now just skip checking placeholder counts in definition columns. + if not def_check_list[-1]: + new_issues += self._validate_pound_sign_count(hed_string_obj, column_type=column_data.column_type) + error_handler.add_context_and_filter(new_issues) issues += new_issues error_handler.pop_error_context() + for col_name, has_def in definition_checks.items(): + error_handler.push_error_context(ErrorContext.SIDECAR_COLUMN_NAME, col_name) + def_check = set(bool(d) for d in has_def) + if len(def_check) != 1: + flat_def_list = [d for defs in has_def for d in defs] + for d in flat_def_list: + issues += error_handler.format_error_with_context(DefinitionErrors.BAD_DEFINITION_LOCATION, d) + error_handler.pop_error_context() error_handler.pop_error_context() issues = sort_issues(issues) return issues diff --git a/hed/validator/spreadsheet_validator.py b/hed/validator/spreadsheet_validator.py index 8b8aa9b1f..afa041327 100644 --- a/hed/validator/spreadsheet_validator.py +++ b/hed/validator/spreadsheet_validator.py @@ -111,7 +111,7 @@ def _validate_column_structure(self, base_input, error_handler): for row_number, value in enumerate(base_input.dataframe[column.column_name]): if value != "n/a" and value not in valid_keys: error_handler.push_error_context(ErrorContext.ROW, row_number) - issues += error_handler.format_error_with_context(ValidationErrors.HED_SIDECAR_KEY_MISSING, + issues += error_handler.format_error_with_context(ValidationErrors.SIDECAR_KEY_MISSING, invalid_key=value, category_keys=list(valid_keys)) error_handler.pop_error_context() @@ -133,7 +133,7 @@ def _validate_column_refs(df, error_handler): if match not in possible_column_references: error_handler.push_error_context(ErrorContext.ROW, row_number) error_handler.push_error_context(ErrorContext.COLUMN, column_name) - error_handler.push_error_context(ErrorContext.HED_STRING, df[column_name][row_number]) + error_handler.push_error_context(ErrorContext.HED_STRING, HedString(df[column_name][row_number])) issues += error_handler.format_error_with_context(ColumnErrors.INVALID_COLUMN_REF, match) diff --git a/hed/validator/tag_validator.py b/hed/validator/tag_validator.py index 783b035b6..8d11668d2 100644 --- a/hed/validator/tag_validator.py +++ b/hed/validator/tag_validator.py @@ -5,6 +5,7 @@ import re from hed.errors.error_reporter import ErrorHandler +from hed.models.model_constants import DefTagNames from hed.schema import HedKey from hed.errors.error_types import ValidationErrors from hed.validator import tag_validator_util @@ -167,7 +168,7 @@ def check_count_tag_group_parentheses(self, hed_string): number_open_parentheses = hed_string.count('(') number_closed_parentheses = hed_string.count(')') if number_open_parentheses != number_closed_parentheses: - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_PARENTHESES_MISMATCH, + validation_issues += ErrorHandler.format_error(ValidationErrors.PARENTHESES_MISMATCH, opening_parentheses_count=number_open_parentheses, closing_parentheses_count=number_closed_parentheses) return validation_issues @@ -192,7 +193,7 @@ def check_delimiter_issues_in_hed_string(self, hed_string): continue if TagValidator._character_is_delimiter(current_character): if current_tag.strip() == current_character: - issues += ErrorHandler.format_error(ValidationErrors.HED_TAG_EMPTY, source_string=hed_string, + issues += ErrorHandler.format_error(ValidationErrors.TAG_EMPTY, source_string=hed_string, char_index=i) current_tag = '' continue @@ -203,7 +204,7 @@ def check_delimiter_issues_in_hed_string(self, hed_string): else: issues += ErrorHandler.format_error(ValidationErrors.COMMA_MISSING, tag=current_tag) elif last_non_empty_valid_character == "," and current_character == self.CLOSING_GROUP_CHARACTER: - issues += ErrorHandler.format_error(ValidationErrors.HED_TAG_EMPTY, source_string=hed_string, + issues += ErrorHandler.format_error(ValidationErrors.TAG_EMPTY, source_string=hed_string, char_index=i) elif TagValidator._comma_is_missing_after_closing_parentheses(last_non_empty_valid_character, current_character): @@ -212,7 +213,7 @@ def check_delimiter_issues_in_hed_string(self, hed_string): last_non_empty_valid_character = current_character last_non_empty_valid_index = i if TagValidator._character_is_delimiter(last_non_empty_valid_character): - issues += ErrorHandler.format_error(ValidationErrors.HED_TAG_EMPTY, + issues += ErrorHandler.format_error(ValidationErrors.TAG_EMPTY, char_index=last_non_empty_valid_index, source_string=hed_string) return issues @@ -230,7 +231,7 @@ def check_tag_formatting(self, original_tag): """ validation_issues = [] for match in self.pattern_doubleslash.finditer(original_tag.org_tag): - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + validation_issues += ErrorHandler.format_error(ValidationErrors.NODE_NAME_EMPTY, tag=original_tag, index_in_tag=match.start(), index_in_tag_end=match.end()) @@ -271,9 +272,13 @@ def check_tag_exists_in_schema(self, original_tag): is_extension_tag = original_tag.is_extension_allowed_tag() if not is_extension_tag: - validation_issues += ErrorHandler.format_error(ValidationErrors.INVALID_EXTENSION, tag=original_tag) + actual_error = None + if "#" in original_tag.extension: + actual_error = ValidationErrors.PLACEHOLDER_INVALID + validation_issues += ErrorHandler.format_error(ValidationErrors.TAG_EXTENSION_INVALID, tag=original_tag, + actual_error=actual_error) else: - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_TAG_EXTENDED, tag=original_tag, + validation_issues += ErrorHandler.format_error(ValidationErrors.TAG_EXTENDED, tag=original_tag, index_in_tag=len(original_tag.org_base_tag), index_in_tag_end=None) return validation_issues @@ -292,24 +297,39 @@ def check_tag_unit_class_units_are_valid(self, original_tag, report_tag_as=None, if original_tag.is_unit_class_tag(): stripped_value, unit = original_tag.get_stripped_unit_value() if not unit: - if self._validate_value_class_portion(original_tag, stripped_value): - # only suggest a unit is missing if this is a valid number - if tag_validator_util.validate_numeric_value_class(stripped_value): - default_unit = original_tag.get_unit_class_default_unit() - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_UNITS_DEFAULT_USED, - tag=report_tag_as if report_tag_as else original_tag, - default_unit=default_unit, - actual_error=error_code) - else: + bad_units = " " in original_tag.extension + + # Todo: in theory this should separately validate the number and the units, for units + # that are prefixes like $. Right now those are marked as unit invalid AND value_invalid. + if bad_units: + stripped_value = stripped_value.split(" ")[0] + if original_tag.is_takes_value_tag() and\ + not self._validate_value_class_portion(original_tag, stripped_value): + validation_issues += ErrorHandler.format_error(ValidationErrors.VALUE_INVALID, + report_tag_as if report_tag_as else original_tag, + actual_error=error_code) + if error_code: + validation_issues += ErrorHandler.format_error(ValidationErrors.VALUE_INVALID, + report_tag_as if report_tag_as else original_tag) + + + if bad_units: tag_unit_class_units = original_tag.get_tag_unit_class_units() if tag_unit_class_units: - default_code = ValidationErrors.HED_UNITS_INVALID - if not error_code: - error_code = default_code - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_UNITS_INVALID, - actual_error=error_code, + validation_issues += ErrorHandler.format_error(ValidationErrors.UNITS_INVALID, tag=report_tag_as if report_tag_as else original_tag, units=tag_unit_class_units) + else: + default_unit = original_tag.get_unit_class_default_unit() + validation_issues += ErrorHandler.format_error(ValidationErrors.UNITS_MISSING, + tag=report_tag_as if report_tag_as else original_tag, + default_unit=default_unit) + + if error_code: + new_issue = validation_issues[0].copy() + new_issue['code'] = error_code + validation_issues += [new_issue] + return validation_issues def check_tag_value_class_valid(self, original_tag, report_tag_as=None, error_code=None): @@ -324,7 +344,7 @@ def check_tag_value_class_valid(self, original_tag, report_tag_as=None, error_co """ validation_issues = [] if not self._validate_value_class_portion(original_tag, original_tag.extension): - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_VALUE_INVALID, + validation_issues += ErrorHandler.format_error(ValidationErrors.VALUE_INVALID, report_tag_as if report_tag_as else original_tag, actual_error=error_code) @@ -341,7 +361,7 @@ def check_tag_requires_child(self, original_tag): """ validation_issues = [] if original_tag.has_attribute(HedKey.RequireChild): - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_TAG_REQUIRES_CHILD, + validation_issues += ErrorHandler.format_error(ValidationErrors.TAG_REQUIRES_CHILD, tag=original_tag) return validation_issues @@ -360,7 +380,7 @@ def check_tag_unit_class_units_exist(self, original_tag): tag_unit_values = original_tag.extension if tag_validator_util.validate_numeric_value_class(tag_unit_values): default_unit = original_tag.get_unit_class_default_unit() - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_UNITS_DEFAULT_USED, + validation_issues += ErrorHandler.format_error(ValidationErrors.UNITS_MISSING, tag=original_tag, default_unit=default_unit) return validation_issues @@ -394,7 +414,7 @@ def check_capitalization(self, original_tag): for tag_name in tag_names: correct_tag_name = tag_name.capitalize() if tag_name != correct_tag_name and not re.search(self.CAMEL_CASE_EXPRESSION, tag_name): - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_STYLE_WARNING, + validation_issues += ErrorHandler.format_error(ValidationErrors.STYLE_WARNING, tag=original_tag) break return validation_issues @@ -424,6 +444,16 @@ def check_tag_level_issue(self, original_tag_list, is_top_level, is_group): tag=tag_group_tag) for top_level_tag in top_level_tags: if not is_top_level: + actual_code = None + if top_level_tag.short_base_tag == DefTagNames.DEFINITION_ORG_KEY: + actual_code = ValidationErrors.DEFINITION_INVALID + elif top_level_tag.short_base_tag in {DefTagNames.ONSET_ORG_KEY, DefTagNames.OFFSET_ORG_KEY}: + actual_code = ValidationErrors.ONSET_OFFSET_ERROR + + if actual_code: + validation_issues += ErrorHandler.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, + tag=top_level_tag, + actual_error=actual_code) validation_issues += ErrorHandler.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=top_level_tag) @@ -448,7 +478,7 @@ def check_for_required_tags(self, tags): required_prefixes = self._hed_schema.get_tags_with_attribute(HedKey.Required) for required_prefix in required_prefixes: if not any(tag.long_tag.lower().startswith(required_prefix.lower()) for tag in tags): - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, + validation_issues += ErrorHandler.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix=required_prefix) return validation_issues @@ -469,7 +499,7 @@ def check_multiple_unique_tags_exist(self, tags): for unique_prefix in unique_prefixes: unique_tag_prefix_bool_mask = [x.long_tag.lower().startswith(unique_prefix.lower()) for x in tags] if sum(unique_tag_prefix_bool_mask) > 1: - validation_issues += ErrorHandler.format_error(ValidationErrors.HED_TAG_NOT_UNIQUE, + validation_issues += ErrorHandler.format_error(ValidationErrors.TAG_NOT_UNIQUE, tag_prefix=unique_prefix) return validation_issues @@ -506,7 +536,7 @@ def _report_invalid_character_error(self, hed_string, index): error_type = ValidationErrors.CHARACTER_INVALID character = hed_string[index] if character == "~": - error_type = ValidationErrors.HED_TILDES_UNSUPPORTED + error_type = ValidationErrors.TILDES_UNSUPPORTED return ErrorHandler.format_error(error_type, char_index=index, source_string=hed_string) @@ -568,7 +598,7 @@ def check_for_placeholder(self, original_tag, is_definition=False): tag=original_tag, index_in_tag=starting_index + i, index_in_tag_end=starting_index + i + 1, - actual_error=ValidationErrors.HED_VALUE_INVALID) + actual_error=ValidationErrors.PLACEHOLDER_INVALID) return validation_issues diff --git a/spec_tests/hed-specification b/spec_tests/hed-specification index 8772db30c..a71c92e73 160000 --- a/spec_tests/hed-specification +++ b/spec_tests/hed-specification @@ -1 +1 @@ -Subproject commit 8772db30cf7c63a4fc224aac9e7daf504f276a1b +Subproject commit a71c92e73af4f89eaf24ae8b760de428804ffbdf diff --git a/spec_tests/test_errors.py b/spec_tests/test_errors.py index c6e52dca1..0313b10e3 100644 --- a/spec_tests/test_errors.py +++ b/spec_tests/test_errors.py @@ -11,17 +11,44 @@ from hed.errors import ErrorHandler, get_printable_issue_string +# To be removed eventually once all errors are being verified. known_errors = [ 'SIDECAR_INVALID', 'CHARACTER_INVALID', 'COMMA_MISSING', "DEF_EXPAND_INVALID", "DEF_INVALID", - "DEFINITION_INVALID" + "DEFINITION_INVALID", + "NODE_NAME_EMPTY", + "ONSET_OFFSET_ERROR", + "PARENTHESES_MISMATCH", + "PLACEHOLDER_INVALID", + "REQUIRED_TAG_MISSING", + "SIDECAR_INVALID", + "SIDECAR_KEY_MISSING", + "STYLE_WARNING", + "TAG_EMPTY", + "TAG_EXPRESSION_REPEATED", + "TAG_EXTENDED", + "TAG_EXTENSION_INVALID", + "TAG_GROUP_ERROR", + "TAG_INVALID", + "TAG_NOT_UNIQUE", + "TAG_PREFIX_INVALID", + "TAG_REQUIRES_CHILD", + "TILDES_UNSUPPORTED", + "UNITS_INVALID", + "UNITS_MISSING", + "VALUE_INVALID" ] -skip_tests = ["VERSION_DEPRECATED", "CHARACTER_INVALID", "STYLE_WARNING"] - +skip_tests = { + "VERSION_DEPRECATED": "Not applicable", + "CHARACTER_INVALID": "Not finalized", + "STYLE_WARNING": "Bad tests", + "onset-offset-error-duplicated-onset-or-offset": "TBD how we implement this", + "tag-extension-invalid-bad-node-name": "Part of character invalid checking", +} class MyTestCase(unittest.TestCase): @classmethod @@ -41,13 +68,15 @@ def run_single_test(self, test_file): verify_code = False if error_code in known_errors: verify_code = True - # To be deprecated once we add this to all tests self._verify_code = verify_code if error_code in skip_tests: - print(f"Skipping {error_code} test") + print(f"Skipping {error_code} test because: {skip_tests[error_code]}") continue name = info.get('name', '') + if name in skip_tests: + print(f"Skipping {name} test because: {skip_tests[name]}") + continue description = info['description'] schema = info['schema'] check_for_warnings = info.get("warning", False) diff --git a/tests/data/sidecar_tests/both_types_events_with_defs.json b/tests/data/sidecar_tests/both_types_events_with_defs.json index 7047a1fdd..b41106a44 100644 --- a/tests/data/sidecar_tests/both_types_events_with_defs.json +++ b/tests/data/sidecar_tests/both_types_events_with_defs.json @@ -7,7 +7,7 @@ "stop": "A blue square is displayed to indicate stopping" }, "HED": { - "go": "Item/ItemTag1, (Definition/JsonFileDef/#, (Item/JsonDef1/#,Item/JsonDef1))", + "go": "Item/ItemTag1", "stop": "Item/ItemTag2" } }, @@ -20,7 +20,7 @@ "stim_file": { "LongName": "Stimulus file name", "Description": "Relative path of the stimulus image file", - "HED": "Age/#, (Definition/JsonFileDef2/#, (Item/JsonDef2/#,Item/JsonDef2)), (Definition/JsonFileDef3/#, (Item/JsonDef3/#))" + "HED": "Age/#, (Definition/JsonFileDef2/#, (Age/#,Item/JsonDef2)), (Definition/JsonFileDef3/#, (Age/#))" }, "takes_value_def": { "LongName": "Def with a takes value tag", @@ -31,5 +31,8 @@ "LongName": "Def with a value class", "Description": "Relative path of the stimulus image file", "HED": "Age/#, (Definition/ValueClassDef/#, (Acceleration/#))" - } + }, + "Defs": { + "HED": "(Definition/JsonFileDef/#, (Acceleration/#,Item/JsonDef1))" + } } \ No newline at end of file diff --git a/tests/data/sidecar_tests/both_types_events_without_definitions.json b/tests/data/sidecar_tests/both_types_events_without_definitions.json index fcf9dc90b..6129f7405 100644 --- a/tests/data/sidecar_tests/both_types_events_without_definitions.json +++ b/tests/data/sidecar_tests/both_types_events_without_definitions.json @@ -20,16 +20,16 @@ "stim_file": { "LongName": "Stimulus file name", "Description": "Relative path of the stimulus image file", - "HED": "Attribute/File/#, (Definition/JsonFileDef2/#, (Item/JsonDef2/#,Item/JsonDef2)), (Definition/JsonFileDef3/#, (Item/JsonDef3/#,InvalidTag))" + "HED": "Attribute/File/#" }, "takes_value_def": { "LongName": "Def with a takes value tag", "Description": "Relative path of the stimulus image file", - "HED": "Attribute/File/#, (Definition/TakesValueDef/#, (Age/#))" + "HED": "Attribute/File/#" }, "unit_class_def": { "LongName": "Def with a value class", "Description": "Relative path of the stimulus image file", - "HED": "Attribute/File/#, (Definition/ValueClassDef/#, (Acceleration/#))" + "HED": "Attribute/File/#" } } \ No newline at end of file diff --git a/tests/data/validator_tests/bids_events.json b/tests/data/validator_tests/bids_events.json index 7d840ebee..893e8038b 100644 --- a/tests/data/validator_tests/bids_events.json +++ b/tests/data/validator_tests/bids_events.json @@ -41,9 +41,9 @@ "3": "Stage 3. BCI was trained on data from stages 1 and 2." }, "HED": { - "1": "Description/BCI acts randomly, (Definition/Random-selection, (Condition-variable, (Random, Predict))), Def/Random-selection", - "2": "Description/BCI was trained on data from stage 1., (Definition/Trained-on-random, (Condition-variable)), Def/Trained-on-random", - "3": "Description/BCI was trained on data from stages 1 and 2., (Definition/Trained-on-all, (Condition-variable)), Def/Trained-on-all" + "1": "Description/BCI acts randomly.", + "2": "Description/BCI was trained on data from stage 1.", + "3": "Description/BCI was trained on data from stages 1 and 2." } }, "trial": { @@ -78,5 +78,12 @@ }, "n_repeated": { "Description": "Number of trials that had to be repeated until the present trial because of invalid participant behavior (within this stage)." + }, + "defs": { + "HED": { + "1": "Description/BCI acts randomly, (Definition/Random-selection, (Condition-variable, (Random, Predict))), Def/Random-selection", + "2": "Description/BCI was trained on data from stage 1., (Definition/Trained-on-random, (Condition-variable)), Def/Trained-on-random", + "3": "Description/BCI was trained on data from stages 1 and 2., (Definition/Trained-on-all, (Condition-variable)), Def/Trained-on-all" + } } } \ No newline at end of file diff --git a/tests/errors/test_error_reporter.py b/tests/errors/test_error_reporter.py index 40765f51b..6cff88a25 100644 --- a/tests/errors/test_error_reporter.py +++ b/tests/errors/test_error_reporter.py @@ -10,21 +10,21 @@ def setUpClass(cls): pass def test_push_error_context(self): - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) name = "DummyFileName.txt" self.error_handler.push_error_context(ErrorContext.FILE_NAME, name) - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(name in error_list[0][ErrorContext.FILE_NAME]) column_name = "DummyColumnName" self.error_handler.push_error_context(ErrorContext.SIDECAR_COLUMN_NAME, column_name) - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(column_name in error_list[0][ErrorContext.SIDECAR_COLUMN_NAME]) self.error_handler.reset_error_context() self.error_handler.push_error_context(ErrorContext.FILE_NAME, name) self.error_handler.push_error_context(ErrorContext.SIDECAR_COLUMN_NAME, column_name) self.error_handler.push_error_context(ErrorContext.COLUMN, column_name) - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(name in error_list[0][ErrorContext.FILE_NAME]) self.assertTrue(column_name in error_list[0][ErrorContext.SIDECAR_COLUMN_NAME]) self.assertTrue(column_name == error_list[0][ErrorContext.COLUMN]) @@ -32,24 +32,24 @@ def test_push_error_context(self): self.error_handler.reset_error_context() def test_pop_error_context(self): - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) name = "DummyFileName.txt" self.error_handler.push_error_context(ErrorContext.FILE_NAME, name) - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) self.assertTrue(name in error_list[0][ErrorContext.FILE_NAME]) self.error_handler.pop_error_context() - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) column_name = "DummyColumnName" self.error_handler.push_error_context(ErrorContext.SIDECAR_COLUMN_NAME, column_name) - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) self.error_handler.push_error_context(ErrorContext.FILE_NAME, name) self.error_handler.push_error_context(ErrorContext.SIDECAR_COLUMN_NAME, column_name) self.error_handler.push_error_context(ErrorContext.COLUMN, column_name) - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) self.assertTrue(name in error_list[0][ErrorContext.FILE_NAME]) self.assertTrue(column_name in error_list[0][ErrorContext.SIDECAR_COLUMN_NAME]) @@ -57,16 +57,16 @@ def test_pop_error_context(self): self.error_handler.pop_error_context() self.error_handler.pop_error_context() self.error_handler.pop_error_context() - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) self.assertTrue(ErrorContext.COLUMN not in error_list[0]) self.error_handler.pop_error_context() - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") self.assertTrue(len(error_list) == 1) self.error_handler.reset_error_context() def test_filter_issues_by_severity(self): - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") error_list += self.error_handler.format_error_with_context(SchemaWarnings.INVALID_CAPITALIZATION, "dummy", problem_char="#", char_index=0) self.assertTrue(len(error_list) == 2) @@ -76,7 +76,7 @@ def test_filter_issues_by_severity(self): def test_printable_issue_string(self): self.error_handler.push_error_context(ErrorContext.CUSTOM_TITLE, "Default Custom Title") - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") error_list += self.error_handler.format_error_with_context(SchemaWarnings.INVALID_CAPITALIZATION, "dummy", problem_char="#", char_index=0) @@ -96,7 +96,7 @@ def test_printable_issue_string_with_filenames(self): myfile = 'my_file.txt' self.error_handler.push_error_context(ErrorContext.CUSTOM_TITLE, "Default Custom Title") self.error_handler.push_error_context(ErrorContext.FILE_NAME, myfile) - error_list = self.error_handler.format_error_with_context(ValidationErrors.HED_TAG_NOT_UNIQUE, "") + error_list = self.error_handler.format_error_with_context(ValidationErrors.TAG_NOT_UNIQUE, "") error_list += self.error_handler.format_error_with_context(SchemaWarnings.INVALID_CAPITALIZATION, "dummy", problem_char="#", char_index=0) diff --git a/tests/models/test_base_input.py b/tests/models/test_base_input.py index 32615fb76..d8e0b4a62 100644 --- a/tests/models/test_base_input.py +++ b/tests/models/test_base_input.py @@ -48,12 +48,13 @@ def tearDownClass(cls): shutil.rmtree(cls.base_output_folder) def test_gathered_defs(self): + # todo: probably remove this test? # todo: add unit tests for definitions in tsv file defs = DefinitionDict.get_as_strings(self.tabular_file._sidecar.extract_definitions(hed_schema=self.hed_schema)) expected_defs = { - 'jsonfiledef': '(Item/JsonDef1,Item/JsonDef1/#)', - 'jsonfiledef2': '(Item/JsonDef2,Item/JsonDef2/#)', - 'jsonfiledef3': '(Item/JsonDef3/#)', + 'jsonfiledef': '(Acceleration/#,Item/JsonDef1)', + 'jsonfiledef2': '(Age/#,Item/JsonDef2)', + 'jsonfiledef3': '(Age/#)', 'takesvaluedef': '(Age/#)', 'valueclassdef': '(Acceleration/#)' } diff --git a/tests/models/test_definition_dict.py b/tests/models/test_definition_dict.py index ee03122aa..eb5490529 100644 --- a/tests/models/test_definition_dict.py +++ b/tests/models/test_definition_dict.py @@ -4,9 +4,10 @@ from hed.models.hed_string import HedString from hed import HedTag from hed import load_schema_version +from tests.validator.test_tag_validator_base import TestHedBase -class TestDefBase(unittest.TestCase): +class TestDefBase(TestHedBase): @classmethod def setUpClass(cls): cls.hed_schema = load_schema_version("8.0.0") @@ -16,7 +17,10 @@ def check_def_base(self, test_strings, expected_issues): def_dict = DefinitionDict() hed_string_obj = HedString(test_strings[test_key], self.hed_schema) test_issues = def_dict.check_for_definitions(hed_string_obj) - expected_issue = expected_issues[test_key] + expected_params = expected_issues[test_key] + expected_issue = self.format_errors_fully(ErrorHandler(), hed_string=hed_string_obj, + params=expected_params) + # print(test_key) # print(test_issues) # print(expected_issue) self.assertCountEqual(test_issues, expected_issue, HedString(test_strings[test_key])) @@ -31,7 +35,7 @@ class TestDefinitionDict(TestDefBase): basic_hed_string = "Item/BasicTestTag1,Item/BasicTestTag2" basic_hed_string_with_def = f"Item/BasicTestTag1,Item/BasicTestTag2,{label_def_string}" - placeholder_def_contents = "(Item/TestDef1/#,Item/TestDef2)" + placeholder_def_contents = "(Age/#,Event)" placeholder_def_string = f"(Definition/TestDefPlaceholder/#,{placeholder_def_contents})" def test_check_for_definitions(self): @@ -50,55 +54,61 @@ def test_check_for_definitions_placeholder(self): new_def_count = len(def_dict.defs) self.assertGreater(new_def_count, original_def_count) - placeholder_invalid_def_contents = "(Item/TestDef1/#,Item/TestDef2/#)" + placeholder_invalid_def_contents = "(Age/#,Item/TestDef2/#)" placeholder_invalid_def_string = f"(Definition/TestDefPlaceholder/#,{placeholder_invalid_def_contents})" def test_definitions(self): test_strings = { - 'noGroupTag': "(Definition/ValidDef1)", + 'noGroupTag': "(Definition/InvalidDef0)", 'placeholderNoGroupTag': "(Definition/InvalidDef1/#)", 'placeholderWrongSpot': "(Definition/InvalidDef1#)", 'twoDefTags': f"(Definition/ValidDef1,Definition/InvalidDef2,{self.def_contents_string})", 'twoGroupTags': f"(Definition/InvalidDef1,{self.def_contents_string},{self.def_contents_string2})", 'extraOtherTags': "(Definition/InvalidDef1, InvalidContents)", - 'duplicateDef': f"(Definition/Def1), (Definition/Def1, {self.def_contents_string})", - 'duplicateDef2': f"(Definition/Def1), (Definition/Def1/#, {self.placeholder_def_contents})", - 'defAlreadyTagInSchema': "(Definition/Item)", + 'duplicateDef': f"(Definition/Def1, {self.def_contents_string}), (Definition/Def1, {self.def_contents_string})", + 'duplicateDef2': f"(Definition/Def1, {self.def_contents_string}), (Definition/Def1/#, {self.placeholder_def_contents})", 'defTooManyPlaceholders': self.placeholder_invalid_def_string, - 'invalidPlaceholder': "(Definition/InvalidDef1/InvalidPlaceholder)", - 'invalidPlaceholderExtension': "(Definition/InvalidDef1/this-part-is-not-allowed/#)", + 'invalidPlaceholder': f"(Definition/InvalidDef1/InvalidPlaceholder, {self.def_contents_string})", + 'invalidPlaceholderExtension': f"(Definition/InvalidDef1/this-part-is-not-allowed/#, {self.def_contents_string})", 'defInGroup': "(Definition/ValidDefName, (Def/ImproperlyPlacedDef))", - 'defExpandInGroup': "(Definition/ValidDefName, (Def-expand/ImproperlyPlacedDef, (ImproperContents)))" + 'defExpandInGroup': "(Definition/ValidDefName, (Def-expand/ImproperlyPlacedDef, (ImproperContents)))", + 'doublePoundSignPlaceholder': f"(Definition/InvalidDef/##, {self.placeholder_def_contents})", + 'doublePoundSignDiffPlaceholder': f"(Definition/InvalidDef/#, (Age/##,Item/TestDef2))", + 'placeholdersWrongSpot': f"(Definition/InvalidDef/#, (Age/#,Item/TestDef2))", } expected_results = { - 'noGroupTag': [], - 'placeholderNoGroupTag': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, - "InvalidDef1", expected_count=1, tag_list=[]), - 'placeholderWrongSpot': ErrorHandler.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, - "InvalidDef1#"), - 'twoDefTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, + 'noGroupTag': self.format_error(DefinitionErrors.NO_DEFINITION_CONTENTS, + "InvalidDef0"), + 'placeholderNoGroupTag': self.format_error(DefinitionErrors.NO_DEFINITION_CONTENTS,"InvalidDef1/#"), + 'placeholderWrongSpot': self.format_error(DefinitionErrors.NO_DEFINITION_CONTENTS,"InvalidDef1#") + self.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, + tag=0, def_name="InvalidDef1#"), + 'twoDefTags': self.format_error(DefinitionErrors.WRONG_NUMBER_TAGS, "ValidDef1", ["Definition/InvalidDef2"]), - 'twoGroupTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, + 'twoGroupTags': self.format_error(DefinitionErrors.WRONG_NUMBER_GROUPS, "InvalidDef1", [self.def_contents_string, self.def_contents_string2]), - 'extraOtherTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, "InvalidDef1", - ['InvalidContents']), - 'duplicateDef': ErrorHandler.format_error(DefinitionErrors.DUPLICATE_DEFINITION, "Def1"), - 'duplicateDef2': ErrorHandler.format_error(DefinitionErrors.DUPLICATE_DEFINITION, "Def1"), - # This is not an error since re-used terms are checked elsewhere. - 'defAlreadyTagInSchema': [], - 'defTooManyPlaceholders': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, + 'extraOtherTags': self.format_error(DefinitionErrors.NO_DEFINITION_CONTENTS, "InvalidDef1") + + self.format_error(DefinitionErrors.WRONG_NUMBER_TAGS, "InvalidDef1", ['InvalidContents']), + 'duplicateDef': self.format_error(DefinitionErrors.DUPLICATE_DEFINITION, "Def1"), + 'duplicateDef2': self.format_error(DefinitionErrors.DUPLICATE_DEFINITION, "Def1"), + + 'defTooManyPlaceholders': self.format_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, "TestDefPlaceholder", expected_count=1, - tag_list=["Item/TestDef1/#", "Item/TestDef2/#"]), - 'invalidPlaceholderExtension': ErrorHandler.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, - "InvalidDef1/this-part-is-not-allowed"), - 'invalidPlaceholder': ErrorHandler.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, - "InvalidDef1/InvalidPlaceholder"), - 'defInGroup': ErrorHandler.format_error(DefinitionErrors.DEF_TAG_IN_DEFINITION, + tag_list=["Age/#", "Item/TestDef2/#"]), + 'invalidPlaceholderExtension': self.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, + tag=0, def_name="InvalidDef1/this-part-is-not-allowed"), + 'invalidPlaceholder': self.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, + tag=0, def_name="InvalidDef1/InvalidPlaceholder"), + 'defInGroup': self.format_error(DefinitionErrors.DEF_TAG_IN_DEFINITION, tag=HedTag("Def/ImproperlyPlacedDef"), def_name="ValidDefName"), - 'defExpandInGroup': ErrorHandler.format_error(DefinitionErrors.DEF_TAG_IN_DEFINITION, + 'defExpandInGroup': self.format_error(DefinitionErrors.DEF_TAG_IN_DEFINITION, tag=HedTag("Def-expand/ImproperlyPlacedDef"), - def_name="ValidDefName") + def_name="ValidDefName"), + 'doublePoundSignPlaceholder': self.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, + tag=0, def_name="InvalidDef/##"), + 'doublePoundSignDiffPlaceholder': self.format_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, + "InvalidDef", expected_count=1, tag_list=['Age/##']), + 'placeholdersWrongSpot': [] } self.check_def_base(test_strings, expected_results) @@ -107,17 +117,17 @@ def test_expand_defs(self): test_strings = { 1: "Def/TestDefPlaceholder/2471,Event", 2: "Event,(Def/TestDefPlaceholder/2471,Event)", - 3: "Def-expand/TestDefPlaceholder/2471,(Item/TestDef1/2471,Item/TestDef2),Event", + 3: "Def-expand/TestDefPlaceholder/2471,(Age/2471,Item/TestDef2),Event", } expected_results = { - 1: "(Def-expand/TestDefPlaceholder/2471,(Item/TestDef1/2471,Item/TestDef2)),Event", - 2: "Event,((Def-expand/TestDefPlaceholder/2471,(Item/TestDef1/2471,Item/TestDef2)),Event)", + 1: "(Def-expand/TestDefPlaceholder/2471,(Age/2471,Item/TestDef2)),Event", + 2: "Event,((Def-expand/TestDefPlaceholder/2471,(Age/2471,Item/TestDef2)),Event)", # this one shouldn't change as it doesn't have a parent - 3: "Def-expand/TestDefPlaceholder/2471,(Item/TestDef1/2471,Item/TestDef2),Event", + 3: "Def-expand/TestDefPlaceholder/2471,(Age/2471,Item/TestDef2),Event", } def_dict = DefinitionDict() - definition_string = "(Definition/TestDefPlaceholder/#,(Item/TestDef1/#,Item/TestDef2))" + definition_string = "(Definition/TestDefPlaceholder/#,(Age/#,Item/TestDef2))" def_dict.check_for_definitions(HedString(definition_string, hed_schema=self.hed_schema)) for key, test_string in test_strings.items(): hed_string = HedString(test_string, hed_schema=self.hed_schema) diff --git a/tests/models/test_definition_entry.py b/tests/models/test_definition_entry.py index 9ff70bc42..8407342f1 100644 --- a/tests/models/test_definition_entry.py +++ b/tests/models/test_definition_entry.py @@ -72,7 +72,7 @@ def test_get_definition(self): # new_def_count = len(def_dict.defs) # self.assertGreater(new_def_count, original_def_count) # - # placeholder_invalid_def_contents = "(Item/TestDef1/#,Item/TestDef2/#)" + # placeholder_invalid_def_contents = "(Age/#,Item/TestDef2/#)" # placeholder_invalid_def_string = f"(Definition/TestDefPlaceholder/#,{placeholder_invalid_def_contents})" # # def test_definitions(self): @@ -98,12 +98,12 @@ def test_get_definition(self): # "InvalidDef1", expected_count=1, tag_list=[]), # 'placeholderWrongSpot': ErrorHandler.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, # "InvalidDef1#"), - # 'twoDefTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, + # 'twoDefTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUPS, # "ValidDef1", ["Definition/InvalidDef2"]), - # 'twoGroupTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, + # 'twoGroupTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUPS, # "InvalidDef1", # [self.def_contents_string, self.def_contents_string2]), - # 'extraOtherTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUP_TAGS, "InvalidDef1", + # 'extraOtherTags': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_GROUPS, "InvalidDef1", # ['InvalidContents']), # 'duplicateDef': ErrorHandler.format_error(DefinitionErrors.DUPLICATE_DEFINITION, "Def1"), # 'duplicateDef2': ErrorHandler.format_error(DefinitionErrors.DUPLICATE_DEFINITION, "Def1"), @@ -111,7 +111,7 @@ def test_get_definition(self): # 'defAlreadyTagInSchema': [], # 'defTooManyPlaceholders': ErrorHandler.format_error(DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, # "TestDefPlaceholder", expected_count=1, - # tag_list=["Item/TestDef1/#", "Item/TestDef2/#"]), + # tag_list=["Age/#", "Item/TestDef2/#"]), # 'invalidPlaceholderExtension': ErrorHandler.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, # "InvalidDef1/this-part-is-not-allowed"), # 'invalidPlaceholder': ErrorHandler.format_error(DefinitionErrors.INVALID_DEFINITION_EXTENSION, diff --git a/tests/models/test_df_util.py b/tests/models/test_df_util.py index ff58e35b2..c88446956 100644 --- a/tests/models/test_df_util.py +++ b/tests/models/test_df_util.py @@ -12,53 +12,53 @@ def setUp(self): self.schema = load_schema_version() def test_shrink_defs_normal(self): - df = pd.DataFrame({"column1": ["(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),Event/SomeEvent"]}) + df = pd.DataFrame({"column1": ["(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),Event/SomeEvent"]}) expected_df = pd.DataFrame({"column1": ["Def/TestDefNormal,Event/SomeEvent"]}) result = shrink_defs(df, self.schema, ['column1']) pd.testing.assert_frame_equal(result, expected_df) def test_shrink_defs_placeholder(self): - df = pd.DataFrame({"column1": ["(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Item/SomeItem"]}) + df = pd.DataFrame({"column1": ["(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Item/SomeItem"]}) expected_df = pd.DataFrame({"column1": ["Def/TestDefPlaceholder/123,Item/SomeItem"]}) result = shrink_defs(df, self.schema, ['column1']) pd.testing.assert_frame_equal(result, expected_df) def test_shrink_defs_no_matching_tags(self): - df = pd.DataFrame({"column1": ["(Event/SomeEvent, Item/SomeItem,Age/25)"]}) - expected_df = pd.DataFrame({"column1": ["(Event/SomeEvent, Item/SomeItem,Age/25)"]}) + df = pd.DataFrame({"column1": ["(Event/SomeEvent, Item/SomeItem,Acceleration/25)"]}) + expected_df = pd.DataFrame({"column1": ["(Event/SomeEvent, Item/SomeItem,Acceleration/25)"]}) result = shrink_defs(df, self.schema, ['column1']) pd.testing.assert_frame_equal(result, expected_df) def test_shrink_defs_multiple_columns(self): - df = pd.DataFrame({"column1": ["(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),Event/SomeEvent"], - "column2": ["(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Item/SomeItem"]}) + df = pd.DataFrame({"column1": ["(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),Event/SomeEvent"], + "column2": ["(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Item/SomeItem"]}) expected_df = pd.DataFrame({"column1": ["Def/TestDefNormal,Event/SomeEvent"], "column2": ["Def/TestDefPlaceholder/123,Item/SomeItem"]}) result = shrink_defs(df, self.schema, ['column1', 'column2']) pd.testing.assert_frame_equal(result, expected_df) def test_shrink_defs_multiple_defs_same_line(self): - df = pd.DataFrame({"column1": ["(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Age/30"]}) - expected_df = pd.DataFrame({"column1": ["Def/TestDefNormal,Def/TestDefPlaceholder/123,Age/30"]}) + df = pd.DataFrame({"column1": ["(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Acceleration/30"]}) + expected_df = pd.DataFrame({"column1": ["Def/TestDefNormal,Def/TestDefPlaceholder/123,Acceleration/30"]}) result = shrink_defs(df, self.schema, ['column1']) pd.testing.assert_frame_equal(result, expected_df) def test_shrink_defs_mixed_tags(self): df = pd.DataFrame({"column1": [ - "(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),Event/SomeEvent,(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Item/SomeItem,Age/25"]}) + "(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),Event/SomeEvent,(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Item/SomeItem,Acceleration/25"]}) expected_df = pd.DataFrame( - {"column1": ["Def/TestDefNormal,Event/SomeEvent,Def/TestDefPlaceholder/123,Item/SomeItem,Age/25"]}) + {"column1": ["Def/TestDefNormal,Event/SomeEvent,Def/TestDefPlaceholder/123,Item/SomeItem,Acceleration/25"]}) result = shrink_defs(df, self.schema, ['column1']) pd.testing.assert_frame_equal(result, expected_df) def test_shrink_defs_series_normal(self): - series = pd.Series(["(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),Event/SomeEvent"]) + series = pd.Series(["(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),Event/SomeEvent"]) expected_series = pd.Series(["Def/TestDefNormal,Event/SomeEvent"]) result = shrink_defs(series, self.schema, None) pd.testing.assert_series_equal(result, expected_series) def test_shrink_defs_series_placeholder(self): - series = pd.Series(["(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Item/SomeItem"]) + series = pd.Series(["(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Item/SomeItem"]) expected_series = pd.Series(["Def/TestDefPlaceholder/123,Item/SomeItem"]) result = shrink_defs(series, self.schema, None) pd.testing.assert_series_equal(result, expected_series) @@ -67,27 +67,27 @@ def test_shrink_defs_series_placeholder(self): class TestExpandDefs(unittest.TestCase): def setUp(self): self.schema = load_schema_version() - self.def_dict = DefinitionDict(["(Definition/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2))", - "(Definition/TestDefPlaceholder/#,(Action/TestDef1/#,Action/TestDef2))"], + self.def_dict = DefinitionDict(["(Definition/TestDefNormal,(Acceleration/2471,Action/TestDef2))", + "(Definition/TestDefPlaceholder/#,(Acceleration/#,Action/TestDef2))"], hed_schema=self.schema) def test_expand_defs_normal(self): df = pd.DataFrame({"column1": ["Def/TestDefNormal,Event/SomeEvent"]}) expected_df = pd.DataFrame( - {"column1": ["(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),Event/SomeEvent"]}) + {"column1": ["(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),Event/SomeEvent"]}) result = expand_defs(df, self.schema, self.def_dict, ['column1']) pd.testing.assert_frame_equal(result, expected_df) def test_expand_defs_placeholder(self): df = pd.DataFrame({"column1": ["Def/TestDefPlaceholder/123,Item/SomeItem"]}) expected_df = pd.DataFrame({"column1": [ - "(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Item/SomeItem"]}) + "(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Item/SomeItem"]}) result = expand_defs(df, self.schema, self.def_dict, ['column1']) pd.testing.assert_frame_equal(result, expected_df) def test_expand_defs_no_matching_tags(self): - df = pd.DataFrame({"column1": ["(Event/SomeEvent,Item/SomeItem,Age/25)"]}) - expected_df = pd.DataFrame({"column1": ["(Event/SomeEvent,Item/SomeItem,Age/25)"]}) + df = pd.DataFrame({"column1": ["(Event/SomeEvent,Item/SomeItem,Acceleration/25)"]}) + expected_df = pd.DataFrame({"column1": ["(Event/SomeEvent,Item/SomeItem,Acceleration/25)"]}) result = expand_defs(df, self.schema, self.def_dict, ['column1']) pd.testing.assert_frame_equal(result, expected_df) @@ -95,21 +95,21 @@ def test_expand_defs_multiple_columns(self): df = pd.DataFrame({"column1": ["Def/TestDefNormal,Event/SomeEvent"], "column2": ["Def/TestDefPlaceholder/123,Item/SomeItem"]}) expected_df = pd.DataFrame( - {"column1": ["(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),Event/SomeEvent"], + {"column1": ["(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),Event/SomeEvent"], "column2": [ - "(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Item/SomeItem"]}) + "(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Item/SomeItem"]}) result = expand_defs(df, self.schema, self.def_dict, ['column1', 'column2']) pd.testing.assert_frame_equal(result, expected_df) def test_expand_defs_series_normal(self): series = pd.Series(["Def/TestDefNormal,Event/SomeEvent"]) - expected_series = pd.Series(["(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2)),Event/SomeEvent"]) + expected_series = pd.Series(["(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2)),Event/SomeEvent"]) result = expand_defs(series, self.schema, self.def_dict, None) pd.testing.assert_series_equal(result, expected_series) def test_expand_defs_series_placeholder(self): series = pd.Series(["Def/TestDefPlaceholder/123,Item/SomeItem"]) - expected_series = pd.Series(["(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2)),Item/SomeItem"]) + expected_series = pd.Series(["(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2)),Item/SomeItem"]) result = expand_defs(series, self.schema, self.def_dict, None) pd.testing.assert_series_equal(result, expected_series) @@ -157,8 +157,8 @@ def test_convert_to_form_multiple_tags_long(self): def test_basic_expand_detection(self): # all simple cases with no duplicates test_strings = [ - "(Def-expand/A1/1, (Action/1, Age/5, Item-count/3))", - "(Def-expand/A1/2, (Action/2, Age/5, Item-count/3))", + "(Def-expand/A1/1, (Action/1, Acceleration/5, Item-count/3))", + "(Def-expand/A1/2, (Action/2, Acceleration/5, Item-count/3))", "(Def-expand/B2/3, (Action/3, Collection/animals, Alert))", "(Def-expand/B2/4, (Action/4, Collection/animals, Alert))", "(Def-expand/C3/5, (Action/5, Joyful, Event))", @@ -170,20 +170,20 @@ def test_mixed_detection(self): # Cases where you can only retroactively identify the first def-expand test_strings = [ # Basic example first just to verify - "(Def-expand/A1/1, (Action/1, Age/5, Item-count/2))", - "(Def-expand/A1/2, (Action/2, Age/5, Item-count/2))", + "(Def-expand/A1/1, (Action/1, Acceleration/5, Item-count/2))", + "(Def-expand/A1/2, (Action/2, Acceleration/5, Item-count/2))", # Out of order ambiguous - "(Def-expand/B2/3, (Action/3, Collection/animals, Age/3))", - "(Def-expand/B2/4, (Action/4, Collection/animals, Age/3))", + "(Def-expand/B2/3, (Action/3, Collection/animals, Acceleration/3))", + "(Def-expand/B2/4, (Action/4, Collection/animals, Acceleration/3))", # Multiple tags - "(Def-expand/C3/5, (Action/5, Age/5, Item-count/5))", - "(Def-expand/C3/6, (Action/6, Age/5, Item-count/5))", + "(Def-expand/C3/5, (Action/5, Acceleration/5, Item-count/5))", + "(Def-expand/C3/6, (Action/6, Acceleration/5, Item-count/5))", # Multiple tags2 - "(Def-expand/D4/7, (Action/7, Age/7, Item-count/8))", - "(Def-expand/D4/8, (Action/8, Age/7, Item-count/8))" + "(Def-expand/D4/7, (Action/7, Acceleration/7, Item-count/8))", + "(Def-expand/D4/8, (Action/8, Acceleration/7, Item-count/8))" # Multiple tags3 - "(Def-expand/D5/7, (Action/7, Age/7, Item-count/8, Event))", - "(Def-expand/D5/8, (Action/8, Age/7, Item-count/8, Event))" + "(Def-expand/D5/7, (Action/7, Acceleration/7, Item-count/8, Event))", + "(Def-expand/D5/8, (Action/8, Acceleration/7, Item-count/8, Event))" ] def_dict, ambiguous_defs, _ = process_def_expands(test_strings, self.schema) self.assertEqual(len(def_dict), 5) @@ -191,11 +191,11 @@ def test_mixed_detection(self): def test_ambiguous_defs(self): # Cases that can't be identified test_strings = [ - "(Def-expand/A1/2, (Action/2, Age/5, Item-count/2))", - "(Def-expand/B2/3, (Action/3, Collection/animals, Age/3))", - "(Def-expand/C3/5, (Action/5, Age/5, Item-count/5))", - "(Def-expand/D4/7, (Action/7, Age/7, Item-count/8))", - "(Def-expand/D5/7, (Action/7, Age/7, Item-count/8, Event))", + "(Def-expand/A1/2, (Action/2, Acceleration/5, Item-count/2))", + "(Def-expand/B2/3, (Action/3, Collection/animals, Acceleration/3))", + "(Def-expand/C3/5, (Action/5, Acceleration/5, Item-count/5))", + "(Def-expand/D4/7, (Action/7, Acceleration/7, Item-count/8))", + "(Def-expand/D5/7, (Action/7, Acceleration/7, Item-count/8, Event))", ] _, ambiguous_defs, _ = process_def_expands(test_strings, self.schema) self.assertEqual(len(ambiguous_defs), 5) @@ -224,25 +224,46 @@ def test_errors(self): _, _, errors = process_def_expands(test_strings, self.schema) self.assertEqual(len(errors), 1) + def test_errors_ambiguous(self): + # Verify we recognize errors when we had a def that can't be resolved. + test_strings = [ + "(Def-expand/A1/1, (Action/1, Age/5, Item-count/1))", + "(Def-expand/A1/2, (Action/2, Age/5, Item-count/3))", + "(Def-expand/A1/3, (Action/3, Age/5, Item-count/3))", + ] + known, ambiguous, errors = process_def_expands(test_strings, self.schema) + self.assertEqual(len(errors), 1) + self.assertEqual(len(errors["a1"]), 3) + + def test_errors_unresolved(self): + # Verify we recognize errors when we had a def that can't be resolved. + test_strings = [ + "(Def-expand/A1/1, (Action/1, Age/5, Item-count/1))", + "(Def-expand/A1/2, (Action/2, Age/5, Item-count/3))", + ] + known, ambiguous, errors = process_def_expands(test_strings, self.schema) + self.assertEqual(len(errors), 1) + self.assertEqual(len(errors["a1"]), 2) + def test_def_expand_detection(self): test_strings = [ - "(Def-expand/A1/1, (Action/1, Age/5, Item-Count/2))", - "(Def-expand/A1/2, (Action/2, Age/5, Item-Count/2))", + "(Def-expand/A1/1, (Action/1, Acceleration/5, Item-Count/2))", + "(Def-expand/A1/2, (Action/2, Acceleration/5, Item-Count/2))", "(Def-expand/B2/3, (Action/3, Collection/animals, Alert))", "(Def-expand/B2/4, (Action/4, Collection/animals, Alert))", "(Def-expand/C3/5, (Action/5, Joyful, Event))", "(Def-expand/C3/6, (Action/6, Joyful, Event))", - "((Def-expand/A1/7, (Action/7, Age/5, Item-Count/2)), Event, Age/10)", - "((Def-expand/A1/8, (Action/8, Age/5, Item-Count/2)), Collection/toys, Item-Count/5)", + "((Def-expand/A1/7, (Action/7, Acceleration/5, Item-Count/2)), Event, Acceleration/10)", + "((Def-expand/A1/8, (Action/8, Acceleration/5, Item-Count/2)), Collection/toys, Item-Count/5)", "((Def-expand/B2/9, (Action/9, Collection/animals, Alert)), Event, Collection/plants)", "((Def-expand/B2/10, (Action/10, Collection/animals, Alert)), Joyful, Item-Count/3)", - "((Def-expand/C3/11, (Action/11, Joyful, Event)), Collection/vehicles, Age/20)", + "((Def-expand/C3/11, (Action/11, Joyful, Event)), Collection/vehicles, Acceleration/20)", "((Def-expand/C3/12, (Action/12, Joyful, Event)), Alert, Item-Count/8)", - "((Def-expand/A1/13, (Action/13, Age/5, Item-Count/2)), (Def-expand/B2/13, (Action/13, Collection/animals, Alert)), Event)", - "((Def-expand/A1/14, (Action/14, Age/5, Item-Count/2)), Joyful, (Def-expand/C3/14, (Action/14, Joyful, Event)))", - "(Def-expand/B2/15, (Action/15, Collection/animals, Alert)), (Def-expand/C3/15, (Action/15, Joyful, Event)), Age/30", - "((Def-expand/A1/16, (Action/16, Age/5, Item-Count/2)), (Def-expand/B2/16, (Action/16, Collection/animals, Alert)), Collection/food)", - "(Def-expand/C3/17, (Action/17, Joyful, Event)), (Def-expand/A1/17, (Action/17, Age/5, Item-Count/2)), Item-Count/6", + "((Def-expand/A1/13, (Action/13, Acceleration/5, Item-Count/2)), (Def-expand/B2/13, (Action/13, Collection/animals, Alert)), Event)", + "((Def-expand/A1/14, (Action/14, Acceleration/5, Item-Count/2)), Joyful, (Def-expand/C3/14, (Action/14, Joyful, Event)))", + "(Def-expand/B2/15, (Action/15, Collection/animals, Alert)), (Def-expand/C3/15, (Action/15, Joyful, Event)), Acceleration/30", + "((Def-expand/A1/16, (Action/16, Acceleration/5, Item-Count/2)), (Def-expand/B2/16, (Action/16, Collection/animals, Alert)), Collection/food)", + "(Def-expand/C3/17, (Action/17, Joyful, Event)), (Def-expand/A1/17, (Action/17, Acceleration/5, Item-Count/2)), Item-Count/6", "((Def-expand/B2/18, (Action/18, Collection/animals, Alert)), (Def-expand/C3/18, (Action/18, Joyful, Event)), Alert)", "(Def-expand/D1/Apple, (Task/Apple, Collection/cars, Attribute/color))", "(Def-expand/D1/Banana, (Task/Banana, Collection/cars, Attribute/color))", @@ -250,7 +271,7 @@ def test_def_expand_detection(self): "(Def-expand/E2/Dog, (Collection/Dog, Collection/plants, Attribute/type))", "((Def-expand/D1/Elephant, (Task/Elephant, Collection/cars, Attribute/color)), (Def-expand/E2/Fox, (Collection/Fox, Collection/plants, Attribute/type)), Event)", "((Def-expand/D1/Giraffe, (Task/Giraffe, Collection/cars, Attribute/color)), Joyful, (Def-expand/E2/Horse, (Collection/Horse, Collection/plants, Attribute/type)))", - "(Def-expand/D1/Iguana, (Task/Iguana, Collection/cars, Attribute/color)), (Def-expand/E2/Jaguar, (Collection/Jaguar, Collection/plants, Attribute/type)), Age/30", + "(Def-expand/D1/Iguana, (Task/Iguana, Collection/cars, Attribute/color)), (Def-expand/E2/Jaguar, (Collection/Jaguar, Collection/plants, Attribute/type)), Acceleration/30", "(Def-expand/F1/Lion, (Task/Lion, Collection/boats, Attribute/length))", "(Def-expand/F1/Monkey, (Task/Monkey, Collection/boats, Attribute/length))", "(Def-expand/G2/Nest, (Collection/Nest, Collection/instruments, Attribute/material))", diff --git a/tests/models/test_sidecar.py b/tests/models/test_sidecar.py index 182a15a3b..caec94043 100644 --- a/tests/models/test_sidecar.py +++ b/tests/models/test_sidecar.py @@ -82,13 +82,13 @@ def test__iter__(self): def test_validate_column_group(self): validation_issues = self.errors_sidecar.validate(self.hed_schema) - self.assertEqual(len(validation_issues), 22) + self.assertEqual(len(validation_issues), 23) validation_issues2 = self.errors_sidecar_minor.validate(self.hed_schema) - self.assertEqual(len(validation_issues2), 18) + self.assertEqual(len(validation_issues2), 19) validation_issues = self.json_without_definitions_sidecar.validate(self.hed_schema) - self.assertEqual(len(validation_issues), 8) + self.assertEqual(len(validation_issues), 7) hed_string = HedString("(Definition/JsonFileDef/#, (Item/JsonDef1/#,Item/JsonDef1))", self.hed_schema) extra_def_dict = DefinitionDict() diff --git a/tests/tools/remodeling/operations/test_summarize_definitions_op.py b/tests/tools/remodeling/operations/test_summarize_definitions_op.py index b5100a652..a55f61c6e 100644 --- a/tests/tools/remodeling/operations/test_summarize_definitions_op.py +++ b/tests/tools/remodeling/operations/test_summarize_definitions_op.py @@ -2,7 +2,6 @@ import os import unittest import pandas as pd -from hed.models.df_util import get_assembled from hed.tools.remodeling.dispatcher import Dispatcher from hed.tools.remodeling.operations.summarize_definitions_op import SummarizeDefinitionsOp, DefinitionSummaryContext @@ -27,8 +26,6 @@ def tearDownClass(cls): def test_constructor(self): parms = json.loads(self.json_parms) - sum_op1 = SummarizeDefinitionsOp(parms) - self.assertIsInstance(sum_op1, SummarizeDefinitionsOp, "constructor creates an object of the correct type") parms["expand_context"] = "" with self.assertRaises(KeyError) as context: SummarizeDefinitionsOp(parms) @@ -43,18 +40,45 @@ def test_do_op(self): dispatch = Dispatcher([], data_root=None, backup_name=None, hed_versions=['8.1.0']) parms = json.loads(self.json_parms) sum_op = SummarizeDefinitionsOp(parms) - self.assertIsInstance(sum_op, SummarizeDefinitionsOp, "constructor creates an object of the correct type") df = pd.read_csv(self.data_path, delimiter='\t', header=0, keep_default_na=False, na_values=",null") df_new = sum_op.do_op(dispatch, dispatch.prep_data(df), 'subj2_run1', sidecar=self.json_path) - self.assertEqual(200, len(df_new), "summarize_hed_type_op dataframe length is correct") - self.assertEqual(10, len(df_new.columns), "summarize_hed_type_op has correct number of columns") + self.assertEqual(200, len(df_new), " dataframe length is correct") + self.assertEqual(10, len(df_new.columns), " has correct number of columns") self.assertIn(sum_op.summary_name, dispatch.context_dict) self.assertIsInstance(dispatch.context_dict[sum_op.summary_name], DefinitionSummaryContext) - # x = dispatch.context_dict[sum_op.summary_name].summary_dict['subj2_run1'] - # self.assertEqual(len(dispatch.context_dict[sum_op.summary_name].summary_dict['subj2_run1'].tag_dict), 47) - # df_new = sum_op.do_op(dispatch, dispatch.prep_data(df), 'subj2_run2', sidecar=self.json_path) - # self.assertEqual(len(dispatch.context_dict[sum_op.summary_name].summary_dict['subj2_run2'].tag_dict), 47) + def test_summary(self): + dispatch = Dispatcher([], data_root=None, backup_name=None, hed_versions=['8.1.0']) + parms = json.loads(self.json_parms) + sum_op = SummarizeDefinitionsOp(parms) + df = pd.read_csv(self.data_path, delimiter='\t', header=0, keep_default_na=False, na_values=",null") + df_new = sum_op.do_op(dispatch, dispatch.prep_data(df), 'subj2_run1', sidecar=self.json_path) + self.assertEqual(200, len(df_new), " dataframe length is correct") + self.assertEqual(10, len(df_new.columns), " has correct number of columns") + self.assertIn(sum_op.summary_name, dispatch.context_dict) + self.assertIsInstance(dispatch.context_dict[sum_op.summary_name], DefinitionSummaryContext) + # print(str(dispatch.context_dict[sum_op.summary_name].get_text_summary()['Dataset'])) + + def test_summary_errors(self): + dispatch = Dispatcher([], data_root=None, backup_name=None, hed_versions=['8.1.0']) + parms = json.loads(self.json_parms) + sum_op = SummarizeDefinitionsOp(parms) + df = pd.DataFrame({"HED": [ + "(Def-expand/A1/1, (Action/1, Acceleration/5, Item-count/2))", + "(Def-expand/B2/3, (Action/3, Collection/animals, Acceleration/3))", + "(Def-expand/C3/5, (Action/5, Acceleration/5, Item-count/5))", + "(Def-expand/D4/7, (Action/7, Acceleration/7, Item-count/8))", + "(Def-expand/D5/7, (Action/7, Acceleration/7, Item-count/8, Event))", + "(Def-expand/A1/2, (Action/2, Age/5, Item-count/2))", + "(Def-expand/A1/3, (Action/3, Age/4, Item-count/3))", + + # This could be identified, but fails due to the above raising errors + "(Def-expand/A1/4, (Action/4, Age/5, Item-count/2))", + ]}) + df_new = sum_op.do_op(dispatch, dispatch.prep_data(df), 'subj2_run1', sidecar=self.json_path) + self.assertIn(sum_op.summary_name, dispatch.context_dict) + self.assertIsInstance(dispatch.context_dict[sum_op.summary_name], DefinitionSummaryContext) + #print(str(dispatch.context_dict[sum_op.summary_name].get_text_summary()['Dataset'])) if __name__ == '__main__': unittest.main() diff --git a/tests/tools/remodeling/operations/test_summarize_hed_validation_op.py b/tests/tools/remodeling/operations/test_summarize_hed_validation_op.py index 0136c205e..451528428 100644 --- a/tests/tools/remodeling/operations/test_summarize_hed_validation_op.py +++ b/tests/tools/remodeling/operations/test_summarize_hed_validation_op.py @@ -63,7 +63,7 @@ def test_do_op(self): sum_op.do_op(dispatch, dispatch.prep_data(df), 'subj2_run3', sidecar=self.bad_json_path) self.assertEqual(len(dispatch.context_dict[sum_op.summary_name].summary_dict), 3) run3 = dispatch.context_dict[sum_op.summary_name].summary_dict['subj2_run3'] - self.assertEqual(run3["total_sidecar_issues"], 2) + self.assertEqual(run3["total_sidecar_issues"], 4) def test_get_summary_details(self): dispatch = Dispatcher([], data_root=None, backup_name=None, hed_versions=['8.1.0']) diff --git a/tests/validator/test_def_validator.py b/tests/validator/test_def_validator.py index 6bef321a7..7464e985d 100644 --- a/tests/validator/test_def_validator.py +++ b/tests/validator/test_def_validator.py @@ -19,7 +19,7 @@ def setUpClass(cls): cls.basic_definition_string = f"(Definition/TestDef,{cls.def_contents_string})" cls.basic_definition_string_no_paren = f"Definition/TestDef,{cls.def_contents_string}" - cls.placeholder_definition_contents = "(Item/TestDef1/#,Item/TestDef2)" + cls.placeholder_definition_contents = "(Acceleration/#,Item/TestDef2)" cls.placeholder_definition_string = f"(Definition/TestDefPlaceholder/#,{cls.placeholder_definition_contents})" cls.placeholder_definition_string_no_paren = \ f"Definition/TestDefPlaceholder/#,{cls.placeholder_definition_contents}" @@ -34,7 +34,7 @@ def setUpClass(cls): cls.basic_hed_string_with_def_first_paren = f"({cls.label_def_string},{cls.basic_hed_string})" cls.placeholder_label_def_string = "Def/TestDefPlaceholder/2471" - cls.placeholder_expanded_def_string = "(Def-expand/TestDefPlaceholder/2471,(Item/TestDef1/2471,Item/TestDef2))" + cls.placeholder_expanded_def_string = "(Def-expand/TestDefPlaceholder/2471,(Acceleration/2471,Item/TestDef2))" cls.placeholder_hed_string_with_def = f"{cls.basic_hed_string},{cls.placeholder_label_def_string}" cls.placeholder_hed_string_with_def_first = f"{cls.placeholder_label_def_string},{cls.basic_hed_string}" @@ -80,27 +80,11 @@ def test_bad_def_expand(self): def_issues = def_validator.validate_def_tags(valid_placeholder) self.assertFalse(def_issues) - invalid_placeholder = HedString("(Def-expand/TestDefPlaceholder/2471,(Item/TestDef1/21,Item/TestDef2))", self.hed_schema) + invalid_placeholder = HedString("(Def-expand/TestDefPlaceholder/2471,(Acceleration/21,Item/TestDef2))", self.hed_schema) def_issues = def_validator.validate_def_tags(invalid_placeholder) self.assertTrue(bool(def_issues)) - def test_def_no_content(self): - - def_validator = DefValidator() - def_string = HedString("(Definition/EmptyDef)", self.hed_schema) - def_validator.check_for_definitions(def_string) - - valid_empty = HedString("Def/EmptyDef", self.hed_schema) - def_issues = def_validator.validate_def_tags(valid_empty) - def_issues += def_validator.expand_def_tags(valid_empty) - self.assertEqual(str(valid_empty), "(Def-expand/EmptyDef)") - self.assertFalse(def_issues) - - valid_empty = HedString("Def/EmptyDef", self.hed_schema) - def_issues = def_validator.validate_def_tags(valid_empty) - self.assertFalse(def_issues) - def test_duplicate_def(self): def_dict = DefinitionDict() def_string = HedString(self.placeholder_definition_string, self.hed_schema) @@ -137,11 +121,11 @@ def setUpClass(cls): cls.basic_hed_string_with_def_first = f"{cls.label_def_string},{cls.basic_hed_string}" cls.basic_hed_string_with_def_first_paren = f"({cls.label_def_string},{cls.basic_hed_string})" cls.placeholder_label_def_string = "Def/TestDefPlaceholder/2471" - cls.placeholder_definition_contents = "(Item/TestDef1/#,Item/TestDef2)" + cls.placeholder_definition_contents = "(Acceleration/#,Item/TestDef2)" cls.placeholder_definition_string = f"(Definition/TestDefPlaceholder/#,{cls.placeholder_definition_contents})" cls.placeholder_definition_string_no_paren = \ f"Definition/TestDefPlaceholder/#,{cls.placeholder_definition_contents}" - cls.placeholder_expanded_def_string = "(Def-expand/TestDefPlaceholder/2471,(Item/TestDef1/2471,Item/TestDef2))" + cls.placeholder_expanded_def_string = "(Def-expand/TestDefPlaceholder/2471,(Acceleration/2471,Item/TestDef2))" cls.placeholder_hed_string_with_def = f"{cls.basic_hed_string},{cls.placeholder_label_def_string}" cls.placeholder_hed_string_with_def_first = f"{cls.placeholder_label_def_string},{cls.basic_hed_string}" @@ -266,7 +250,7 @@ def test_expand_def_tags_placeholder(self): remove_definitions=True, basic_definition_string=self.placeholder_definition_string) - # todo ian: finish updating these + # todo: finish updating these # # special case test # def test_changing_tag_then_def_mapping(self): # def_dict = DefinitionDict() diff --git a/tests/validator/test_hed_validator.py b/tests/validator/test_hed_validator.py index a523e33c3..848beda34 100644 --- a/tests/validator/test_hed_validator.py +++ b/tests/validator/test_hed_validator.py @@ -88,6 +88,7 @@ def test_complex_file_validation_with_index(self): self.assertEqual(len(validation_issues), 0) def test_complex_file_validation_invalid(self): + # todo: Update or remove schema_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '../data/validator_tests/bids_schema.mediawiki')) events_path = os.path.realpath(os.path.join(os.path.dirname(__file__), @@ -98,13 +99,14 @@ def test_complex_file_validation_invalid(self): '../data/validator_tests/bids_events_bad_defs.json')) sidecar = Sidecar(json_path) issues = sidecar.validate(hed_schema) - self.assertEqual(len(issues), 4) + self.assertEqual(len(issues), 8) input_file = TabularInput(events_path, sidecar=sidecar) validation_issues = input_file.validate(hed_schema) - self.assertEqual(len(validation_issues), 63) + self.assertEqual(len(validation_issues), 105) def test_complex_file_validation_invalid_definitions_removed(self): + # todo: update this/remove # This verifies definitions are being removed from sidecar strings before being added, or it will produce # extra errors. schema_path = os.path.realpath(os.path.join(os.path.dirname(__file__), @@ -117,11 +119,11 @@ def test_complex_file_validation_invalid_definitions_removed(self): '../data/validator_tests/bids_events_bad_defs2.json')) sidecar = Sidecar(json_path) issues = sidecar.validate(hed_schema) - self.assertEqual(len(issues), 4) + self.assertEqual(len(issues), 7) input_file = TabularInput(events_path, sidecar=sidecar) validation_issues = input_file.validate(hed_schema) - self.assertEqual(len(validation_issues), 42) + self.assertEqual(len(validation_issues), 63) def test_file_bad_defs_in_spreadsheet(self): schema_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -137,7 +139,7 @@ def test_file_bad_defs_in_spreadsheet(self): worksheet_name='LKT Events') validation_issues = loaded_file.validate(hed_schema=hed_schema) - self.assertEqual(len(validation_issues), 2) + self.assertEqual(len(validation_issues), 4) def test_tabular_input_with_HED_col_in_json(self): schema_path = os.path.realpath(os.path.join(os.path.dirname(__file__), diff --git a/tests/validator/test_onset_validator.py b/tests/validator/test_onset_validator.py index ef7e3aa54..c6908646a 100644 --- a/tests/validator/test_onset_validator.py +++ b/tests/validator/test_onset_validator.py @@ -18,19 +18,19 @@ def setUpClass(cls): hed_xml_file = os.path.join(cls.base_data_dir, "schema_tests/HED8.0.0.mediawiki") cls.hed_schema = schema.load_schema(hed_xml_file) cls.placeholder_label_def_string = "Def/TestDefPlaceholder/2471" - cls.placeholder_def_contents = "(Action/TestDef1/#,Action/TestDef2)" + cls.placeholder_def_contents = "(Acceleration/#,Action/TestDef2)" cls.placeholder_definition_string = f"(Definition/TestDefPlaceholder/#,{cls.placeholder_def_contents})" - cls.placeholder_expanded_def_string = "(Def-expand/TestDefPlaceholder/2471,(Action/TestDef1/2471,Action/TestDef2))" + cls.placeholder_expanded_def_string = "(Def-expand/TestDefPlaceholder/2471,(Acceleration/2471,Action/TestDef2))" cls.label_def_string = "Def/TestDefNormal" cls.def_contents = "(Action/TestDef1,Action/TestDef2)" cls.definition_string = f"(Definition/TestDefNormal,{cls.def_contents})" - cls.expanded_def_string = "(Def-expand/TestDefNormal,(Action/TestDef1/2471,Action/TestDef2))" + cls.expanded_def_string = "(Def-expand/TestDefNormal,(Acceleration/2471,Action/TestDef2))" cls.placeholder_label_def_string2 = "Def/TestDefPlaceholder/123" - cls.placeholder_def_contents2 = "(Action/TestDef1/#,Action/TestDef2)" + cls.placeholder_def_contents2 = "(Acceleration/#,Action/TestDef2)" cls.placeholder_definition_string2 = f"(Definition/TestDefPlaceholder/#,{cls.placeholder_def_contents2})" - cls.placeholder_expanded_def_string2 = "(Def-expand/TestDefPlaceholder/123,(Action/TestDef1/123,Action/TestDef2))" + cls.placeholder_expanded_def_string2 = "(Def-expand/TestDefPlaceholder/123,(Acceleration/123,Action/TestDef2))" cls.def_dict_placeholder = DefinitionDict() def_string = HedString(cls.placeholder_definition_string, hed_schema=cls.hed_schema) @@ -263,9 +263,13 @@ def test_onset_multiple_or_misplaced_errors(self): f"({self.placeholder_label_def_string},Onset, Offset)", ] test_issues = [ - self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1), - self.format_error(OnsetErrors.ONSET_TAG_OUTSIDE_OF_GROUP, tag=2, def_tag="Def/TestDefPlaceholder/2471"), - self.format_error(OnsetErrors.ONSET_TAG_OUTSIDE_OF_GROUP, tag=2, def_tag="Def/TestDefPlaceholder/2471"), + self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1, actual_error=ValidationErrors.ONSET_OFFSET_ERROR) + + self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1), + self.format_error(ValidationErrors.HED_MULTIPLE_TOP_TAGS, tag=1, multiple_tags=["Onset"]) + + self.format_error(ValidationErrors.HED_TAG_REPEATED, tag=2) + + self.format_error(OnsetErrors.ONSET_TAG_OUTSIDE_OF_GROUP, tag=2, def_tag="Def/TestDefPlaceholder/2471"), + self.format_error(ValidationErrors.HED_MULTIPLE_TOP_TAGS, tag=1, multiple_tags=["Offset"]) + + self.format_error(OnsetErrors.ONSET_TAG_OUTSIDE_OF_GROUP, tag=2, def_tag="Def/TestDefPlaceholder/2471"), ] self._test_issues_no_context(test_strings, test_issues) diff --git a/tests/validator/test_tag_validator.py b/tests/validator/test_tag_validator.py index 939423638..5f453f806 100644 --- a/tests/validator/test_tag_validator.py +++ b/tests/validator/test_tag_validator.py @@ -1,6 +1,6 @@ import unittest -from hed.errors.error_types import ValidationErrors +from hed.errors.error_types import ValidationErrors, DefinitionErrors from tests.validator.test_tag_validator_base import TestValidatorBase from functools import partial @@ -26,7 +26,7 @@ def test_exist_in_schema(self): 'usedToBeIllegalComma': 'Label/This is a label,This/Is/A/Tag', 'legalDef': 'Def/Item', 'legalDefExpand': 'Def-expand/Item', - 'legalDefinition': 'Definition/Item', + 'illegalDefinition': 'Definition/Item', } expected_results = { 'takesValue': True, @@ -39,14 +39,14 @@ def test_exist_in_schema(self): 'usedToBeIllegalComma': False, 'legalDef': True, 'legalDefExpand': True, - 'legalDefinition': True, + 'illegalDefinition': False, } expected_issues = { 'takesValue': [], 'full': [], 'extensionsAllowed': [], - 'leafExtension': self.format_error(ValidationErrors.INVALID_EXTENSION, tag=0), - 'nonExtensionsAllowed': self.format_error(ValidationErrors.INVALID_EXTENSION, tag=0), + 'leafExtension': self.format_error(ValidationErrors.TAG_EXTENSION_INVALID, tag=0), + 'nonExtensionsAllowed': self.format_error(ValidationErrors.TAG_EXTENSION_INVALID, tag=0), 'invalidExtension': self.format_error( ValidationErrors.INVALID_PARENT_NODE, tag=0, index_in_tag=6, index_in_tag_end=9, expected_parent_tag="Property/Sensory-property/Sensory-attribute/Visual-attribute" + @@ -59,7 +59,7 @@ def test_exist_in_schema(self): index_in_tag=0, index_in_tag_end=4), 'legalDef': [], 'legalDefExpand': [], - 'legalDefinition': [] + 'illegalDefinition': self.format_error(DefinitionErrors.BAD_DEFINITION_LOCATION, tag=0) } self.validator_semantic(test_strings, expected_results, expected_issues, False) @@ -83,7 +83,7 @@ def test_proper_capitalization(self): 'camelCase': [], 'takesValue': [], 'numeric': [], - 'lowercase': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0) + 'lowercase': self.format_error(ValidationErrors.STYLE_WARNING, tag=0) } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -106,11 +106,11 @@ def test_proper_capitalization(self): # } # expected_issues = { # 'proper': [], - # 'camelCase': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0), + # 'camelCase': self.format_error(ValidationErrors.STYLE_WARNING, tag=0), # 'takesValue': [], # 'numeric': [], - # 'lowercase': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0), - # 'multipleUpper': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0) + # 'lowercase': self.format_error(ValidationErrors.STYLE_WARNING, tag=0), + # 'multipleUpper': self.format_error(ValidationErrors.STYLE_WARNING, tag=0) # } # self.validator_semantic(test_strings, expected_results, expected_issues, True) # @@ -133,11 +133,11 @@ def test_proper_capitalization(self): # } # expected_issues = { # 'proper': [], - # 'camelCase': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0), + # 'camelCase': self.format_error(ValidationErrors.STYLE_WARNING, tag=0), # 'takesValue': [], # 'numeric': [], - # 'lowercase': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0), - # 'multipleUpper': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0) + # 'lowercase': self.format_error(ValidationErrors.STYLE_WARNING, tag=0), + # 'multipleUpper': self.format_error(ValidationErrors.STYLE_WARNING, tag=0) # } # self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -152,7 +152,7 @@ def test_child_required(self): } expected_issues = { 'hasChild': [], - 'missingChild': self.format_error(ValidationErrors.HED_TAG_REQUIRES_CHILD, tag=0) + 'missingChild': self.format_error(ValidationErrors.TAG_REQUIRES_CHILD, tag=0) } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -179,14 +179,14 @@ def test_required_units(self): # legal_clock_time_units = ['hour:min', 'hour:min:sec'] expected_issues = { 'hasRequiredUnit': [], - 'missingRequiredUnit': self.format_error(ValidationErrors.HED_UNITS_DEFAULT_USED, tag=0, + 'missingRequiredUnit': self.format_error(ValidationErrors.UNITS_MISSING, tag=0, default_unit='s'), 'notRequiredNoNumber': [], 'notRequiredNumber': [], 'notRequiredScientific': [], - 'timeValue': self.format_error(ValidationErrors.HED_TAG_EXTENDED, tag=0, + 'timeValue': self.format_error(ValidationErrors.TAG_EXTENDED, tag=0, index_in_tag=10, index_in_tag_end=None), - 'invalidTimeValue': self.format_error(ValidationErrors.HED_TAG_EXTENDED, tag=0, + 'invalidTimeValue': self.format_error(ValidationErrors.TAG_EXTENDED, tag=0, index_in_tag=10, index_in_tag_end=None), } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -249,27 +249,27 @@ def test_correct_units(self): 'correctNoPluralUnit': [], 'correctNonSymbolCapitalizedUnit': [], 'correctSymbolCapitalizedUnit': [], - 'incorrectUnit': self.format_error(ValidationErrors.HED_UNITS_INVALID, + 'incorrectUnit': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=legal_time_units), - 'incorrectSiUsage': self.format_error(ValidationErrors.HED_UNITS_INVALID, + 'incorrectSiUsage': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=legal_time_units), - 'incorrectPluralUnit': self.format_error(ValidationErrors.HED_UNITS_INVALID, + 'incorrectPluralUnit': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=legal_freq_units), - 'incorrectSymbolCapitalizedUnit': self.format_error(ValidationErrors.HED_UNITS_INVALID, + 'incorrectSymbolCapitalizedUnit': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=legal_freq_units), 'incorrectSymbolCapitalizedUnitModifier': self.format_error( - ValidationErrors.HED_UNITS_INVALID, tag=0, units=legal_freq_units), + ValidationErrors.UNITS_INVALID, tag=0, units=legal_freq_units), 'notRequiredNumber': [], 'notRequiredScientific': [], - 'specialAllowedCharBadUnit': self.format_error(ValidationErrors.HED_VALUE_INVALID, + 'specialAllowedCharBadUnit': self.format_error(ValidationErrors.VALUE_INVALID, tag=0), 'specialAllowedCharUnit': [], # 'properTime': [], - # 'invalidTime': self.format_error(ValidationErrors.HED_UNITS_INVALID, tag=0, + # 'invalidTime': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, # units=legal_clock_time_units) # 'specialAllowedCharCurrency': [], - # 'specialNotAllowedCharCurrency': self.format_error(ValidationErrors.HED_UNITS_INVALID, + # 'specialNotAllowedCharCurrency': self.format_error(ValidationErrors.UNITS_INVALID, # tag=0, # units=legal_currency_units), } @@ -300,7 +300,7 @@ def test_extension_warning(self): } expected_issues = { 'noWarning': [], - 'warning': self.format_error(ValidationErrors.HED_TAG_EXTENDED, tag=0, + 'warning': self.format_error(ValidationErrors.TAG_EXTENDED, tag=0, index_in_tag=13, index_in_tag_end=None), } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -319,12 +319,12 @@ def test_invalid_placeholder_in_normal_string(self): expected_issues = { 'invalidPlaceholder': self.format_error(ValidationErrors.INVALID_TAG_CHARACTER, tag=0, index_in_tag=9, index_in_tag_end=10, - actual_error=ValidationErrors.HED_VALUE_INVALID), + actual_error=ValidationErrors.PLACEHOLDER_INVALID), 'invalidMiscPoundSign': self.format_error(ValidationErrors.NO_VALID_TAG_FOUND, tag=0, index_in_tag=0, index_in_tag_end=8), 'invalidAfterBaseTag': self.format_error(ValidationErrors.INVALID_TAG_CHARACTER, tag=0, index_in_tag=14, index_in_tag_end=15, - actual_error=ValidationErrors.HED_VALUE_INVALID), + actual_error=ValidationErrors.PLACEHOLDER_INVALID), } self.validator_semantic(test_strings, expected_results, expected_issues, False) @@ -339,12 +339,12 @@ def test_span_reporting(self): } tag_unit_class_units = ['day', 'hour', 'minute', 's', 'second'] expected_issues = { - 'orgTagDifferent': self.format_error(ValidationErrors.HED_UNITS_INVALID, + 'orgTagDifferent': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=tag_unit_class_units), 'orgTagDifferent2': - self.format_error(ValidationErrors.HED_UNITS_INVALID, + self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=tag_unit_class_units) - + self.format_error(ValidationErrors.HED_UNITS_INVALID, tag=1, + + self.format_error(ValidationErrors.UNITS_INVALID, tag=1, units=tag_unit_class_units), } self.validator_semantic(test_strings, expected_results, expected_issues, False) @@ -429,10 +429,12 @@ def test_topLevelTagGroup_validation(self): 'invalid2TwoInOne': False, } expected_issues = { - 'invalid1': self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=0), + 'invalid1': self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=0, actual_error=ValidationErrors.DEFINITION_INVALID) + + self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=0), 'valid1': [], 'valid2': [], - 'invalid2': self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1), + 'invalid2': self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1, actual_error=ValidationErrors.DEFINITION_INVALID) + + self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1), 'invalidTwoInOne': self.format_error( ValidationErrors.HED_MULTIPLE_TOP_TAGS, tag=0, multiple_tags="Definition/InvalidDef3".split(", ")), @@ -533,13 +535,13 @@ def test_mismatched_parentheses(self): 'valid': True } expected_issues = { - 'extraOpening': self.format_error(ValidationErrors.HED_PARENTHESES_MISMATCH, + 'extraOpening': self.format_error(ValidationErrors.PARENTHESES_MISMATCH, opening_parentheses_count=2, closing_parentheses_count=1), - 'extraClosing': self.format_error(ValidationErrors.HED_PARENTHESES_MISMATCH, + 'extraClosing': self.format_error(ValidationErrors.PARENTHESES_MISMATCH, opening_parentheses_count=1, closing_parentheses_count=2) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, source_string=test_strings['extraClosing'], + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['extraClosing'], char_index=84), 'valid': [] } @@ -625,49 +627,49 @@ def test_malformed_delimiters(self): tag="Action/Reach/To touch("), 'missingClosingComma': self.format_error(ValidationErrors.COMMA_MISSING, tag="Participant/Effect/Body part/Arm)"), - 'extraOpeningComma': self.format_error(ValidationErrors.HED_TAG_EMPTY, + 'extraOpeningComma': self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['extraOpeningComma'], char_index=0), - 'extraClosingComma': self.format_error(ValidationErrors.HED_TAG_EMPTY, + 'extraClosingComma': self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['extraClosingComma'], char_index=len( test_strings['extraClosingComma']) - 1), - # 'extraOpeningParen': self.format_error(ValidationErrors.HED_TAG_EMPTY, + # 'extraOpeningParen': self.format_error(ValidationErrors.TAG_EMPTY, # character='(', index_in_tag=0), - # 'extraClosingParen': self.format_error(ValidationErrors.HED_TAG_EMPTY, character=')', + # 'extraClosingParen': self.format_error(ValidationErrors.TAG_EMPTY, character=')', # index_in_tag=len(test_strings['extraClosingParen']) - 1), - 'extraOpeningParen': self.format_error(ValidationErrors.HED_PARENTHESES_MISMATCH, + 'extraOpeningParen': self.format_error(ValidationErrors.PARENTHESES_MISMATCH, opening_parentheses_count=2, closing_parentheses_count=1), - 'extraClosingParen': self.format_error(ValidationErrors.HED_PARENTHESES_MISMATCH, + 'extraClosingParen': self.format_error(ValidationErrors.PARENTHESES_MISMATCH, opening_parentheses_count=1, closing_parentheses_count=2), 'multipleExtraOpeningDelimiters': - self.format_error(ValidationErrors.HED_TAG_EMPTY, + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraOpeningDelimiters'], char_index=0) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraOpeningDelimiters'], char_index=1) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraOpeningDelimiters'], char_index=2), 'multipleExtraClosingDelimiters': - self.format_error(ValidationErrors.HED_TAG_EMPTY, + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraClosingDelimiters'], char_index=len(test_strings['multipleExtraClosingDelimiters']) - 1) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraClosingDelimiters'], char_index=len(test_strings['multipleExtraClosingDelimiters']) - 2) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraClosingDelimiters'], char_index=len(test_strings['multipleExtraClosingDelimiters']) - 3) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraClosingDelimiters'], char_index=len(test_strings['multipleExtraClosingDelimiters']) - 4), 'multipleExtraMiddleDelimiters': - self.format_error(ValidationErrors.HED_TAG_EMPTY, + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraMiddleDelimiters'], char_index=22) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraMiddleDelimiters'], char_index=121) - + self.format_error(ValidationErrors.HED_TAG_EMPTY, + + self.format_error(ValidationErrors.TAG_EMPTY, source_string=test_strings['multipleExtraMiddleDelimiters'], char_index=122), 'valid': [], 'validNestedParentheses': [], @@ -743,40 +745,40 @@ def test_string_extra_slash_space(self): 'trailingDoubleSlashWithSpace': False, } expected_errors = { - 'twoLevelDoubleSlash': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'twoLevelDoubleSlash': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=5, index_in_tag_end=7, tag=0), 'threeLevelDoubleSlash': - self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=7, index_in_tag_end=9, tag=0) - + self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + + self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=13, index_in_tag_end=15, tag=0), 'tripleSlashes': - self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, index_in_tag=7, index_in_tag_end=10, tag=0) - + self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=7, index_in_tag_end=10, tag=0) + + self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=14, index_in_tag_end=17, tag=0), - 'mixedSingleAndDoubleSlashes': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'mixedSingleAndDoubleSlashes': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=7, index_in_tag_end=9, tag=0), - 'singleSlashWithSpace': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'singleSlashWithSpace': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=5, index_in_tag_end=7, tag=0), - 'doubleSlashSurroundingSpace': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'doubleSlashSurroundingSpace': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=5, index_in_tag_end=8, tag=0), - 'doubleSlashThenSpace': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'doubleSlashThenSpace': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=5, index_in_tag_end=8, tag=0), - 'sosPattern': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, index_in_tag=5, + 'sosPattern': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=5, index_in_tag_end=14, tag=0), 'alternatingSlashSpace': - self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, index_in_tag=7, index_in_tag_end=11, tag=0) - + self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=7, index_in_tag_end=11, tag=0) + + self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=15, index_in_tag_end=19, tag=0), - 'leadingDoubleSlash': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'leadingDoubleSlash': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=0, index_in_tag_end=2, tag=0), - 'trailingDoubleSlash': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'trailingDoubleSlash': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=15, index_in_tag_end=17, tag=0), - 'leadingDoubleSlashWithSpace': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'leadingDoubleSlashWithSpace': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=0, index_in_tag_end=3, tag=0), - 'trailingDoubleSlashWithSpace': self.format_error(ValidationErrors.HED_NODE_NAME_EMPTY, + 'trailingDoubleSlashWithSpace': self.format_error(ValidationErrors.NODE_NAME_EMPTY, index_in_tag=15, index_in_tag_end=18, tag=0), } @@ -803,20 +805,20 @@ def test_no_more_than_two_tildes(self): } expected_issues = { 'noTildeGroup': [], - 'oneTildeGroup': self.format_error(ValidationErrors.HED_TILDES_UNSUPPORTED, + 'oneTildeGroup': self.format_error(ValidationErrors.TILDES_UNSUPPORTED, source_string=test_strings['oneTildeGroup'], char_index=56), 'twoTildeGroup': - self.format_error(ValidationErrors.HED_TILDES_UNSUPPORTED, + self.format_error(ValidationErrors.TILDES_UNSUPPORTED, source_string=test_strings['twoTildeGroup'], char_index=49) - + self.format_error(ValidationErrors.HED_TILDES_UNSUPPORTED, + + self.format_error(ValidationErrors.TILDES_UNSUPPORTED, source_string=test_strings['twoTildeGroup'], char_index=77), 'invalidTildeGroup': - self.format_error(ValidationErrors.HED_TILDES_UNSUPPORTED, + self.format_error(ValidationErrors.TILDES_UNSUPPORTED, source_string=test_strings['invalidTildeGroup'], char_index=49) - + self.format_error(ValidationErrors.HED_TILDES_UNSUPPORTED, + + self.format_error(ValidationErrors.TILDES_UNSUPPORTED, source_string=test_strings['invalidTildeGroup'], char_index=77) - + self.format_error(ValidationErrors.HED_TILDES_UNSUPPORTED, + + self.format_error(ValidationErrors.TILDES_UNSUPPORTED, source_string=test_strings['invalidTildeGroup'], char_index=147) } self.validator_semantic(test_strings, expected_results, expected_issues, False) @@ -846,13 +848,13 @@ def test_includes_all_required_tags(self): } expected_issues = { 'complete': [], - 'missingAgent': self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, + 'missingAgent': self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Agent/Animal-agent'), - 'missingAction': self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='Action'), + 'missingAction': self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Action'), 'inSubGroup': [], 'missingAll': - self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='Action') - + self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='Agent/Animal-agent'), + self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Action') + + self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Agent/Animal-agent'), } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -874,9 +876,9 @@ def test_multiple_copies_unique_tags(self): } expected_issues = { 'legal': [], - 'multipleDesc': self.format_error(ValidationErrors.HED_TAG_NOT_UNIQUE, + 'multipleDesc': self.format_error(ValidationErrors.TAG_NOT_UNIQUE, tag_prefix='Property/Organizational-property/Event-context'), - 'multipleDescIncShort': self.format_error(ValidationErrors.HED_TAG_NOT_UNIQUE, + 'multipleDescIncShort': self.format_error(ValidationErrors.TAG_NOT_UNIQUE, tag_prefix='Property/Organizational-property/Event-context'), } self.validator_semantic(test_strings, expected_results, expected_issues, False) @@ -912,10 +914,12 @@ def test_special_units(self): # 'properTime': [], # 'invalidTime': [], 'specialAllowedCharCurrency': [], - 'specialNotAllowedCharCurrency': self.format_error(ValidationErrors.HED_UNITS_INVALID, + 'specialNotAllowedCharCurrency': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, - units=legal_currency_units), - 'specialAllowedCharCurrencyAsSuffix': self.format_error(ValidationErrors.HED_UNITS_INVALID, + units=legal_currency_units) + + self.format_error(ValidationErrors.VALUE_INVALID, + tag=0), + 'specialAllowedCharCurrencyAsSuffix': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=legal_currency_units), } diff --git a/tests/validator/test_tag_validator_library.py b/tests/validator/test_tag_validator_library.py index c4552f689..571fa2b85 100644 --- a/tests/validator/test_tag_validator_library.py +++ b/tests/validator/test_tag_validator_library.py @@ -3,7 +3,7 @@ from hed.errors import error_reporter from hed import schema -from hed.errors.error_types import ValidationErrors +from hed.errors.error_types import ValidationErrors, DefinitionErrors from hed.schema.hed_schema_group import HedSchemaGroup from hed.errors.exceptions import HedFileError from tests.validator.test_tag_validator_base import TestValidatorBase @@ -58,7 +58,7 @@ def test_exist_in_schema(self): 'usedToBeIllegalComma': 'tl:Label/This is a label,tl:This/Is/A/Tag', 'legalDef': 'tl:Def/Item', 'legalDefExpand': 'tl:Def-expand/Item', - 'legalDefinition': 'tl:Definition/Item', + 'illegalDefinition': 'tl:Definition/Item', 'unknownPrefix': 'ul:Definition/Item' } expected_results = { @@ -72,15 +72,15 @@ def test_exist_in_schema(self): 'usedToBeIllegalComma': False, 'legalDef': True, 'legalDefExpand': True, - 'legalDefinition': True, + 'illegalDefinition': False, 'unknownPrefix': False } expected_issues = { 'takesValue': [], 'full': [], 'extensionsAllowed': [], - 'leafExtension': self.format_error(ValidationErrors.INVALID_EXTENSION, tag=0), - 'nonExtensionsAllowed': self.format_error(ValidationErrors.INVALID_EXTENSION, tag=0), + 'leafExtension': self.format_error(ValidationErrors.TAG_EXTENSION_INVALID, tag=0), + 'nonExtensionsAllowed': self.format_error(ValidationErrors.TAG_EXTENSION_INVALID, tag=0), 'invalidExtension': self.format_error( ValidationErrors.INVALID_PARENT_NODE, tag=0, index_in_tag=9, index_in_tag_end=12, expected_parent_tag="Property/Sensory-property/Sensory-attribute/Visual-attribute" + @@ -93,7 +93,7 @@ def test_exist_in_schema(self): index_in_tag=3, index_in_tag_end=7), 'legalDef': [], 'legalDefExpand': [], - 'legalDefinition': [], + 'illegalDefinition': self.format_error(DefinitionErrors.BAD_DEFINITION_LOCATION, tag=0), 'unknownPrefix': self.format_error( ValidationErrors.HED_LIBRARY_UNMATCHED, tag=0, unknown_prefix="ul:", known_prefixes=["", "tl:"]), } @@ -119,7 +119,7 @@ def test_proper_capitalization(self): 'camelCase': [], 'takesValue': [], 'numeric': [], - 'lowercase': self.format_error(ValidationErrors.HED_STYLE_WARNING, tag=0) + 'lowercase': self.format_error(ValidationErrors.STYLE_WARNING, tag=0) } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -134,7 +134,7 @@ def test_child_required(self): } expected_issues = { 'hasChild': [], - 'missingChild': self.format_error(ValidationErrors.HED_TAG_REQUIRES_CHILD, tag=0) + 'missingChild': self.format_error(ValidationErrors.TAG_REQUIRES_CHILD, tag=0) } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -162,14 +162,14 @@ def test_required_units(self): expected_issues = { 'hasRequiredUnit': [], 'missingRequiredUnit': self.format_error( - ValidationErrors.HED_UNITS_DEFAULT_USED, tag=0, default_unit='s'), + ValidationErrors.UNITS_MISSING, tag=0, default_unit='s'), 'notRequiredNoNumber': [], 'notRequiredNumber': [], 'notRequiredScientific': [], 'timeValue': self.format_error( - ValidationErrors.HED_TAG_EXTENDED, tag=0, index_in_tag=10, index_in_tag_end=None), + ValidationErrors.TAG_EXTENDED, tag=0, index_in_tag=10, index_in_tag_end=None), 'invalidTimeValue': self.format_error( - ValidationErrors.HED_TAG_EXTENDED, tag=0, index_in_tag=10, index_in_tag_end=None), + ValidationErrors.TAG_EXTENDED, tag=0, index_in_tag=10, index_in_tag_end=None), } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -230,22 +230,22 @@ def test_correct_units(self): 'correctNonSymbolCapitalizedUnit': [], 'correctSymbolCapitalizedUnit': [], 'incorrectUnit': self.format_error( - ValidationErrors.HED_UNITS_INVALID, tag=0, units=legal_time_units), + ValidationErrors.UNITS_INVALID, tag=0, units=legal_time_units), 'incorrectPluralUnit': self.format_error( - ValidationErrors.HED_UNITS_INVALID, tag=0, units=legal_freq_units), + ValidationErrors.UNITS_INVALID, tag=0, units=legal_freq_units), 'incorrectSymbolCapitalizedUnit': self.format_error( - ValidationErrors.HED_UNITS_INVALID, tag=0, units=legal_freq_units), + ValidationErrors.UNITS_INVALID, tag=0, units=legal_freq_units), 'incorrectSymbolCapitalizedUnitModifier': self.format_error( - ValidationErrors.HED_UNITS_INVALID, tag=0, units=legal_freq_units), + ValidationErrors.UNITS_INVALID, tag=0, units=legal_freq_units), 'notRequiredNumber': [], 'notRequiredScientific': [], - 'specialAllowedCharBadUnit': self.format_error(ValidationErrors.HED_VALUE_INVALID, tag=0), + 'specialAllowedCharBadUnit': self.format_error(ValidationErrors.VALUE_INVALID, tag=0), 'specialAllowedCharUnit': [], # 'properTime': [], - # 'invalidTime': self.format_error(ValidationErrors.HED_UNITS_INVALID, tag=0, + # 'invalidTime': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, # units=legal_clock_time_units) # 'specialAllowedCharCurrency': [], - # 'specialNotAllowedCharCurrency': self.format_error(ValidationErrors.HED_UNITS_INVALID, + # 'specialNotAllowedCharCurrency': self.format_error(ValidationErrors.UNITS_INVALID, # tag=0, # units=legal_currency_units), } @@ -275,7 +275,7 @@ def test_invalid_placeholder_in_normal_string(self): expected_issues = { 'invalidPlaceholder': self.format_error(ValidationErrors.INVALID_TAG_CHARACTER, tag=0, index_in_tag=12, index_in_tag_end=13, - actual_error=ValidationErrors.HED_VALUE_INVALID), + actual_error=ValidationErrors.PLACEHOLDER_INVALID), } self.validator_semantic(test_strings, expected_results, expected_issues, False) @@ -290,11 +290,11 @@ def test_span_reporting(self): } tag_unit_class_units = ['day', 'hour', 'minute', 's', 'second'] expected_issues = { - 'orgTagDifferent': self.format_error(ValidationErrors.HED_UNITS_INVALID, tag=0, + 'orgTagDifferent': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=tag_unit_class_units), - 'orgTagDifferent2': self.format_error(ValidationErrors.HED_UNITS_INVALID, tag=0, + 'orgTagDifferent2': self.format_error(ValidationErrors.UNITS_INVALID, tag=0, units=tag_unit_class_units) - + self.format_error(ValidationErrors.HED_UNITS_INVALID, tag=1, + + self.format_error(ValidationErrors.UNITS_INVALID, tag=1, units=tag_unit_class_units), } self.validator_semantic(test_strings, expected_results, expected_issues, False) @@ -367,10 +367,12 @@ def test_topLevelTagGroup_validation(self): } expected_issues = { 'invalid1': self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, - tag=0), + tag=0, actual_error=ValidationErrors.DEFINITION_INVALID) + + self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=0), 'valid1': [], 'valid2': [], - 'invalid2': self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1), + 'invalid2': self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1, actual_error=ValidationErrors.DEFINITION_INVALID) + + self.format_error(ValidationErrors.HED_TOP_LEVEL_TAG, tag=1), 'invalidTwoInOne': self.format_error( ValidationErrors.HED_MULTIPLE_TOP_TAGS, tag=0, multiple_tags="tl:Definition/InvalidDef3".split(", ")), @@ -437,15 +439,15 @@ def test_includes_all_required_tags(self): } expected_issues = { 'complete': [], - 'missingAgent': self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, + 'missingAgent': self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Agent/Animal-agent'), - 'missingAction': self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='Action'), + 'missingAction': self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Action'), 'inSubGroup': [], 'missingAll': - self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='Action') - + self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='Agent/Animal-agent') - + self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='tl:Action') - + self.format_error(ValidationErrors.HED_REQUIRED_TAG_MISSING, tag_prefix='tl:Agent/Animal-agent'), + self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Action') + + self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='Agent/Animal-agent') + + self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='tl:Action') + + self.format_error(ValidationErrors.REQUIRED_TAG_MISSING, tag_prefix='tl:Agent/Animal-agent'), } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -467,9 +469,9 @@ def test_multiple_copies_unique_tags(self): } expected_issues = { 'legal': [], - 'multipleDesc': self.format_error(ValidationErrors.HED_TAG_NOT_UNIQUE, + 'multipleDesc': self.format_error(ValidationErrors.TAG_NOT_UNIQUE, tag_prefix='tl:Property/Organizational-property/Event-context'), - 'multipleDescIncShort': self.format_error(ValidationErrors.HED_TAG_NOT_UNIQUE, + 'multipleDescIncShort': self.format_error(ValidationErrors.TAG_NOT_UNIQUE, tag_prefix='tl:Property/Organizational-property/Event-context'), } self.validator_semantic(test_strings, expected_results, expected_issues, False)