Update def handling/validation

hed/models/definition_dict.py

@@ -54,14 +54,20 @@ def _add_definitions_from_dict(self, def_dict):
              def_dict (DefinitionDict): DefDict whose definitions should be added.
 
         """
-        for def_tag, def_value in def_dict:
+        for def_tag, def_value in def_dict.items():
             self._add_definition(def_tag, def_value)
 
     def get(self, def_name):
         return self.defs.get(def_name.lower())
 
     def __iter__(self):
-        return iter(self.defs.items())
+        return iter(self.defs)
+
+    def __len__(self):
+        return len(self.defs)
+
+    def items(self):
+        return self.defs.items()
 
     @property
     def issues(self):
@@ -94,7 +100,7 @@ def check_for_definitions(self, hed_string_obj, error_handler=None):
         """
         new_def_issues = []
         for definition_tag, group in hed_string_obj.find_top_level_tags(anchor_tags={DefTagNames.DEFINITION_KEY}):
-            def_tag_name = definition_tag.extension_or_value_portion
+            def_tag_name = definition_tag.extension
 
             # initial validation
             groups = group.groups()
@@ -224,7 +230,7 @@ def _get_definition_contents(self, def_tag):
             def_contents: HedGroup
             The contents to replace the previous def-tag with.
         """
-        is_label_tag = def_tag.extension_or_value_portion
+        is_label_tag = def_tag.extension
         placeholder = None
         found_slash = is_label_tag.find("/")
         if found_slash != -1:

hed/models/definition_entry.py

@@ -18,13 +18,13 @@ def __init__(self, name, contents, takes_value, source_context):
         """
         self.name = name
         if contents:
-            contents = contents.copy()
+            contents = contents.copy().sort()
         self.contents = contents
         self.takes_value = takes_value
         self.source_context = source_context
-        self.tag_dict = {}
-        if contents:
-            add_group_to_dict(contents, self.tag_dict)
+        # self.tag_dict = {}
+        # if contents:
+        #     add_group_to_dict(contents, self.tag_dict)
 
     def get_definition(self, replace_tag, placeholder_value=None, return_copy_of_tag=False):
         """ Return a copy of the definition with the tag expanded and the placeholder plugged in.
@@ -66,28 +66,28 @@ def get_definition(self, replace_tag, placeholder_value=None, return_copy_of_tag
                                    startpos=replace_tag.span[0], endpos=replace_tag.span[1], contents=output_contents)
         return f"{DefTagNames.DEF_EXPAND_ORG_KEY}/{name}", output_contents
 
-
-def add_group_to_dict(group, tag_dict=None):
-    """ Add the tags and their values from a HED group to a tag dictionary.
-
-    Parameters:
-        group (HedGroup):   Contents to add to the tag dict.
-        tag_dict (dict):    The starting tag dictionary to add to.
-
-    Returns:
-        dict:  The updated tag_dict containing the tags with a list of values.
-
-    Notes:
-        - Expects tags to have forms calculated already.
-
-    """
-    if tag_dict is None:
-        tag_dict = {}
-    for tag in group.get_all_tags():
-        short_base_tag = tag.short_base_tag
-        value = tag.extension_or_value_portion
-        value_dict = tag_dict.get(short_base_tag, {})
-        if value:
-            value_dict[value] = ''
-        tag_dict[short_base_tag] = value_dict
-    return tag_dict
+#
+# def add_group_to_dict(group, tag_dict=None):
+#     """ Add the tags and their values from a HED group to a tag dictionary.
+#
+#     Parameters:
+#         group (HedGroup):   Contents to add to the tag dict.
+#         tag_dict (dict):    The starting tag dictionary to add to.
+#
+#     Returns:
+#         dict:  The updated tag_dict containing the tags with a list of values.
+#
+#     Notes:
+#         - Expects tags to have forms calculated already.
+#
+#     """
+#     if tag_dict is None:
+#         tag_dict = {}
+#     for tag in group.get_all_tags():
+#         short_base_tag = tag.short_base_tag
+#         value = tag.extension
+#         value_dict = tag_dict.get(short_base_tag, {})
+#         if value:
+#             value_dict[value] = ''
+#         tag_dict[short_base_tag] = value_dict
+#     return tag_dict

hed/models/df_util.py

@@ -4,6 +4,8 @@
 from hed.models.sidecar import Sidecar
 from hed.models.tabular_input import TabularInput
 from hed import HedString
+from hed.models.definition_dict import DefinitionDict
+from hed.models.definition_entry import DefinitionEntry
 
 
 def get_assembled(tabular_file, sidecar, hed_schema, extra_def_dicts=None, join_columns=True,
@@ -136,3 +138,84 @@ def _shrink_defs(hed_string, hed_schema):
 def _expand_defs(hed_string, hed_schema, def_dict):
     from hed import HedString
     return str(HedString(hed_string, hed_schema, def_dict).expand_defs())
+
+
+def process_def_expands(hed_strings, hed_schema, known_defs=None, ambiguous_defs=None):
+    """
+    Processes a list of HED strings according to a given HED schema, using known definitions and ambiguous definitions.
+
+    Args:
+        hed_strings (list or pd.Series): A list of HED strings to process.
+        hed_schema (HedSchema): The schema to use
+        known_defs (DefinitionDict or list or str), optional):
+            A DefinitionDict or anything its constructor takes.  These are the known definitions going in, that must
+            match perfectly.
+        ambiguous_defs (dict): A dictionary containing ambiguous definitions
+            format TBD.  Currently def name key: list of lists of hed tags values
+    """
+    if not isinstance(hed_strings, pd.Series):
+        hed_strings = pd.Series(hed_strings)
+
+    if ambiguous_defs is None:
+        ambiguous_defs = {}
+    errors = {}
+    def_dict = DefinitionDict(known_defs)
+
+    def_expand_mask = hed_strings.str.contains('Def-Expand/', case=False)
+
+    # Iterate over the strings that contain def-expand tags
+    for i in hed_strings[def_expand_mask].index:
+        string = hed_strings.loc[i]
+        hed_str = HedString(string, hed_schema)
+
+        for def_tag, def_expand_group, def_group in hed_str.find_def_tags(recursive=True):
+            if def_tag == def_expand_group:
+                continue
+
+            def_tag_name = def_tag.extension.split('/')[0]
+            # First check for known definitions.  If this is known, it's done either way.
+            def_group_contents = def_dict._get_definition_contents(def_tag)
+            def_expand_group.sort()
+            if def_group_contents:
+                if def_group_contents != def_expand_group:
+                    errors.setdefault(def_tag_name.lower(), []).append(def_group)
+                continue
+
+            has_extension = "/" in def_tag.extension
+
+            # If there's no extension, this is fine.
+            if not has_extension:
+                group_tag = def_expand_group.get_first_group()
+                def_dict.defs[def_tag_name.lower()] = DefinitionEntry(name=def_tag_name, contents=group_tag,
+                                                                      takes_value=False,
+                                                                      source_context=[])
+            else:
+                def_extension = def_tag.extension.split('/')[-1]
+                # Find any other tags in def_group.get_all_tags() with tags with the same extension
+                matching_tags = [tag for tag in def_expand_group.get_all_tags() if tag.extension == def_extension and tag != def_tag]
+
+                for tag in matching_tags:
+                    tag.extension = "#"
+
+                group_tag = def_expand_group.get_first_group()
+
+                these_defs = ambiguous_defs.setdefault(def_tag_name.lower(), [])
+                these_defs.append(group_tag)
+
+                value_per_tag = []
+                if len(these_defs) >= 1:
+                    all_tags_list = [group.get_all_tags() for group in these_defs]
+                    for tags in zip(*all_tags_list):
+                        value_per_tag.append(next((tag.extension for tag in tags if tag.extension != "#"), None))
+                    ambiguous_values = value_per_tag.count(None)
+                    if ambiguous_values == 1:
+                        new_contents = group_tag.copy()
+                        for tag, value in zip(new_contents.get_all_tags(), value_per_tag):
+                            if value is not None:
+                                tag.extension = f"/{value}"
+                        def_dict.defs[def_tag_name.lower()] = DefinitionEntry(name=def_tag_name, contents=new_contents,
+                                                                              takes_value=True,
+                                                                              source_context=[])
+                        del ambiguous_defs[def_tag_name.lower()]
+
+    return def_dict, ambiguous_defs, errors

hed/models/hed_group.py

@@ -157,15 +157,18 @@ def sorted(self, update_self=False):
         Returns:
             list: The list of all tags in this group, with subgroups being returned as further nested lists
         """
-        output_list = []
+        tag_list = []
+        group_list = []
         queue_list = list(self.children)
         for child in queue_list:
             if isinstance(child, HedTag):
-                output_list.append((child, child))
+                tag_list.append((child, child))
             else:
-                output_list.append((child, child.sorted(update_self)))
+                group_list.append((child, child.sorted(update_self)))
 
-        output_list.sort(key=lambda x: str(x[0]))
+        tag_list.sort(key=lambda x: str(x[0]))
+        group_list.sort(key=lambda x: str(x[0]))
+        output_list = tag_list + group_list
         if update_self:
             self._children = [x[0] for x in output_list]
         return [x[1] for x in output_list]
@@ -260,6 +263,19 @@ def groups(self):
         """
         return [group for group in self.children if isinstance(group, HedGroup)]
 
+    def get_first_group(self):
+        """ Returns the first group in this hed string or group.
+
+            Useful for things like Def-expand where they only have a single group.
+
+            Raises a ValueError if there are no groups.
+
+        Returns:
+            HedGroup: The first group
+
+        """
+        return self.groups()[0]
+
     def get_original_hed_string(self):
         """ Get the original hed string.
 
@@ -343,7 +359,7 @@ def find_placeholder_tag(self):
 
         """
         for tag in self.get_all_tags():
-            if "#" in tag.org_tag:
+            if tag.is_placeholder():
                 return tag
 
         return None
@@ -356,6 +372,10 @@ def __eq__(self, other):
         if self is other:
             return True
 
+        # Allow us to compare to a list of groups.
+        # Note this comparison will NOT check if the list has the outer parenthesis
+        if isinstance(other, list):
+            return self.children == other
         if isinstance(other, str):
             return str(self) == other
         if not isinstance(other, HedGroup) or self.children != other.children or self.is_group != other.is_group:

hed/models/hed_tag.py

@@ -215,7 +215,7 @@ def tag(self, new_tag_val):
         self.convert_to_canonical_forms(self._schema)
 
     @property
-    def extension_or_value_portion(self):
+    def extension(self):
         """ Get the extension or value of tag
 
             Generally this is just the portion after the last slash.
@@ -233,6 +233,11 @@ def extension_or_value_portion(self):
 
         return ""
 
+    @extension.setter
+    def extension(self, x):
+        self._extension_value = f"/{x}"
+
+
     @property
     def long_tag(self):
         """ Long form including value or extension.
@@ -347,7 +352,7 @@ def get_stripped_unit_value(self):
         if stripped_value:
             return stripped_value, unit
 
-        return self.extension_or_value_portion, None
+        return self.extension, None
 
     @property
     def unit_classes(self):
@@ -450,7 +455,7 @@ def is_basic_tag(self):
             bool:  True if this is a known tag without extension or value.
 
         """
-        return bool(self._schema_entry and not self.extension_or_value_portion)
+        return bool(self._schema_entry and not self.extension)
 
     def has_attribute(self, attribute):
         """ Return true if this is an attribute this tag has.
@@ -570,7 +575,7 @@ def _get_tag_units_portion(self, tag_unit_classes):
             stripped_value (str): The value with the units removed.
 
         """
-        value, _, units = self.extension_or_value_portion.rpartition(" ")
+        value, _, units = self.extension.rpartition(" ")
         if not units:
             return None, None
 
@@ -598,14 +603,19 @@ def _find_modifier_unit_entry(units, all_valid_unit_permutations):
 
         return possible_match
 
+    def is_placeholder(self):
+        if "#" in self.org_tag or "#" in self._extension_value:
+            return True
+        return False
+
     def replace_placeholder(self, placeholder_value):
         """ If tag has a placeholder character(#), replace with value.
 
         Parameters:
             placeholder_value (str): Value to replace placeholder with.
 
         """
-        if "#" in self.org_tag:
+        if self.is_placeholder():
             if self._schema_entry:
                 self._extension_value = self._extension_value.replace("#", placeholder_value)
             else:

hed/tools/analysis/event_manager.py

@@ -66,7 +66,7 @@ def _create_event_list(self):
             for tup in group_tuples:
                 group = tup[1]
                 anchor_tag = group.find_def_tags(recursive=False, include_groups=0)[0]
-                anchor = anchor_tag.extension_or_value_portion.lower()
+                anchor = anchor_tag.extension.lower()
                 if anchor in onset_dict or tup[0].short_base_tag.lower() == "offset":
                     temporal_event = onset_dict.pop(anchor)
                     temporal_event.set_end(event_index, self.data.dataframe.loc[event_index, "onset"])
@@ -114,7 +114,7 @@ def _update_onset_list(self, group, onset_dict, event_index):
             - Modifies onset_dict and onset_list.
         """
         # def_tags = group.find_def_tags(recursive=False, include_groups=0)
-        # name = def_tags[0].extension_or_value_portion
+        # name = def_tags[0].extension
         # onset_element = onset_dict.pop(name, None)
         # if onset_element:
         #     onset_element.end_index = event_index

hed/tools/analysis/hed_context_manager.py

@@ -129,7 +129,7 @@ def _update_onset_list(self, group, onset_dict, event_index, is_offset=False):
             - Modifies onset_dict and onset_list.
         """
         def_tags = group.find_def_tags(recursive=False, include_groups=0)
-        name = def_tags[0].extension_or_value_portion
+        name = def_tags[0].extension
         onset_element = onset_dict.pop(name, None)
         if onset_element:
             onset_element.end_index = event_index

hed/tools/analysis/hed_tag_counts.py

@@ -29,7 +29,7 @@ def set_value(self, hed_tag):
         """
         if not hed_tag:
             return
-        value = hed_tag.extension_or_value_portion
+        value = hed_tag.extension
         if not value:
             value = None
         if value in self.value_dict: