Updated tabular input and sidecar to always take schema in tools

hed/models/hed_tag.py

@@ -252,6 +252,7 @@ def tag_terms(self):
 
         # TODO: Potentially remove this.  It's just a quick hack for testing
         return tuple(str(self).lower())
+        #return tuple()
 
     def __str__(self):
         """ Convert this HedTag to a string.

hed/tools/analysis/hed_tag_counts.py

@@ -1,11 +1,11 @@
-""" Keeps the counts of HED tags in a file's annotations. """
+""" Counts of HED tags in a file's annotations. """
 
 import copy
 
 
 class HedTagCount:
     def __init__(self, hed_tag, file_name):
-        """ Keeps the counts for a particular HedTag.
+        """ Counts for a particular HedTag in particular file.
 
         Parameters:
             hed_tag (HedTag):  The HedTag to keep track of.
@@ -21,6 +21,12 @@ def __init__(self, hed_tag, file_name):
         self.set_value(hed_tag)
 
     def set_value(self, hed_tag):
+        """ Update the tag term value counts for a HedTag. 
+
+        Parameters:
+            hed_tag (HedTag or None):  Item to use to update the value counts. 
+
+        """
         if not hed_tag:
             return
         value = hed_tag.extension_or_value_portion
@@ -39,6 +45,12 @@ def get_info(self, verbose=False):
         return {'tag': self.tag, 'events': self.events, 'files': files}
 
     def get_summary(self):
+        """ Return a dictionary summary of the events and files for this tag.
+
+        Returns:
+            dict:  dictionary summary of events and files that contain this tag.
+
+        """
         return {'tag': self.tag, 'events': self.events, 'files': [name for name in self.files]}
 
     def get_empty(self):
@@ -50,7 +62,11 @@ def get_empty(self):
 
 
 class HedTagCounts:
-    """ Keeps a summary of tag counts for a tabular file.
+    """ Counts of HED tags for a tabular file.
+
+    Parameters:
+        name (str):  An identifier for these counts (usually the filename of the tabular file)
+        total_events (int):  The total number of events in the tabular file.
 
 
     """
@@ -61,7 +77,15 @@ def __init__(self, name, total_events=0):
         self.files = {}
         self.total_events = total_events
 
-    def update_event_counts(self, hed_string_obj, file_name):
+    def update_event_counts(self, hed_string_obj, file_name, definitions=None):
+        """ Update the tag counts based on a hed string object. 
+
+        Parameters:
+            hed_string_obj (HedString): The HED string whose tags should be counted.
+            file_name (str): The name of the file corresponding to these counts.
+            definitions (dict): The definitions associated with the HED string.
+
+        """
         if file_name not in self.files:
             self.files[file_name] = ""
         tag_list = hed_string_obj.get_all_tags()

hed/tools/analysis/hed_type_counts.py

@@ -5,8 +5,8 @@ class HedTypeCount:
     """ Keeps a summary of one value of one type of variable.
 
     Parameters:
-        type_value (str)  The value of the variable to be counted
-        type_tag (str)   The type of variable.
+        type_value (str):  The value of the variable to be counted
+        type_tag (str):   The type of variable.
 
     Examples:
         HedTypeCounts('SymmetricCond', 'condition-variable') keeps counts of Condition-variable/Symmetric

hed/tools/remodeling/operations/factor_hed_tags_op.py

@@ -5,6 +5,7 @@
 import numpy as np
 from hed.tools.remodeling.operations.base_op import BaseOp
 from hed.models.tabular_input import TabularInput
+from hed.models.sidecar import Sidecar
 from hed.models.expression_parser import QueryParser
 from hed.tools.analysis.analysis_util import get_assembled_strings
 
@@ -99,6 +100,8 @@ def do_op(self, dispatcher, df, name, sidecar=None):
 
         """
 
+        if sidecar and not isinstance(sidecar, Sidecar):
+            sidecar = Sidecar(sidecar, hed_schema=dispatcher.hed_schema)
         input_data = TabularInput(df, hed_schema=dispatcher.hed_schema, sidecar=sidecar)
         column_names = list(df.columns)
         for name in self.query_names:

hed/tools/remodeling/operations/factor_hed_type_op.py

@@ -4,6 +4,7 @@
 import numpy as np
 from hed.tools.remodeling.operations.base_op import BaseOp
 from hed.models.tabular_input import TabularInput
+from hed.models.sidecar import Sidecar
 from hed.tools.analysis.analysis_util import get_assembled_strings
 from hed.tools.analysis.hed_type_manager import HedTypeManager
 
@@ -68,6 +69,8 @@ def do_op(self, dispatcher, df, name, sidecar=None):
 
         """
 
+        if sidecar and not isinstance(sidecar, Sidecar):
+            sidecar = Sidecar(sidecar, hed_schema=dispatcher.hed_schema)
         input_data = TabularInput(df, hed_schema=dispatcher.hed_schema, sidecar=sidecar)
         df = input_data.dataframe.copy()
         df_list = [df]

hed/tools/remodeling/operations/summarize_hed_tags_op.py

@@ -1,6 +1,7 @@
 """ Summarize the HED tags in collection of tabular files.  """
 
 from hed.models.tabular_input import TabularInput
+from hed.models.sidecar import Sidecar
 from hed.tools.analysis.hed_tag_counts import HedTagCounts
 from hed.tools.remodeling.operations.base_op import BaseOp
 from hed.tools.remodeling.operations.base_context import BaseContext
@@ -13,11 +14,10 @@ class SummarizeHedTagsOp(BaseOp):
     Required remodeling parameters:   
         - **summary_name** (*str*): The name of the summary.   
         - **summary_filename** (*str*): Base filename of the summary.   
-        - **type_tags** (*list*): Type tag to get_summary separately (e.g. 'condition-variable' or 'task').   
-        - **include_context** (*bool*): If True, expand Onset and Offset tags.   
-
+        - **tags** (*dict*): Type tag to get_summary separately (e.g. 'condition-variable' or 'task').   
+
     Optional remodeling parameters:    
-        - **breakout_list** (*list*):  A list of tags to be separated.  
+       - **expand_context** (*bool*): If True, include counts from expanded context (not supported).   
 
 
     The purpose of this op is to produce a summary of the occurrences of specified tag. This summary
@@ -31,10 +31,10 @@ class SummarizeHedTagsOp(BaseOp):
         "required_parameters": {
             "summary_name": str,
             "summary_filename": str,
-            "tags": dict,
-            "expand_context": bool
+            "tags": dict
         },
         "optional_parameters": {
+            "expand_context": bool
         }
     }
 
@@ -59,7 +59,7 @@ def __init__(self, parameters):
         self.summary_name = parameters['summary_name']
         self.summary_filename = parameters['summary_filename']
         self.tags = parameters['tags']
-        self.expand_context = parameters['expand_context']
+        self.expand_context = parameters.get('expand_context', False)
 
     def do_op(self, dispatcher, df, name, sidecar=None):
         """ Create factor columns corresponding to values in a specified column.
@@ -95,10 +95,13 @@ def __init__(self, sum_op):
 
     def update_context(self, new_context):
         counts = HedTagCounts(new_context['name'], total_events=len(new_context['df']))
-        input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=new_context['sidecar'])
-        definitions = input_data.get_definitions().gathered_defs
+        sidecar = new_context['sidecar']
+        if sidecar and not isinstance(sidecar, Sidecar):
+            sidecar = Sidecar(sidecar, hed_schema=new_context['schema'])
+        input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=sidecar)
+        # definitions = input_data.get_definitions().gathered_defs
         for objs in input_data.iter_dataframe(hed_ops=[new_context['schema']], return_string_only=False,
-                                              expand_defs=False, remove_definitions=True):
+                                              expand_defs=True, remove_definitions=True):
             counts.update_event_counts(objs['HED'], new_context['name'])
         self.summary_dict[new_context["name"]] = counts
 

hed/tools/remodeling/operations/summarize_hed_type_op.py

@@ -1,6 +1,7 @@
 """ Summarize a HED type tag in a collection of tabular files. """
 
 from hed.models.tabular_input import TabularInput
+from hed.models.sidecar import Sidecar
 from hed.tools.analysis.analysis_util import get_assembled_strings
 from hed.tools.analysis.hed_type_values import HedTypeValues
 from hed.tools.analysis.hed_type_counts import HedTypeCounts
@@ -87,7 +88,10 @@ def __init__(self, sum_op):
         self.type_tag = sum_op.type_tag
 
     def update_context(self, new_context):
-        input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=new_context['sidecar'])
+        sidecar = new_context['sidecar']
+        if sidecar and not isinstance(sidecar, Sidecar):
+            sidecar = Sidecar(sidecar, hed_schema=new_context['schema'])
+        input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=sidecar)
         hed_strings = get_assembled_strings(input_data, hed_schema=new_context['schema'], expand_defs=False)
         definitions = input_data.get_definitions().gathered_defs
         context_manager = HedContextManager(hed_strings, new_context['schema'])

hed/tools/remodeling/operations/summarize_hed_validation_op.py

@@ -111,7 +111,8 @@ def update_context(self, new_context):
         filtered_issues = []
         if sidecar:
             if not isinstance(sidecar, Sidecar):
-                sidecar = Sidecar(files=new_context['sidecar'], name=os.path.basename(sidecar))
+                sidecar = Sidecar(files=new_context['sidecar'], name=os.path.basename(sidecar),
+                                  hed_schema=new_context['schema'])
             results["sidecar_issues"][sidecar.name] = []
             sidecar_issues = sidecar.validate_entries(validator, check_for_warnings=self.check_for_warnings)
             filtered_issues = ErrorHandler.filter_issues_by_severity(sidecar_issues, ErrorSeverity.ERROR)

tests/tools/analysis/test_analysis_util_assemble_hed.py

@@ -20,10 +20,11 @@ def setUpClass(cls):
         events_path = os.path.realpath(os.path.join(bids_root_path,
                                                     'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
 
-        cls.hed_schema = hedschema.load_schema(schema_path)
-        sidecar1 = Sidecar(json_path, name='face_sub1_json')
+        hed_schema = hedschema.load_schema(schema_path)
+        cls.hed_schema = hed_schema
+        sidecar1 = Sidecar(json_path, name='face_sub1_json', hed_schema=hed_schema)
         cls.sidecar_path = sidecar1
-        cls.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events")
+        cls.input_data = TabularInput(events_path, hed_schema=hed_schema, sidecar=sidecar1, name="face_sub1_events")
         cls.input_data_no_sidecar = TabularInput(events_path, name="face_sub1_events_no_sidecar")
 
     def test_assemble_hed_included_no_expand(self):

tests/tools/analysis/test_analysis_util_get_assembled_strings.py

@@ -26,7 +26,8 @@ def setUpClass(cls):
         # cls.input_data_no_sidecar = TabularInput(events_path, name="face_sub1_events_no_sidecar")
 
     def setUp(self):
-        self.input_data = TabularInput(self.events_path, sidecar=self.json_path, name="face_sub1_events")
+        self.input_data = TabularInput(self.events_path, hed_schema=self.hed_schema, 
+                                       sidecar=self.json_path, name="face_sub1_events")
 
     def test_get_assembled_strings_no_schema_no_def_expand(self):
         hed_list1 = get_assembled_strings(self.input_data, expand_defs=False)

tests/tools/analysis/test_hed_context_manager.py

@@ -37,7 +37,7 @@ def setUpClass(cls):
                                                     'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
         sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json'))
         sidecar1 = Sidecar(sidecar_path, name='face_sub1_json')
-        cls.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events")
+        cls.input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events")
         cls.schema = schema
 
     # def test_onset_group(self):

tests/tools/analysis/test_hed_tag_counts.py

@@ -21,9 +21,10 @@ def setUpClass(cls):
         events_path = os.path.realpath(os.path.join(bids_root_path,
                                                     'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
 
-        cls.hed_schema = hedschema.load_schema(schema_path)
+        schema = hedschema.load_schema(schema_path)
+        cls.hed_schema = schema
         sidecar1 = Sidecar(json_path, name='face_sub1_json')
-        input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events")
+        input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events")
         input_df, def_dict = assemble_hed(input_data, expand_defs=False)
         cls.input_df = input_df
         cls.def_dict = def_dict

tests/tools/analysis/test_hed_type_counts.py

@@ -19,8 +19,8 @@ def setUpClass(cls):
         events_path = os.path.realpath(os.path.join(bids_root_path,
                                        'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
         sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json'))
-        sidecar1 = Sidecar(sidecar_path, name='face_sub1_json')
-        input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events")
+        sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json')
+        input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events")
         hed_strings1 = get_assembled_strings(input_data, hed_schema=schema, expand_defs=False)
         definitions1 = input_data.get_definitions(as_strings=False).gathered_defs
         cls.var_type1 = HedTypeValues(HedContextManager(hed_strings1, schema), definitions1, 'run-01',

tests/tools/analysis/test_hed_type_definitions.py

@@ -42,8 +42,8 @@ def setUpClass(cls):
         events_path = os.path.realpath(os.path.join(bids_root_path,
                                                     'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
         sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json'))
-        sidecar1 = Sidecar(sidecar_path, name='face_sub1_json')
-        cls.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events")
+        sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json')
+        cls.input_data = TabularInput(events_path, hed_schema=schema, sidecar=sidecar1, name="face_sub1_events")
         cls.schema = schema
 
     def test_constructor(self):

tests/tools/analysis/test_hed_type_factors.py

@@ -57,8 +57,8 @@ def setUpClass(cls):
         events_path = os.path.realpath(os.path.join(bids_root_path,
                                                     'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
         sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json'))
-        sidecar1 = Sidecar(sidecar_path, name='face_sub1_json')
-        cls.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events")
+        sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json')
+        cls.input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events")
         cls.schema = schema
 
     def test_with_mixed(self):

tests/tools/analysis/test_hed_type_manager.py

@@ -18,8 +18,8 @@ def setUp(self):
         events_path = os.path.realpath(os.path.join(bids_root_path,
                                        'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
         sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json'))
-        sidecar1 = Sidecar(sidecar_path, name='face_sub1_json')
-        self.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events")
+        sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json')
+        self.input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events")
         self.hed_strings = get_assembled_strings(self.input_data, hed_schema=schema, expand_defs=False)
         self.hed_schema = schema
         self.definitions = self.input_data.get_definitions()