diff --git a/hed/models/df_util.py b/hed/models/df_util.py index 66b5c75be..f9fa19dcc 100644 --- a/hed/models/df_util.py +++ b/hed/models/df_util.py @@ -26,7 +26,8 @@ def get_assembled(tabular_file, sidecar, hed_schema, extra_def_dicts=None, join_ expand_defs: bool Expand any def tags found Returns: - A list of HedStrings, or a list of lists of HedStrings + tuple: A list of HedStrings, or a list of lists of HedStrings, DefinitionDict + """ if isinstance(sidecar, str): sidecar = Sidecar(sidecar) diff --git a/hed/models/sidecar.py b/hed/models/sidecar.py index 8b808c6d1..280eba77d 100644 --- a/hed/models/sidecar.py +++ b/hed/models/sidecar.py @@ -156,9 +156,10 @@ def validate(self, hed_schema, extra_def_dicts=None, name=None, error_handler=No Parameters: hed_schema (HedSchema): Input data to be validated. - extra_def_dicts(list or DefinitionDict): extra def dicts in addition to sidecar - name(str): The name to report this sidecar as - error_handler (ErrorHandler): Error context to use. Creates a new one if None + extra_def_dicts(list or DefinitionDict): Extra def dicts in addition to sidecar. + name(str): The name to report this sidecar as. + error_handler (ErrorHandler): Error context to use. Creates a new one if None. + Returns: issues (list of dict): A list of issues associated with each level in the HED string. """ diff --git a/hed/tools/__init__.py b/hed/tools/__init__.py index 8b1f6fd90..fd1dfbbce 100644 --- a/hed/tools/__init__.py +++ b/hed/tools/__init__.py @@ -47,7 +47,8 @@ from .analysis.annotation_util import \ check_df_columns, extract_tags, generate_sidecar_entry, hed_to_df, df_to_hed, merge_hed_dict from .analysis import analysis_util -from .analysis.analysis_util import assemble_hed, search_tabular, get_assembled_strings +from .analysis.analysis_util import assemble_hed +# from .analysis.analysis_util import search_tabular, get_assembled_strings from .remodeling.cli import run_remodel from .remodeling.cli import run_remodel_backup diff --git a/hed/tools/analysis/analysis_util.py b/hed/tools/analysis/analysis_util.py index fcfd5284c..a4c57c9f6 100644 --- a/hed/tools/analysis/analysis_util.py +++ b/hed/tools/analysis/analysis_util.py @@ -2,7 +2,6 @@ import pandas as pd from hed.models.tabular_input import TabularInput -from hed.models.expression_parser import QueryParser from hed.tools.util.data_util import separate_values from hed.models.hed_tag import HedTag from hed.models.hed_group import HedGroup @@ -45,57 +44,57 @@ def assemble_hed(data_input, sidecar, schema, columns_included=None, expand_defs return df, definitions -def get_assembled_strings(table, hed_schema=None, expand_defs=False): - """ Return HED string objects for a tabular file. - - Parameters: - table (TabularInput): The input file to be searched. - hed_schema (HedSchema or HedschemaGroup): If provided the HedStrings are converted to canonical form. - expand_defs (bool): If True, definitions are expanded when the events are assembled. - - Returns: - list: A list of HedString or HedStringGroup objects. - - """ - hed_list = list(table.iter_dataframe(hed_ops=[hed_schema], return_string_only=True, - expand_defs=expand_defs, remove_definitions=True)) - return hed_list - - -def search_tabular(data_input, hed_schema, query, columns_included=None): - """ Return a dataframe with results of query. - - Parameters: - data_input (TabularInput): The tabular input file (e.g., events) to be searched. - hed_schema (HedSchema or HedSchemaGroup): The schema(s) under which to make the query. - query (str or list): The str query or list of string queries to make. - columns_included (list or None): List of names of columns to include - - Returns: - DataFrame or None: A DataFrame with the results of the query or None if no events satisfied the query. - - """ - - eligible_columns, missing_columns = separate_values(list(data_input.dataframe.columns), columns_included) - hed_list = get_assembled_strings(data_input, hed_schema=hed_schema, expand_defs=True) - expression = QueryParser(query) - hed_tags = [] - row_numbers = [] - for index, next_item in enumerate(hed_list): - match = expression.search(next_item) - if not match: - continue - hed_tags.append(next_item) - row_numbers.append(index) - - if not row_numbers: - df = None - elif not eligible_columns: - df = pd.DataFrame({'row_number': row_numbers, 'HED_assembled': hed_tags}) - else: - df = data_input.dataframe.iloc[row_numbers][eligible_columns].reset_index() - df.rename(columns={'index': 'row_number'}) - return df +# def get_assembled_strings(table, hed_schema=None, expand_defs=False): +# """ Return HED string objects for a tabular file. +# +# Parameters: +# table (TabularInput): The input file to be searched. +# hed_schema (HedSchema or HedschemaGroup): If provided the HedStrings are converted to canonical form. +# expand_defs (bool): If True, definitions are expanded when the events are assembled. +# +# Returns: +# list: A list of HedString or HedStringGroup objects. +# +# """ +# hed_list = list(table.iter_dataframe(hed_ops=[hed_schema], return_string_only=True, +# expand_defs=expand_defs, remove_definitions=True)) +# return hed_list +# + +# def search_tabular(data_input, hed_schema, query, columns_included=None): +# """ Return a dataframe with results of query. +# +# Parameters: +# data_input (TabularInput): The tabular input file (e.g., events) to be searched. +# hed_schema (HedSchema or HedSchemaGroup): The schema(s) under which to make the query. +# query (str or list): The str query or list of string queries to make. +# columns_included (list or None): List of names of columns to include +# +# Returns: +# DataFrame or None: A DataFrame with the results of the query or None if no events satisfied the query. +# +# """ +# +# eligible_columns, missing_columns = separate_values(list(data_input.dataframe.columns), columns_included) +# hed_list = get_assembled_strings(data_input, hed_schema=hed_schema, expand_defs=True) +# expression = QueryParser(query) +# hed_tags = [] +# row_numbers = [] +# for index, next_item in enumerate(hed_list): +# match = expression.search(next_item) +# if not match: +# continue +# hed_tags.append(next_item) +# row_numbers.append(index) +# +# if not row_numbers: +# df = None +# elif not eligible_columns: +# df = pd.DataFrame({'row_number': row_numbers, 'HED_assembled': hed_tags}) +# else: +# df = data_input.dataframe.iloc[row_numbers][eligible_columns].reset_index() +# df.rename(columns={'index': 'row_number'}) +# return df # def remove_defs(hed_strings): diff --git a/hed/tools/analysis/event_manager.py b/hed/tools/analysis/event_manager.py index 2d6da7adc..f8bf5e5f5 100644 --- a/hed/tools/analysis/event_manager.py +++ b/hed/tools/analysis/event_manager.py @@ -3,25 +3,26 @@ from hed.schema import HedSchema, HedSchemaGroup from hed.tools.analysis.temporal_event import TemporalEvent from hed.models.model_constants import DefTagNames +from hed.models.df_util import get_assembled class EventManager: - def __init__(self, data, hed_schema): + def __init__(self, data, schema): """ Create an event manager for an events file. Parameters: data (TabularInput): A tabular input file. - hed_schema (HedSchema): A HED schema + schema (HedSchema): A HED schema Raises: HedFileError: if there are any unmatched offsets. """ - if not isinstance(hed_schema, HedSchema) and not isinstance(hed_schema, HedSchemaGroup): + if not isinstance(schema, HedSchema) and not isinstance(schema, HedSchemaGroup): raise ValueError("ContextRequiresSchema", f"Context manager must have a valid HedSchema of HedSchemaGroup") - self.hed_schema = hed_schema + self.schema = schema self.data = data self.event_list = [[] for _ in range(len(self.data.dataframe))] self.hed_strings = [None for _ in range(len(self.data.dataframe))] @@ -56,10 +57,10 @@ def _create_event_list(self): onset_dict = {} event_index = 0 - for hed in self.data.iter_dataframe(hed_ops=[self.hed_schema], return_string_only=True, - expand_defs=False, remove_definitions=True): + self.hed_strings, definitions = get_assembled(self.data, self.data._sidecar, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + for hed in self.hed_strings: # to_remove = [] # tag_tuples = hed.find_tags(['Onset'], recursive=False, include_groups=1) - self.hed_strings[event_index] = hed group_tuples = hed.find_top_level_tags(anchor_tags={DefTagNames.ONSET_KEY, DefTagNames.OFFSET_KEY}, include_groups=2) for tup in group_tuples: diff --git a/hed/tools/analysis/hed_context_manager.py b/hed/tools/analysis/hed_context_manager.py index 06a02dc82..5c565a9a4 100644 --- a/hed/tools/analysis/hed_context_manager.py +++ b/hed/tools/analysis/hed_context_manager.py @@ -5,6 +5,7 @@ from hed.schema import HedSchema, HedSchemaGroup from hed.tools.analysis.analysis_util import hed_to_str +#TODO: [Refactor] clean up distinction between hed as strings versus objects -- maybe replace by event manager. class OnsetGroup: def __init__(self, name, contents, start_index, end_index=None): @@ -23,7 +24,8 @@ def __init__(self, hed_strings, hed_schema): """ Create an context manager for an events file. Parameters: - hed_strings (list): A list of hed_strings to be managed. + hed_strings (list): A list of HedString objects to be managed. + hed_schema (HedSchema): A HedSchema Raises: HedFileError: if there are any unmatched offsets. @@ -46,6 +48,12 @@ def __init__(self, hed_strings, hed_schema): self._create_onset_list() self._set_event_contexts() + # def _extract_hed_objs(self, assembled): + # hed_objs = [None for _ in range(len(assembled))] + # for index, value in assembled["HED_assembled"].items(): + # hed_objs[index] = HedString(value, hed_schema=self.hed_schema) + # return hed_objs + def iter_context(self): """ Iterate rows of context. diff --git a/hed/tools/remodeling/operations/factor_hed_tags_op.py b/hed/tools/remodeling/operations/factor_hed_tags_op.py index 41d3f805a..aa02224b9 100644 --- a/hed/tools/remodeling/operations/factor_hed_tags_op.py +++ b/hed/tools/remodeling/operations/factor_hed_tags_op.py @@ -7,7 +7,7 @@ from hed.models.tabular_input import TabularInput from hed.models.sidecar import Sidecar from hed.models.expression_parser import QueryParser -from hed.tools.analysis.analysis_util import get_assembled_strings +from hed.models.df_util import get_assembled class FactorHedTagsOp(BaseOp): @@ -101,16 +101,16 @@ def do_op(self, dispatcher, df, name, sidecar=None): """ if sidecar and not isinstance(sidecar, Sidecar): - sidecar = Sidecar(sidecar, hed_schema=dispatcher.hed_schema) - input_data = TabularInput(df, hed_schema=dispatcher.hed_schema, sidecar=sidecar) + sidecar = Sidecar(sidecar) + input_data = TabularInput(df.copy(), sidecar=sidecar, name=name) column_names = list(df.columns) - for name in self.query_names: - if name in column_names: + for query_name in self.query_names: + if query_name in column_names: raise ValueError("QueryNameAlreadyColumn", - f"Query [{name}]: is already a column name of the data frame") - df = input_data.dataframe.copy() - df_list = [df] - hed_strings = get_assembled_strings(input_data, hed_schema=dispatcher.hed_schema, expand_defs=True) + f"Query [{query_name}]: is already a column name of the data frame") + df_list = [input_data.dataframe] + hed_strings, _ = get_assembled(input_data, sidecar, dispatcher.hed_schema, extra_def_dicts=None, + join_columns=True, shrink_defs=False, expand_defs=True) df_factors = pd.DataFrame(0, index=range(len(hed_strings)), columns=self.query_names) for parse_ind, parser in enumerate(self.expression_parsers): for index, next_item in enumerate(hed_strings): diff --git a/hed/tools/remodeling/operations/factor_hed_type_op.py b/hed/tools/remodeling/operations/factor_hed_type_op.py index e4a43c181..668886c88 100644 --- a/hed/tools/remodeling/operations/factor_hed_type_op.py +++ b/hed/tools/remodeling/operations/factor_hed_type_op.py @@ -5,7 +5,7 @@ from hed.tools.remodeling.operations.base_op import BaseOp from hed.models.tabular_input import TabularInput from hed.models.sidecar import Sidecar -from hed.tools.analysis.analysis_util import get_assembled_strings +from hed.models.df_util import get_assembled from hed.tools.analysis.hed_type_manager import HedTypeManager # TODO: restricted factor values are not implemented yet. @@ -69,13 +69,13 @@ def do_op(self, dispatcher, df, name, sidecar=None): """ if sidecar and not isinstance(sidecar, Sidecar): - sidecar = Sidecar(sidecar, hed_schema=dispatcher.hed_schema) - input_data = TabularInput(df, hed_schema=dispatcher.hed_schema, sidecar=sidecar) - df = input_data.dataframe.copy() - df_list = [df] - hed_strings = get_assembled_strings(input_data, hed_schema=dispatcher.hed_schema, expand_defs=False) + sidecar = Sidecar(sidecar) + input_data = TabularInput(df, sidecar=sidecar, name=name) + df_list = [input_data.dataframe.copy()] + hed_strings, definitions = get_assembled(input_data, sidecar, dispatcher.hed_schema, + extra_def_dicts=None, join_columns=True, + shrink_defs=False, expand_defs=True) - definitions = input_data.get_definitions() var_manager = HedTypeManager(hed_strings, dispatcher.hed_schema, definitions) var_manager.add_type_variable(self.type_tag.lower()) diff --git a/hed/tools/remodeling/operations/summarize_hed_tags_op.py b/hed/tools/remodeling/operations/summarize_hed_tags_op.py index 09f7e3a48..a8d220df8 100644 --- a/hed/tools/remodeling/operations/summarize_hed_tags_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_tags_op.py @@ -5,6 +5,7 @@ from hed.tools.analysis.hed_tag_counts import HedTagCounts from hed.tools.remodeling.operations.base_op import BaseOp from hed.tools.remodeling.operations.base_context import BaseContext +from hed.models.df_util import get_assembled class SummarizeHedTagsOp(BaseOp): @@ -97,12 +98,14 @@ def update_context(self, new_context): counts = HedTagCounts(new_context['name'], total_events=len(new_context['df'])) sidecar = new_context['sidecar'] if sidecar and not isinstance(sidecar, Sidecar): - sidecar = Sidecar(sidecar, hed_schema=new_context['schema']) - input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=sidecar) + sidecar = Sidecar(sidecar) + input_data = TabularInput(new_context['df'], sidecar=sidecar, name=new_context['name']) + hed_strings, definitions = get_assembled(input_data, sidecar, new_context['schema'], + extra_def_dicts=None, join_columns=True, + shrink_defs=False, expand_defs=True) # definitions = input_data.get_definitions().gathered_defs - for objs in input_data.iter_dataframe(hed_ops=[new_context['schema']], return_string_only=False, - expand_defs=True, remove_definitions=True): - counts.update_event_counts(objs['HED'], new_context['name']) + for hed in hed_strings: + counts.update_event_counts(hed, new_context['name']) self.summary_dict[new_context["name"]] = counts def _get_summary_details(self, merge_counts): diff --git a/hed/tools/remodeling/operations/summarize_hed_type_op.py b/hed/tools/remodeling/operations/summarize_hed_type_op.py index 2c7ab7c64..0e2664698 100644 --- a/hed/tools/remodeling/operations/summarize_hed_type_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_type_op.py @@ -2,7 +2,7 @@ from hed.models.tabular_input import TabularInput from hed.models.sidecar import Sidecar -from hed.tools.analysis.analysis_util import get_assembled_strings +from hed.models.df_util import get_assembled from hed.tools.analysis.hed_type_values import HedTypeValues from hed.tools.analysis.hed_type_counts import HedTypeCounts from hed.tools.analysis.hed_context_manager import HedContextManager @@ -90,10 +90,11 @@ def __init__(self, sum_op): def update_context(self, new_context): sidecar = new_context['sidecar'] if sidecar and not isinstance(sidecar, Sidecar): - sidecar = Sidecar(sidecar, hed_schema=new_context['schema']) - input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=sidecar) - hed_strings = get_assembled_strings(input_data, hed_schema=new_context['schema'], expand_defs=False) - definitions = input_data.get_definitions().gathered_defs + sidecar = Sidecar(sidecar) + input_data = TabularInput(new_context['df'], sidecar=sidecar, name=new_context['name']) + hed_strings, definitions = get_assembled(input_data, sidecar, new_context['schema'], + extra_def_dicts=None, join_columns=True, + shrink_defs=False, expand_defs=True) context_manager = HedContextManager(hed_strings, new_context['schema']) type_values = HedTypeValues(context_manager, definitions, new_context['name'], type_tag=self.type_tag) diff --git a/hed/tools/remodeling/operations/summarize_hed_validation_op.py b/hed/tools/remodeling/operations/summarize_hed_validation_op.py index 771b49e5c..d1bd8f53e 100644 --- a/hed/tools/remodeling/operations/summarize_hed_validation_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_validation_op.py @@ -102,7 +102,6 @@ def _get_result_string(self, name, result, indent=BaseContext.DISPLAY_INDENT): return "\n".join(sum_list) def update_context(self, new_context): - validator = HedValidator(hed_schema=new_context['schema']) results = self.get_empty_results() results["total_event_files"] = 1 results["event_issues"][new_context["name"]] = [] @@ -111,10 +110,9 @@ def update_context(self, new_context): filtered_issues = [] if sidecar: if not isinstance(sidecar, Sidecar): - sidecar = Sidecar(files=new_context['sidecar'], name=os.path.basename(sidecar), - hed_schema=new_context['schema']) + sidecar = Sidecar(files=new_context['sidecar'], name=os.path.basename(sidecar)) results["sidecar_issues"][sidecar.name] = [] - sidecar_issues = sidecar.validate_entries(validator, check_for_warnings=self.check_for_warnings) + sidecar_issues = sidecar.validate(new_context['schema']) filtered_issues = ErrorHandler.filter_issues_by_severity(sidecar_issues, ErrorSeverity.ERROR) if not self.check_for_warnings: sidecar_issues = filtered_issues @@ -123,8 +121,8 @@ def update_context(self, new_context): results['total_sidecar_files'] = 1 if not filtered_issues: results['validation_completed'] = True - input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=sidecar) - issues = input_data.validate_file(validator, check_for_warnings=self.check_for_warnings) + input_data = TabularInput(new_context['df'], sidecar=sidecar) + issues = input_data.validate(new_context['schema']) if not self.check_for_warnings: issues = ErrorHandler.filter_issues_by_severity(issues, ErrorSeverity.ERROR) results['event_issues'][new_context["name"]] = issues diff --git a/tests/tools/analysis/test_analysis_util_assemble_hed.py b/tests/tools/analysis/test_analysis_util_assemble_hed.py index 9c37b8620..318c3aa54 100644 --- a/tests/tools/analysis/test_analysis_util_assemble_hed.py +++ b/tests/tools/analysis/test_analysis_util_assemble_hed.py @@ -2,8 +2,10 @@ import unittest from pandas import DataFrame from hed import schema as hedschema -from hed.models import Sidecar, TabularInput -from hed.tools import assemble_hed, search_tabular +from hed.models import Sidecar, TabularInput, DefinitionDict +from hed.tools.analysis.analysis_util import assemble_hed + + # noinspection PyBroadException @@ -20,8 +22,8 @@ def setUpClass(cls): events_path = os.path.realpath(os.path.join(bids_root_path, 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv')) - hed_schema = hedschema.load_schema(schema_path) - cls.hed_schema = hed_schema + schema = hedschema.load_schema(schema_path) + cls.schema = schema sidecar1 = Sidecar(json_path, name='face_sub1_json') cls.sidecar_path = sidecar1 cls.sidecar1 = sidecar1 @@ -29,8 +31,8 @@ def setUpClass(cls): cls.input_data_no_sidecar = TabularInput(events_path, name="face_sub1_events_no_sidecar") def test_assemble_hed_included_no_expand(self): - df1, dict1 = assemble_hed(self.input_data, self.sidecar1, self.hed_schema, - columns_included=["onset", "duration", "event_type"], expand_defs=False) + df1, dict1 = assemble_hed(self.input_data, self.sidecar1, self.schema, expand_defs=False, + columns_included=["onset", "duration", "event_type"]) self.assertIsInstance(df1, DataFrame, "hed_assemble should return a dataframe when columns are included") columns1 = list(df1.columns) self.assertEqual(len(columns1), 4, @@ -43,28 +45,29 @@ def test_assemble_hed_included_no_expand(self): self.assertEqual(len(dict1.defs), 17, "hed_assemble definition dictionary has the right number of elements.") def test_assemble_hed_included_expand(self): - df2, dict2 = assemble_hed(self.input_data, self.sidecar1, self.hed_schema, - columns_included=["onset", "duration", "event_type"], expand_defs=True) + df2, dict2 = assemble_hed(self.input_data, self.sidecar1, self.schema, expand_defs=True, + columns_included=["onset", "duration", "event_type"]) first_str2 = df2.iloc[0]['HED_assembled'] self.assertEqual(first_str2.find('Def/'), -1, "assemble_hed with def expand has no Def tag") self.assertNotEqual(first_str2.find('Def-expand/'), -1, "assemble_hed with def expand has Def-expand tags") def test_assemble_hed_included_no_expand_bad_column(self): - df3, dict3 = assemble_hed(self.input_data, - columns_included=["onset", "baloney", "duration", "event_type"], expand_defs=False) + df3, dict3 = assemble_hed(self.input_data, self.sidecar1, self.schema, expand_defs=True, + columns_included=["onset", "baloney", "duration", "event_type"]) columns3 = list(df3.columns) self.assertEqual(len(columns3), 4, "assemble_hed should return the correct number of columns when bad columns are included ") def test_assemble_hed_included_expand_bad_column(self): - df3, dict3 = assemble_hed(self.input_data, - columns_included=["onset", "baloney", "duration", "event_type"], expand_defs=True) + df3, dict3 = assemble_hed(self.input_data, self.sidecar1, self.schema, expand_defs=True, + columns_included=["onset", "baloney", "duration", "event_type"]) columns3 = list(df3.columns) self.assertEqual(len(columns3), 4, "assemble_hed should return the correct number of columns when bad columns are included ") def test_assemble_hed_no_included_no_expand(self): - df1, dict1 = assemble_hed(self.input_data, columns_included=None, expand_defs=False) + df1, dict1 = assemble_hed(self.input_data, self.sidecar1, self.schema, + columns_included=None, expand_defs=False) self.assertIsInstance(df1, DataFrame, "hed_assemble returns a dataframe when no columns are included") columns1 = list(df1.columns) self.assertEqual(len(columns1), 1, @@ -73,17 +76,18 @@ def test_assemble_hed_no_included_no_expand(self): self.assertNotEqual(first_str1.find('Def/'), -1, "assemble_hed with no def expand has Def tags") self.assertEqual(first_str1.find('Def-expand'), -1, "assemble_hed with no def expand does not have Def-expand tags") - self.assertIsInstance(dict1, dict, "hed_assemble returns a dictionary of definitions") - self.assertEqual(len(dict1), 17, "hed_assemble definition dictionary has the right number of elements.") + self.assertIsInstance(dict1, DefinitionDict, "hed_assemble returns a dictionary of definitions") + self.assertEqual(len(dict1.defs), 17, "hed_assemble definition dictionary has the right number of elements.") def test_assemble_hed_no_included_expand(self): - df2, dict2 = assemble_hed(self.input_data, columns_included=None, expand_defs=True) + df2, dict2 = assemble_hed(self.input_data, self.sidecar1, self.schema, + columns_included=None, expand_defs=True) first_str2 = df2.iloc[0]['HED_assembled'] self.assertEqual(first_str2.find('Def/'), -1, "assemble_hed with def expand has no Def tag") self.assertNotEqual(first_str2.find('Def-expand/'), -1, "assemble_hed with def expand has Def-expand tags") def test_assemble_hed_bad_column_no_expand(self): - df3, dict3 = assemble_hed(self.input_data, + df3, dict3 = assemble_hed(self.input_data, self.sidecar1, self.schema, columns_included=["onset", "baloney", "duration", "event_type"], expand_defs=False) columns3 = list(df3.columns) self.assertEqual(len(columns3), 4, @@ -92,27 +96,27 @@ def test_assemble_hed_bad_column_no_expand(self): self.assertNotEqual(first_str2.find('Def/'), -1, "assemble_hed with def expand has no Def tag") self.assertEqual(first_str2.find('Def-expand/'), -1, "assemble_hed with def expand has Def-expand tags") - def test_search_tabular(self): - query1 = "sensory-event" - df1 = search_tabular(self.input_data, self.hed_schema, query1, columns_included=None) - self.assertIsInstance(df1, DataFrame, "search_tabular returns a dataframe when the query is satisfied.") - self.assertEqual(len(df1.columns), 2, "search_tabular has the right number of columns when query okay") - self.assertEqual(len(df1.index), 155, "search_tabular has right number of rows when query okay") - query2 = 'data-feature' - df2 = search_tabular(self.input_data, self.hed_schema, query2, columns_included=None) - self.assertFalse(df2, "search_tabular returns None when query is not satisfied.") - - query3 = "sensory-event" - df3 = search_tabular(self.input_data, self.hed_schema, query3, columns_included=['event_type', 'rep_status']) - self.assertIsInstance(df3, DataFrame, "search_tabular returns a DataFrame when extra columns") - self.assertEqual(len(df3.columns), 3, "search_tabular returns right number of columns when extra columns") - self.assertEqual(len(df3.index), 155, "search_tabular has right number of rows when query okay") - - df4 = search_tabular(self.input_data, self.hed_schema, query3, - columns_included=['onset', 'event_type', 'rep_status']) - self.assertIsInstance(df4, DataFrame, "search_tabular returns a DataFrame when extra columns") - self.assertEqual(len(df4.columns), 4, "search_tabular returns right number of columns when extra columns") - self.assertEqual(len(df4.index), 155, "search_tabular has right number of rows when query okay") + # def test_search_tabular(self): + # query1 = "sensory-event" + # df1 = search_tabular(self.input_data, self.schema, query1, columns_included=None) + # self.assertIsInstance(df1, DataFrame, "search_tabular returns a dataframe when the query is satisfied.") + # self.assertEqual(len(df1.columns), 2, "search_tabular has the right number of columns when query okay") + # self.assertEqual(len(df1.index), 155, "search_tabular has right number of rows when query okay") + # query2 = 'data-feature' + # df2 = search_tabular(self.input_data, self.hed_schema, query2, columns_included=None) + # self.assertFalse(df2, "search_tabular returns None when query is not satisfied.") + # + # query3 = "sensory-event" + # df3 = search_tabular(self.input_data, self.hed_schema, query3, columns_included=['event_type', 'rep_status']) + # self.assertIsInstance(df3, DataFrame, "search_tabular returns a DataFrame when extra columns") + # self.assertEqual(len(df3.columns), 3, "search_tabular returns right number of columns when extra columns") + # self.assertEqual(len(df3.index), 155, "search_tabular has right number of rows when query okay") + # + # df4 = search_tabular(self.input_data, self.hed_schema, query3, + # columns_included=['onset', 'event_type', 'rep_status']) + # self.assertIsInstance(df4, DataFrame, "search_tabular returns a DataFrame when extra columns") + # self.assertEqual(len(df4.columns), 4, "search_tabular returns right number of columns when extra columns") + # self.assertEqual(len(df4.index), 155, "search_tabular has right number of rows when query okay") if __name__ == '__main__': diff --git a/tests/tools/analysis/test_analysis_util_get_assembled_strings.py b/tests/tools/analysis/test_analysis_util_get_assembled_strings.py index 143db3305..036b4c938 100644 --- a/tests/tools/analysis/test_analysis_util_get_assembled_strings.py +++ b/tests/tools/analysis/test_analysis_util_get_assembled_strings.py @@ -3,7 +3,7 @@ from hed import schema as hedschema from hed.models.hed_string import HedString from hed.models.tabular_input import TabularInput -from hed.tools.analysis.analysis_util import get_assembled_strings +# from hed.tools.analysis.analysis_util import get_assembled_strings # noinspection PyBroadException @@ -26,90 +26,89 @@ def setUpClass(cls): # cls.input_data_no_sidecar = TabularInput(events_path, name="face_sub1_events_no_sidecar") def setUp(self): - self.input_data = TabularInput(self.events_path, hed_schema=self.hed_schema, - sidecar=self.json_path, name="face_sub1_events") + self.input_data = TabularInput(self.events_path, sidecar=self.json_path, name="face_sub1_events") - def test_get_assembled_strings_no_schema_no_def_expand(self): - hed_list1 = get_assembled_strings(self.input_data, expand_defs=False) - self.assertIsInstance(hed_list1, list, "get_assembled_groups should return a list when expand defs is False") - self.assertIsInstance(hed_list1[0], HedString) - hed_strings1 = [str(hed) for hed in hed_list1] - self.assertIsInstance(hed_strings1[0], str, "get_assembled_strings can be converted.") - self.assertIsInstance(hed_strings1, list) - hed_strings_joined1 = ",".join(hed_strings1) - self.assertEqual(hed_strings_joined1.find("Def-expand/"), -1, - "get_assembled_strings should not have Def-expand when expand_defs is False") - self.assertNotEqual(hed_strings_joined1.find("Def/"), -1, - "get_assembled_strings should have Def/ when expand_defs is False") - - def test_get_assembled_strings_no_schema_def_expand(self): - hed_list2 = get_assembled_strings(self.input_data, expand_defs=True) - self.assertIsInstance(hed_list2, list, "get_assembled_groups should return a list") - self.assertIsInstance(hed_list2[0], HedString) - hed_strings2 = [str(hed) for hed in hed_list2] - self.assertIsInstance(hed_strings2[0], str, "get_assembled_strings can be converted.") - self.assertIsInstance(hed_strings2, list, "get_assembled") - hed_strings_joined2 = ",".join(hed_strings2) - self.assertNotEqual(hed_strings_joined2.find("Def-expand/"), -1, - "get_assembled_strings should have Def-expand when expand_defs is True") - self.assertEqual(hed_strings_joined2.find("Def/"), -1, - "get_assembled_strings should not have Def/ when expand_defs is True") - - def test_get_assembled_strings_with_schema_no_def_expand(self): - hed_list1 = get_assembled_strings(self. input_data, hed_schema=self.hed_schema, expand_defs=False) - self.assertIsInstance(hed_list1, list, "get_assembled_strings returns a list when expand defs is False") - self.assertIsInstance(hed_list1[0], HedString) - hed_strings1 = [str(hed) for hed in hed_list1] - self.assertIsInstance(hed_strings1[0], str, "get_assembled_strings can be converted.") - self.assertIsInstance(hed_strings1, list) - hed_strings_joined1 = ",".join(hed_strings1) - self.assertEqual(hed_strings_joined1.find("Def-expand/"), -1, - "get_assembled_strings does not have Def-expand when expand_defs is False") - self.assertNotEqual(hed_strings_joined1.find("Def/"), -1, - "get_assembled_strings should have Def/ when expand_defs is False") - - def test_get_assembled_strings_with_schema_def_expand(self): - hed_list2 = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=True) - self.assertIsInstance(hed_list2, list, "get_assembled_groups should return a list") - self.assertIsInstance(hed_list2[0], HedString) - hed_strings2 = [str(hed) for hed in hed_list2] - self.assertIsInstance(hed_strings2[0], str, "get_assembled_strings can be converted.") - self.assertIsInstance(hed_strings2, list, "get_assembled") - hed_strings_joined2 = ",".join(hed_strings2) - self.assertNotEqual(hed_strings_joined2.find("Def-expand/"), -1, - "get_assembled_strings should have Def-expand when expand_defs is True") - self.assertEqual(hed_strings_joined2.find("Def/"), -1, - "get_assembled_strings should not have Def/ when expand_defs is True") - - def test_get_assembled_strings_no_sidecar_no_schema(self): - input_data = TabularInput(self.events_path, name="face_sub1_events") - hed_list1 = get_assembled_strings(input_data, expand_defs=False) - self.assertEqual(len(hed_list1), 200, - "get_assembled_strings should have right number of entries when no sidecar") - self.assertIsInstance(hed_list1[0], HedString, - "get_assembled_string should return an HedString when no sidecar") - self.assertFalse(hed_list1[0].children, "get_assembled_string returned HedString is empty when no sidecar") - hed_list2 = get_assembled_strings(input_data, expand_defs=True) - self.assertEqual(len(hed_list2), 200, - "get_assembled_strings should have right number of entries when no sidecar") - self.assertIsInstance(hed_list2[0], HedString, - "get_assembled_string should return an HedString when no sidecar") - self.assertFalse(hed_list2[0].children, "get_assembled_string returned HedString is empty when no sidecar") - - def test_get_assembled_strings_no_sidecar_schema(self): - input_data = TabularInput(self.events_path, hed_schema=self.hed_schema, name="face_sub1_events") - hed_list1 = get_assembled_strings(input_data, expand_defs=False) - self.assertEqual(len(hed_list1), 200, - "get_assembled_strings should have right number of entries when no sidecar") - self.assertIsInstance(hed_list1[0], HedString, - "get_assembled_string should return an HedString when no sidecar") - self.assertFalse(hed_list1[0].children, "get_assembled_string returned HedString is empty when no sidecar") - hed_list2 = get_assembled_strings(input_data, expand_defs=True) - self.assertEqual(len(hed_list2), 200, - "get_assembled_strings should have right number of entries when no sidecar") - self.assertIsInstance(hed_list2[0], HedString, - "get_assembled_string should return an HedString when no sidecar") - self.assertFalse(hed_list2[0].children, "get_assembled_string returned HedString is empty when no sidecar") + # def test_get_assembled_strings_no_schema_no_def_expand(self): + # hed_list1 = get_assembled_strings(self.input_data, expand_defs=False) + # self.assertIsInstance(hed_list1, list, "get_assembled_groups should return a list when expand defs is False") + # self.assertIsInstance(hed_list1[0], HedString) + # hed_strings1 = [str(hed) for hed in hed_list1] + # self.assertIsInstance(hed_strings1[0], str, "get_assembled_strings can be converted.") + # self.assertIsInstance(hed_strings1, list) + # hed_strings_joined1 = ",".join(hed_strings1) + # self.assertEqual(hed_strings_joined1.find("Def-expand/"), -1, + # "get_assembled_strings should not have Def-expand when expand_defs is False") + # self.assertNotEqual(hed_strings_joined1.find("Def/"), -1, + # "get_assembled_strings should have Def/ when expand_defs is False") + # + # def test_get_assembled_strings_no_schema_def_expand(self): + # hed_list2 = get_assembled_strings(self.input_data, self.hed_schema, expand_defs=True) + # self.assertIsInstance(hed_list2, list, "get_assembled_groups should return a list") + # self.assertIsInstance(hed_list2[0], HedString) + # hed_strings2 = [str(hed) for hed in hed_list2] + # self.assertIsInstance(hed_strings2[0], str, "get_assembled_strings can be converted.") + # self.assertIsInstance(hed_strings2, list, "get_assembled") + # hed_strings_joined2 = ",".join(hed_strings2) + # self.assertNotEqual(hed_strings_joined2.find("Def-expand/"), -1, + # "get_assembled_strings should have Def-expand when expand_defs is True") + # self.assertEqual(hed_strings_joined2.find("Def/"), -1, + # "get_assembled_strings should not have Def/ when expand_defs is True") + # + # def test_get_assembled_strings_with_schema_no_def_expand(self): + # hed_list1 = get_assembled_strings(self. input_data, hed_schema=self.hed_schema, expand_defs=False) + # self.assertIsInstance(hed_list1, list, "get_assembled_strings returns a list when expand defs is False") + # self.assertIsInstance(hed_list1[0], HedString) + # hed_strings1 = [str(hed) for hed in hed_list1] + # self.assertIsInstance(hed_strings1[0], str, "get_assembled_strings can be converted.") + # self.assertIsInstance(hed_strings1, list) + # hed_strings_joined1 = ",".join(hed_strings1) + # self.assertEqual(hed_strings_joined1.find("Def-expand/"), -1, + # "get_assembled_strings does not have Def-expand when expand_defs is False") + # self.assertNotEqual(hed_strings_joined1.find("Def/"), -1, + # "get_assembled_strings should have Def/ when expand_defs is False") + # + # def test_get_assembled_strings_with_schema_def_expand(self): + # hed_list2 = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=True) + # self.assertIsInstance(hed_list2, list, "get_assembled_groups should return a list") + # self.assertIsInstance(hed_list2[0], HedString) + # hed_strings2 = [str(hed) for hed in hed_list2] + # self.assertIsInstance(hed_strings2[0], str, "get_assembled_strings can be converted.") + # self.assertIsInstance(hed_strings2, list, "get_assembled") + # hed_strings_joined2 = ",".join(hed_strings2) + # self.assertNotEqual(hed_strings_joined2.find("Def-expand/"), -1, + # "get_assembled_strings should have Def-expand when expand_defs is True") + # self.assertEqual(hed_strings_joined2.find("Def/"), -1, + # "get_assembled_strings should not have Def/ when expand_defs is True") + # + # def test_get_assembled_strings_no_sidecar_no_schema(self): + # input_data = TabularInput(self.events_path, name="face_sub1_events") + # hed_list1 = get_assembled_strings(input_data, expand_defs=False) + # self.assertEqual(len(hed_list1), 200, + # "get_assembled_strings should have right number of entries when no sidecar") + # self.assertIsInstance(hed_list1[0], HedString, + # "get_assembled_string should return an HedString when no sidecar") + # self.assertFalse(hed_list1[0].children, "get_assembled_string returned HedString is empty when no sidecar") + # hed_list2 = get_assembled_strings(input_data, expand_defs=True) + # self.assertEqual(len(hed_list2), 200, + # "get_assembled_strings should have right number of entries when no sidecar") + # self.assertIsInstance(hed_list2[0], HedString, + # "get_assembled_string should return an HedString when no sidecar") + # self.assertFalse(hed_list2[0].children, "get_assembled_string returned HedString is empty when no sidecar") + # + # def test_get_assembled_strings_no_sidecar_schema(self): + # input_data = TabularInput(self.events_path, hed_schema=self.hed_schema, name="face_sub1_events") + # hed_list1 = get_assembled_strings(input_data, expand_defs=False) + # self.assertEqual(len(hed_list1), 200, + # "get_assembled_strings should have right number of entries when no sidecar") + # self.assertIsInstance(hed_list1[0], HedString, + # "get_assembled_string should return an HedString when no sidecar") + # self.assertFalse(hed_list1[0].children, "get_assembled_string returned HedString is empty when no sidecar") + # hed_list2 = get_assembled_strings(input_data, expand_defs=True) + # self.assertEqual(len(hed_list2), 200, + # "get_assembled_strings should have right number of entries when no sidecar") + # self.assertIsInstance(hed_list2[0], HedString, + # "get_assembled_string should return an HedString when no sidecar") + # self.assertFalse(hed_list2[0].children, "get_assembled_string returned HedString is empty when no sidecar") if __name__ == '__main__': diff --git a/tests/tools/analysis/test_annotation_util.py b/tests/tools/analysis/test_annotation_util.py index fcf2ce03a..f54dd1dc8 100644 --- a/tests/tools/analysis/test_annotation_util.py +++ b/tests/tools/analysis/test_annotation_util.py @@ -232,19 +232,19 @@ def test_hed_to_df_with_definitions(self): "hed_to_df should have right description when in parentheses") def test_hed_to_df_to_hed(self): - validator = HedValidator(self.hed_schema) + # validator = HedValidator(self.hed_schema) side1 = Sidecar(files=self.json_path, name="sidecar_face.json") - issues1 = side1.validate_entries(validator, check_for_warnings=True) + issues1 = side1.validate(self.hed_schema) self.assertFalse(issues1, "hed_to_df_to_hed is starting with a valid JSON sidecar") df1 = hed_to_df(self.sidecar_face) self.assertIsInstance(df1, DataFrame, "hed_to_df_to_hed starting sidecar can be converted to df") hed2 = df_to_hed(df1, description_tag=True) side2 = Sidecar(files=io.StringIO(json.dumps(hed2)), name='JSON_Sidecar2') - issues2 = side2.validate_entries(validator, check_for_warnings=True) + issues2 = side2.validate(self.hed_schema) self.assertFalse(issues2, "hed_to_df_to_hed is valid after conversion back and forth with description True") hed3 = df_to_hed(df1, description_tag=False) side3 = Sidecar(files=io.StringIO(json.dumps(hed3)), name='JSON_Sidecar2') - issues3 = side3.validate_entries(validator, check_for_warnings=True) + issues3 = side3.validate(self.hed_schema) self.assertFalse(issues3, "hed_to_df_to_hed is valid after conversion back and forth with description False") def test_merge_hed_dict_cat_col(self): diff --git a/tests/tools/analysis/test_event_manager.py b/tests/tools/analysis/test_event_manager.py index dd920256a..09eb17a50 100644 --- a/tests/tools/analysis/test_event_manager.py +++ b/tests/tools/analysis/test_event_manager.py @@ -1,13 +1,9 @@ import os import unittest -from hed.errors.exceptions import HedFileError -from hed.models.hed_group import HedGroup -from hed.models.hed_string import HedString + from hed.models.sidecar import Sidecar from hed.models.tabular_input import TabularInput from hed.schema.hed_schema_io import load_schema_version -from hed.tools.analysis.hed_context_manager import HedContextManager, OnsetGroup -from hed.tools.analysis.analysis_util import get_assembled_strings from hed.tools.analysis.event_manager import EventManager from hed.tools.analysis.temporal_event import TemporalEvent @@ -23,7 +19,7 @@ def setUpClass(cls): 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv')) sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json')) sidecar1 = Sidecar(sidecar_path, name='face_sub1_json') - cls.input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events") + cls.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events") cls.schema = schema def test_constructor(self): @@ -33,14 +29,13 @@ def test_constructor(self): for index, item in enumerate(manager1.event_list): for event in item: event_count = event_count + 1 - self.assertFalse(event.duration) + self.assertFalse(event.duration) self.assertTrue(event.end_index) self.assertEqual(event.start_index, index) self.assertEqual(event.start_index, index) self.assertEqual(event.start_time, manager1.data.dataframe.loc[index, "onset"]) if not event.end_time: self.assertEqual(event.end_index, len(manager1.data.dataframe)) - print("to here") # def test_constructor(self): @@ -56,7 +51,7 @@ def test_constructor(self): # self.assertEqual(hed, manager1.hed_strings[i]) # self.assertEqual(context, manager1.contexts[i]) # i = i + 1 - # + # def test_constructor_from_assembled(self): # hed_strings = get_assembled_strings(self.input_data, hed_schema=self.schema, expand_defs=False) # manager1 = HedContextManager(hed_strings, self.schema) @@ -64,12 +59,12 @@ def test_constructor(self): # "The constructor for assembled strings has expected # of strings") # self.assertEqual(len(manager1.onset_list), 261, # "The constructor for assembled strings has onset_list of correct length") - # + # def test_constructor_unmatched(self): # with self.assertRaises(HedFileError) as context: # HedContextManager(self.test_strings2, self.schema) # self.assertEqual(context.exception.args[0], 'UnmatchedOffset') - # + # def test_constructor_multiple_values(self): # manager = HedContextManager(self.test_strings3, self.schema) # self.assertEqual(len(manager.onset_list), 3, "Constructor should have right number of onsets") diff --git a/tests/tools/analysis/test_hed_context_manager.py b/tests/tools/analysis/test_hed_context_manager.py index 26e0f4e87..2ac042453 100644 --- a/tests/tools/analysis/test_hed_context_manager.py +++ b/tests/tools/analysis/test_hed_context_manager.py @@ -77,8 +77,8 @@ def test_constructor1(self): self.assertEqual(cont.exception.args[0], "ContextRequiresSchema") def test_iter(self): - hed_strings, _ = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, - join_columns=True, shrink_defs=True, expand_defs=False) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) manager1 = HedContextManager(hed_strings, self.schema) i = 0 for hed, context in manager1.iter_context(): @@ -87,8 +87,8 @@ def test_iter(self): i = i + 1 def test_constructor_from_assembled(self): - hed_strings, _ = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, - join_columns=True, shrink_defs=True, expand_defs=False) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) manager1 = HedContextManager(hed_strings, self.schema) self.assertEqual(len(manager1.hed_strings), 200, "The constructor for assembled strings has expected # of strings") diff --git a/tests/tools/analysis/test_hed_tag_counts.py b/tests/tools/analysis/test_hed_tag_counts.py index 76b0a9eaf..0950ea909 100644 --- a/tests/tools/analysis/test_hed_tag_counts.py +++ b/tests/tools/analysis/test_hed_tag_counts.py @@ -25,7 +25,7 @@ def setUpClass(cls): cls.hed_schema = schema sidecar1 = Sidecar(json_path, name='face_sub1_json') input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events") - input_df, def_dict = assemble_hed(input_data, expand_defs=False) + input_df, def_dict = assemble_hed(input_data, sidecar1, schema, expand_defs=False) cls.input_df = input_df cls.def_dict = def_dict diff --git a/tests/tools/analysis/test_hed_type_counts.py b/tests/tools/analysis/test_hed_type_counts.py index 711b8d4c9..c4fd22cab 100644 --- a/tests/tools/analysis/test_hed_type_counts.py +++ b/tests/tools/analysis/test_hed_type_counts.py @@ -6,7 +6,7 @@ from hed.tools.analysis.hed_context_manager import HedContextManager from hed.tools.analysis.hed_type_values import HedTypeValues from hed.tools.analysis.hed_type_counts import HedTypeCount, HedTypeCounts -from hed.tools.analysis.analysis_util import get_assembled_strings +from hed.models.df_util import get_assembled class Test(unittest.TestCase): @@ -19,10 +19,10 @@ def setUpClass(cls): events_path = os.path.realpath(os.path.join(bids_root_path, 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv')) sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json')) - sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json') - input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events") - hed_strings1 = get_assembled_strings(input_data, hed_schema=schema, expand_defs=False) - definitions1 = input_data.get_definitions(as_strings=False).gathered_defs + sidecar1 = Sidecar(sidecar_path, name='face_sub1_json') + input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events") + hed_strings1, definitions1 = get_assembled(input_data, sidecar1, schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) cls.var_type1 = HedTypeValues(HedContextManager(hed_strings1, schema), definitions1, 'run-01', type_tag='condition-variable') diff --git a/tests/tools/analysis/test_hed_type_definitions.py b/tests/tools/analysis/test_hed_type_definitions.py index 7a66d7e8e..15cbedce2 100644 --- a/tests/tools/analysis/test_hed_type_definitions.py +++ b/tests/tools/analysis/test_hed_type_definitions.py @@ -42,9 +42,10 @@ def setUpClass(cls): events_path = os.path.realpath(os.path.join(bids_root_path, 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv')) sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json')) - sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json') - cls.input_data = TabularInput(events_path, hed_schema=schema, sidecar=sidecar1, name="face_sub1_events") + sidecar1 = Sidecar(sidecar_path, name='face_sub1_json') + cls.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events") cls.schema = schema + cls.sidecar1 = sidecar1 def test_constructor(self): def_man = HedTypeDefinitions(self.definitions1, self.schema) @@ -54,8 +55,8 @@ def test_constructor(self): self.assertEqual(len(def_man.def_map), len(def_man.definitions), "Constructor condition_map should be the same length as the definitions dictionary") - def test_constructor_from_tabular_input(self): - definitions = self.input_data.get_definitions(as_strings=False).gathered_defs + def test_constructor_from_sidecar(self): + definitions = self.sidecar1.get_def_dict(self.schema) def_man = HedTypeDefinitions(definitions, self.schema) self.assertIsInstance(def_man, HedTypeDefinitions, "Constructor should create a HedTypeDefinitions from a tabular input") diff --git a/tests/tools/analysis/test_hed_type_factors.py b/tests/tools/analysis/test_hed_type_factors.py index 5615453da..5821e2675 100644 --- a/tests/tools/analysis/test_hed_type_factors.py +++ b/tests/tools/analysis/test_hed_type_factors.py @@ -10,7 +10,7 @@ from hed.tools.analysis.hed_context_manager import HedContextManager from hed.tools.analysis.hed_type_values import HedTypeValues from hed.tools.analysis.hed_type_factors import HedTypeFactors -from hed.tools.analysis.analysis_util import get_assembled_strings +from hed.models.df_util import get_assembled class Test(unittest.TestCase): @@ -57,8 +57,9 @@ def setUpClass(cls): events_path = os.path.realpath(os.path.join(bids_root_path, 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv')) sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json')) - sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json') - cls.input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events") + sidecar1 = Sidecar(sidecar_path, name='face_sub1_json') + cls.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events") + cls.sidecar1 = sidecar1 cls.schema = schema def test_with_mixed(self): @@ -73,9 +74,9 @@ def test_with_mixed(self): self.assertIsInstance(summary1, dict) def test_tabular_input(self): - test_strings1 = get_assembled_strings(self.input_data, hed_schema=self.schema, expand_defs=False) - definitions = self.input_data.get_definitions(as_strings=False).gathered_defs - var_manager = HedTypeValues(HedContextManager(test_strings1, self.schema), definitions, 'run-01') + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeValues(HedContextManager(hed_strings, self.schema), definitions, 'run-01') self.assertIsInstance(var_manager, HedTypeValues, "Constructor should create a HedTypeManager from a tabular input") var_fact = var_manager.get_type_value_factors('face-type') @@ -154,8 +155,8 @@ def test_count_events(self): self.assertIsNone(max_multiple2, "_count_level_events should not have a max multiple for empty list") def test_get_summary(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.schema, expand_defs=False) - definitions = self.input_data.get_definitions(as_strings=False).gathered_defs + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) var_manager = HedTypeValues(HedContextManager(hed_strings, self.schema), definitions, 'run-01') var_key = var_manager.get_type_value_factors('key-assignment') sum_key = var_key.get_summary() diff --git a/tests/tools/analysis/test_hed_type_manager.py b/tests/tools/analysis/test_hed_type_manager.py index 82bdf0e8b..9fd7abce2 100644 --- a/tests/tools/analysis/test_hed_type_manager.py +++ b/tests/tools/analysis/test_hed_type_manager.py @@ -6,7 +6,7 @@ from hed.tools.analysis.hed_type_values import HedTypeValues from hed.tools.analysis.hed_type_factors import HedTypeFactors from hed.tools.analysis.hed_type_manager import HedTypeManager -from hed.tools.analysis.analysis_util import get_assembled_strings +from hed.models.df_util import get_assembled class Test(unittest.TestCase): @@ -18,14 +18,16 @@ def setUp(self): events_path = os.path.realpath(os.path.join(bids_root_path, 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv')) sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json')) - sidecar1 = Sidecar(sidecar_path, hed_schema=schema, name='face_sub1_json') - self.input_data = TabularInput(events_path, sidecar=sidecar1, hed_schema=schema, name="face_sub1_events") - self.hed_strings = get_assembled_strings(self.input_data, hed_schema=schema, expand_defs=False) - self.hed_schema = schema - self.definitions = self.input_data.get_definitions() + sidecar1 = Sidecar(sidecar_path, name='face_sub1_json') + self.input_data = TabularInput(events_path, sidecar=sidecar1, name="face_sub1_events") + self.hed_strings, self.definitions = get_assembled(self.input_data, sidecar1, schema, + extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + self.sidecar1 = sidecar1 + self.schema = schema def test_constructor(self): - var_manager = HedTypeManager(self.hed_strings, self.hed_schema, self.definitions) + var_manager = HedTypeManager(self.hed_strings, self.schema, self.definitions) self.assertIsInstance(var_manager, HedTypeManager, "Constructor should create a HedTypeManager from a tabular input") self.assertEqual(len(var_manager.context_manager.hed_strings), len(var_manager.context_manager.contexts), @@ -33,7 +35,7 @@ def test_constructor(self): self.assertFalse(var_manager._type_tag_map, "constructor has empty map") def test_add_type_variable(self): - var_manager = HedTypeManager(self.hed_strings, self.hed_schema, self.definitions) + var_manager = HedTypeManager(self.hed_strings, self.schema, self.definitions) self.assertFalse(var_manager._type_tag_map, "constructor has empty map") var_manager.add_type_variable("Condition-variable") self.assertEqual(len(var_manager._type_tag_map), 1, @@ -48,10 +50,10 @@ def test_add_type_variable(self): "add_type_variable has 2 element map after two types are added") def test_get_factor_vectors(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=False) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) base_length = len(hed_strings) - def_mapper = self.input_data._def_mapper - var_manager = HedTypeManager(hed_strings, self.hed_schema, def_mapper) + var_manager = HedTypeManager(hed_strings, self.schema, definitions) var_manager.add_type_variable("Condition-variable") var_manager.add_type_variable("task") df_cond = var_manager.get_factor_vectors("condition-variable") @@ -64,9 +66,9 @@ def test_get_factor_vectors(self): self.assertIsNone(df_baloney, "get_factor_vectors returns None if no factors") def test_get_type_variable(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=False) - def_mapper = self.input_data._def_mapper - var_manager = HedTypeManager(hed_strings, self.hed_schema, def_mapper) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeManager(hed_strings, self.schema, definitions) var_manager.add_type_variable("Condition-variable") type_var = var_manager.get_type_variable("condition-variable") self.assertIsInstance(type_var, HedTypeValues, @@ -75,9 +77,9 @@ def test_get_type_variable(self): self.assertIsNone(type_var, "get_type_variable returns None if the key does not exist") def test_get_type_variable_def_names(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=False) - def_mapper = self.input_data._def_mapper - var_manager = HedTypeManager(hed_strings, self.hed_schema, def_mapper) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeManager(hed_strings, self.schema, definitions) var_manager.add_type_variable("Condition-variable") def_names = var_manager.get_type_tag_def_names("condition-variable") self.assertEqual(len(def_names), 7, @@ -88,9 +90,9 @@ def test_get_type_variable_def_names(self): self.assertFalse(def_names, "get_type_tag_def_names returns empty if the type does not exist") def test_get_variable_type_map(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=False) - def_mapper = self.input_data._def_mapper - var_manager = HedTypeManager(hed_strings, self.hed_schema, def_mapper) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeManager(hed_strings, self.schema, definitions) var_manager.add_type_variable("Condition-variable") this_var = var_manager.get_type_variable("condition-variable") self.assertIsInstance(this_var, HedTypeValues, @@ -104,9 +106,9 @@ def test_get_variable_type_map(self): "get_type_variable_map map has right length when key upper case") def test_get_type_variable_factor(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=False) - def_mapper = self.input_data._def_mapper - var_manager = HedTypeManager(hed_strings, self.hed_schema, def_mapper) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeManager(hed_strings, self.schema, definitions) var_manager.add_type_variable("Condition-variable") var_factor1 = var_manager.get_type_tag_factor("condition-variable", "key-assignment") self.assertIsInstance(var_factor1, HedTypeFactors, @@ -117,9 +119,9 @@ def test_get_type_variable_factor(self): self.assertIsNone(var_factor3, "get_type_tag_factor returns None if type variable does not exist") def test_type_variables(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=False) - definitions = self.input_data.get_definitions - var_manager = HedTypeManager(hed_strings, self.hed_schema, definitions) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeManager(hed_strings, self.schema, definitions) vars1 = var_manager.type_variables self.assertFalse(vars1, "type_variables is empty if no types have been added") var_manager.add_type_variable("Condition-variable") @@ -129,9 +131,9 @@ def test_type_variables(self): self.assertEqual(len(vars2), 2, "type_variables return list is right length") def test_summarize_all(self): - hed_strings = get_assembled_strings(self.input_data, hed_schema=self.hed_schema, expand_defs=False) - def_mapper = self.input_data._def_mapper - var_manager = HedTypeManager(hed_strings, self.hed_schema, def_mapper) + hed_strings, definitions = get_assembled(self.input_data, self.sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeManager(hed_strings, self.schema, definitions) summary1 = var_manager.summarize_all() self.assertIsInstance(summary1, dict, "summarize_all returns a dictionary when nothing has been added") self.assertFalse(summary1, "summarize_all return dictionary is empty when nothing has been added") diff --git a/tests/tools/analysis/test_hed_type_values.py b/tests/tools/analysis/test_hed_type_values.py index c5ad5557a..4b3125353 100644 --- a/tests/tools/analysis/test_hed_type_values.py +++ b/tests/tools/analysis/test_hed_type_values.py @@ -10,7 +10,7 @@ from hed.schema.hed_schema_io import load_schema_version from hed.tools.analysis.hed_context_manager import HedContextManager from hed.tools.analysis.hed_type_values import HedTypeValues -from hed.tools.analysis.analysis_util import get_assembled_strings +from hed.models.df_util import get_assembled class Test(unittest.TestCase): @@ -53,12 +53,11 @@ def setUpClass(cls): cls.events_path = os.path.realpath(os.path.join(bids_root_path, 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv')) cls.sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json')) - cls.hed_schema = schema + cls.schema = schema def test_constructor(self): - strings1 = [HedString(hed, hed_schema=self.hed_schema) for hed in self.test_strings1] - strings2 = [HedString(hed, hed_schema=self.hed_schema) for hed in self.test_strings1] - con_man = HedContextManager(strings1, hed_schema=self.hed_schema) + strings1 = [HedString(hed, hed_schema=self.schema) for hed in self.test_strings1] + con_man = HedContextManager(strings1, hed_schema=self.schema) type_var = HedTypeValues(con_man, self.defs, 'run-01') self.assertIsInstance(type_var, HedTypeValues, "Constructor should create a HedTypeManager from strings") @@ -66,22 +65,20 @@ def test_constructor(self): "Constructor ConditionVariables should have the right length") def test_constructor_from_tabular_input(self): - sidecar1 = Sidecar(self.sidecar_path, hed_schema=self.hed_schema, name='face_sub1_json') - input_data = TabularInput(self.events_path, hed_schema=self.hed_schema, - sidecar=sidecar1, name="face_sub1_events") - test_strings1 = get_assembled_strings(input_data, hed_schema=self.hed_schema, expand_defs=False) - definitions = input_data.get_definitions(as_strings=False).gathered_defs - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), definitions, 'run-01') + sidecar1 = Sidecar(self.sidecar_path, name='face_sub1_json') + input_data = TabularInput(self.events_path, sidecar=sidecar1, name="face_sub1_events") + test_strings1, definitions = get_assembled(input_data, sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeValues(HedContextManager(test_strings1, self.schema), definitions, 'run-01') self.assertIsInstance(var_manager, HedTypeValues, "Constructor should create a HedTypeManager from a tabular input") def test_constructor_variable_caps(self): - sidecar1 = Sidecar(self.sidecar_path, hed_schema=self.hed_schema, name='face_sub1_json') - input_data = TabularInput(self.events_path, sidecar=sidecar1, hed_schema=self.hed_schema, - name="face_sub1_events") - test_strings1 = get_assembled_strings(input_data, hed_schema=self.hed_schema, expand_defs=False) - definitions = input_data.get_definitions(as_strings=False).gathered_defs - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), + sidecar1 = Sidecar(self.sidecar_path, name='face_sub1_json') + input_data = TabularInput(self.events_path, sidecar1, name="face_sub1_events") + test_strings1, definitions = get_assembled(input_data, sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeValues(HedContextManager(test_strings1, self.schema), definitions, 'run-01', type_tag="Condition-variable") self.assertIsInstance(var_manager, HedTypeValues, "Constructor should create a HedTypeManager variable caps") @@ -89,34 +86,33 @@ def test_constructor_variable_caps(self): def test_constructor_variable_task(self): sidecar1 = Sidecar(self.sidecar_path, name='face_sub1_json') input_data = TabularInput(self.events_path, sidecar=sidecar1, name="face_sub1_events") - test_strings1 = get_assembled_strings(input_data, hed_schema=self.hed_schema, expand_defs=False) - definitions = input_data.get_definitions(as_strings=False).gathered_defs - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), + test_strings1, definitions = get_assembled(input_data, sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeValues(HedContextManager(test_strings1, self.schema), definitions, 'run-01', type_tag="task") self.assertIsInstance(var_manager, HedTypeValues, "Constructor should create a HedTypeManager variable task") def test_constructor_multiple_values(self): - test_strings1 = [HedString(hed, hed_schema=self.hed_schema) for hed in self.test_strings2] - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), self.defs, 'run-01') + hed_strings = [HedString(hed, self.schema) for hed in self.test_strings2] + var_manager = HedTypeValues(HedContextManager(hed_strings, self.schema), self.defs, 'run-01') self.assertIsInstance(var_manager, HedTypeValues, "Constructor should create a HedTypeManager from strings") self.assertEqual(len(var_manager._type_value_map), 3, "Constructor should have right number of type_variables if multiple") def test_constructor_unmatched(self): - test_strings1 = [HedString(hed, hed_schema=self.hed_schema) for hed in self.test_strings3] + hed_strings = [HedString(hed, self.schema) for hed in self.test_strings3] with self.assertRaises(HedFileError) as context: - HedTypeValues(HedContextManager(test_strings1, self.hed_schema), self.defs, 'run-01') + HedTypeValues(HedContextManager(hed_strings, self.schema), self.defs, 'run-01') self.assertEqual(context.exception.args[0], 'UnmatchedOffset') def test_get_variable_factors(self): - sidecar1 = Sidecar(self.sidecar_path, hed_schema=self.hed_schema, name='face_sub1_json') - input_data = TabularInput(self.events_path, sidecar=sidecar1, hed_schema=self.hed_schema, - name="face_sub1_events") - test_strings1 = get_assembled_strings(input_data, hed_schema=self.hed_schema, expand_defs=False) - definitions = input_data.get_definitions(as_strings=False).gathered_defs - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), definitions, 'run-01') + sidecar1 = Sidecar(self.sidecar_path, name='face_sub1_json') + input_data = TabularInput(self.events_path, sidecar1, name="face_sub1_events") + test_strings1, definitions = get_assembled(input_data, sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeValues(HedContextManager(test_strings1, self.schema), definitions, 'run-01') df_new1 = var_manager.get_type_factors() self.assertIsInstance(df_new1, DataFrame) self.assertEqual(len(df_new1), 200) @@ -128,47 +124,45 @@ def test_get_variable_factors(self): self.assertIsNone(df_new3) def test_str(self): - sidecar1 = Sidecar(self.sidecar_path, hed_schema=self.hed_schema, name='face_sub1_json') - input_data = TabularInput(self.events_path, hed_schema=self.hed_schema, - sidecar=sidecar1, name="face_sub1_events") - test_strings1 = get_assembled_strings(input_data, hed_schema=self.hed_schema, expand_defs=False) - definitions = input_data.get_definitions(as_strings=False).gathered_defs - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), definitions, 'run-01') + sidecar1 = Sidecar(self.sidecar_path, name='face_sub1_json') + input_data = TabularInput(self.events_path, sidecar1, name="face_sub1_events") + test_strings1, definitions = get_assembled(input_data, sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeValues(HedContextManager(test_strings1, self.schema), definitions, 'run-01') new_str = str(var_manager) self.assertIsInstance(new_str, str) def test_summarize_variables(self): - sidecar1 = Sidecar(self.sidecar_path, hed_schema=self.hed_schema, name='face_sub1_json') - input_data = TabularInput(self.events_path, hed_schema=self.hed_schema, - sidecar=sidecar1, name="face_sub1_events") - test_strings1 = get_assembled_strings(input_data, hed_schema=self.hed_schema, expand_defs=False) - definitions = input_data.get_definitions(as_strings=False).gathered_defs - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), definitions, 'run-01') + sidecar1 = Sidecar(self.sidecar_path, name='face_sub1_json') + input_data = TabularInput(self.events_path, sidecar1, name="face_sub1_events") + test_strings1, definitions = get_assembled(input_data, sidecar1, self.schema, extra_def_dicts=None, + join_columns=True, shrink_defs=True, expand_defs=False) + var_manager = HedTypeValues(HedContextManager(test_strings1, self.schema), definitions, 'run-01') summary = var_manager.get_summary() self.assertIsInstance(summary, dict, "get_summary produces a dictionary if not json") self.assertEqual(len(summary), 3, "Summarize_variables has right number of condition type_variables") self.assertIn("key-assignment", summary, "get_summary has a correct key") def test_extract_definition_variables(self): - test_strings1 = [HedString(hed, hed_schema=self.hed_schema) for hed in self.test_strings1] - var_manager = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), self.defs, 'run-01') + hed_strings = [HedString(hed, self.schema) for hed in self.test_strings1] + var_manager = HedTypeValues(HedContextManager(hed_strings, self.schema), self.defs, 'run-01') var_levels = var_manager._type_value_map['var3'].levels self.assertNotIn('cond3/7', var_levels, "_extract_definition_variables before extraction def/cond3/7 not in levels") - tag = HedTag("Def/Cond3/7", hed_schema=self.hed_schema) + tag = HedTag("Def/Cond3/7", hed_schema=self.schema) var_manager._extract_definition_variables(tag, 5) self.assertIn('cond3/7', var_levels, "_extract_definition_variables after extraction def/cond3/7 not in levels") def test_get_variable_names(self): - test_strings1 = [HedString(hed, hed_schema=self.hed_schema) for hed in self.test_strings1] - conditions1 = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), self.defs, 'run-01') + hed_strings = [HedString(hed, self.schema) for hed in self.test_strings1] + conditions1 = HedTypeValues(HedContextManager(hed_strings, self.schema), self.defs, 'run-01') list1 = conditions1.get_type_value_names() self.assertEqual(len(list1), 8, "get_variable_tags list should have the right length") def test_get_variable_def_names(self): - test_strings1 = [HedString(hed, hed_schema=self.hed_schema) for hed in self.test_strings1] - conditions1 = HedTypeValues(HedContextManager(test_strings1, self.hed_schema), self.defs, 'run-01') + hed_strings = [HedString(hed, self.schema) for hed in self.test_strings1] + conditions1 = HedTypeValues(HedContextManager(hed_strings, self.schema), self.defs, 'run-01') list1 = conditions1.get_type_def_names() self.assertEqual(len(list1), 5, "get_type_def_names list should have the right length") diff --git a/tests/tools/remodeling/operations/test_summarize_hed_tags_op.py b/tests/tools/remodeling/operations/test_summarize_hed_tags_op.py index d82d14ea0..5f5ee41bf 100644 --- a/tests/tools/remodeling/operations/test_summarize_hed_tags_op.py +++ b/tests/tools/remodeling/operations/test_summarize_hed_tags_op.py @@ -2,6 +2,7 @@ import os import unittest import pandas as pd +from hed.models.df_util import get_assembled from hed.tools.remodeling.dispatcher import Dispatcher from hed.tools.remodeling.operations.summarize_hed_tags_op import SummarizeHedTagsOp, HedTagSummaryContext @@ -96,16 +97,17 @@ def test_quick3(self): } } my_json_str = json.dumps(my_json) - my_sidecar = Sidecar(StringIO(my_json_str), hed_schema=my_schema) + my_sidecar = Sidecar(StringIO(my_json_str)) data = [[0.5, 0, 'code1', 'Description/This is a test, Label/Temp, (Def/Blech1, Green)'], [0.6, 0, 'code2', 'Sensory-event, ((Description/Animal, Condition-variable/Blech))']] df = pd.DataFrame(data, columns=['onset', 'duration', 'code', 'HED']) - input_data = TabularInput(df, hed_schema=my_schema, sidecar=my_sidecar) + input_data = TabularInput(df, sidecar=my_sidecar) counts = HedTagCounts('myName', 2) summary_dict = {} - for objs in input_data.iter_dataframe(hed_ops=[my_schema], return_string_only=False, - expand_defs=True, remove_definitions=True): - counts.update_event_counts(objs['HED'], 'myName') + hed_strings = get_assembled(input_data, my_sidecar, my_schema, extra_def_dicts=None, join_columns=True, + shrink_defs=False, expand_defs=True) + for hed in hed_strings: + counts.update_event_counts(hed, 'myName') summary_dict['myName'] = counts def test_quick4(self): @@ -117,10 +119,13 @@ def test_quick4(self): data_path = os.path.realpath(os.path.join(path, 'sub-002_task-FacePerception_run-1_events.tsv')) json_path = os.path.realpath(os.path.join(path, 'task-FacePerception_events.json')) my_schema = load_schema_version('8.1.0') - sidecar = Sidecar(json_path, hed_schema=my_schema) - input_data = TabularInput(data_path, hed_schema=my_schema, sidecar=sidecar) + sidecar = Sidecar(json_path,) + input_data = TabularInput(data_path, sidecar=sidecar) counts = HedTagCounts('myName', 2) summary_dict = {} + hed_strings, definitions = get_assembled(input_data, sidecar, my_schema, + extra_def_dicts=None, join_columns=True, + shrink_defs=False, expand_defs=True) for objs in input_data.iter_dataframe(hed_ops=[my_schema], return_string_only=False, expand_defs=True, remove_definitions=True): x = objs['HED']