diff --git a/hed/tools/remodeling/cli/run_remodel.py b/hed/tools/remodeling/cli/run_remodel.py index 7cb7fc24a..0ba6a9262 100644 --- a/hed/tools/remodeling/cli/run_remodel.py +++ b/hed/tools/remodeling/cli/run_remodel.py @@ -42,7 +42,8 @@ def get_parser(): parser.add_argument("-r", "--hed-versions", dest="hed_versions", nargs="*", default=[], help="Optional list of HED schema versions used for annotation, include prefixes.") parser.add_argument("-s", "--save-formats", nargs="*", default=['.json', '.txt'], dest="save_formats", - help="Format for saving any summaries, if any. If no summaries are to be written, use the -ns option.") + help="Format for saving any summaries, if any. If no summaries are to be written," + + "use the -ns option.") parser.add_argument("-t", "--task-names", dest="task_names", nargs="*", default=[], help="The names of the task.") parser.add_argument("-v", "--verbose", action='store_true', help="If present, output informative messages as computation progresses.") @@ -122,7 +123,7 @@ def run_direct_ops(dispatch, args): Parameters: dispatch (Dispatcher): Controls the application of the operations and backup. - args (dict): Dictionary of arguments and their values. + args (argparse.Namespace): Dictionary of arguments and their values. """ diff --git a/hed/tools/remodeling/operations/base_summary.py b/hed/tools/remodeling/operations/base_summary.py index aedb0429d..9153ed670 100644 --- a/hed/tools/remodeling/operations/base_summary.py +++ b/hed/tools/remodeling/operations/base_summary.py @@ -137,7 +137,10 @@ def _save_summary_files(self, save_dir, file_format, summary, individual_summari summary (dictionary): Dictionary of summaries (has "Dataset" and "Individual files" keys. """ - time_stamp = '_' + get_timestamp() + if self.op.append_timecode: + time_stamp = '_' + get_timestamp() + else: + time_stamp = '' this_save = os.path.join(save_dir, self.op.summary_name + '/') os.makedirs(os.path.realpath(this_save), exist_ok=True) filename = os.path.realpath(os.path.join(this_save, self.op.summary_filename + time_stamp + file_format)) diff --git a/hed/tools/remodeling/operations/factor_hed_tags_op.py b/hed/tools/remodeling/operations/factor_hed_tags_op.py index ae1f35e63..ee8cf3df1 100644 --- a/hed/tools/remodeling/operations/factor_hed_tags_op.py +++ b/hed/tools/remodeling/operations/factor_hed_tags_op.py @@ -6,7 +6,6 @@ from hed.tools.remodeling.operations.base_op import BaseOp from hed.models.tabular_input import TabularInput from hed.models.sidecar import Sidecar -from hed.models.expression_parser import QueryParser from hed.models.df_util import get_assembled from hed.tools.analysis.analysis_util import get_expression_parsers, search_strings @@ -48,7 +47,7 @@ def __init__(self, parameters): KeyError - If a required parameter is missing. - + - If an unexpected parameter is provided. TypeError @@ -56,9 +55,9 @@ def __init__(self, parameters): ValueError - If the specification is missing a valid operation. - + - If the length of query names is not empty and not same length as queries. - + - If there are duplicate query names. """ diff --git a/hed/tools/remodeling/operations/merge_consecutive_op.py b/hed/tools/remodeling/operations/merge_consecutive_op.py index c7acdb26d..22ba34745 100644 --- a/hed/tools/remodeling/operations/merge_consecutive_op.py +++ b/hed/tools/remodeling/operations/merge_consecutive_op.py @@ -38,7 +38,7 @@ def __init__(self, parameters): KeyError - If a required parameter is missing. - If an unexpected parameter is provided. - + TypeError - If a parameter has the wrong type. diff --git a/hed/tools/remodeling/operations/number_groups_op.py b/hed/tools/remodeling/operations/number_groups_op.py index d23b9b5dc..bc9b96033 100644 --- a/hed/tools/remodeling/operations/number_groups_op.py +++ b/hed/tools/remodeling/operations/number_groups_op.py @@ -1,10 +1,9 @@ """ Implementation in progress. """ -import numpy as np + from hed.tools.remodeling.operations.base_op import BaseOp -from hed.tools.util.data_util import get_indices, tuple_to_range -import itertools -#TODO: This class is under development + +# TODO: This class is under development class NumberGroupsOp(BaseOp): @@ -35,7 +34,8 @@ def __init__(self, parameters): required_missing = required.difference(set(param_to_test.keys())) if required_missing: raise KeyError("MissingRequiredParameters", - f"Specified {param_to_test} for number_rows requires parameters {list(required_missing)}") + f"Specified {param_to_test} for number_rows requires parameters" + f"{list(required_missing)}") for param_name, param_value in param_to_test.items(): param_type = str if param_name in required: diff --git a/hed/tools/remodeling/operations/number_rows_op.py b/hed/tools/remodeling/operations/number_rows_op.py index 3157b7b3e..e37b180fb 100644 --- a/hed/tools/remodeling/operations/number_rows_op.py +++ b/hed/tools/remodeling/operations/number_rows_op.py @@ -2,7 +2,7 @@ import numpy as np from hed.tools.remodeling.operations.base_op import BaseOp -#TODO: This class is under development +# TODO: This class is under development class NumberRowsOp(BaseOp): diff --git a/hed/tools/remodeling/operations/remap_columns_op.py b/hed/tools/remodeling/operations/remap_columns_op.py index 2448d45b3..3f437ff97 100644 --- a/hed/tools/remodeling/operations/remap_columns_op.py +++ b/hed/tools/remodeling/operations/remap_columns_op.py @@ -48,7 +48,7 @@ def __init__(self, parameters): KeyError - If a required parameter is missing. - If an unexpected parameter is provided. - + TypeError - If a parameter has the wrong type. diff --git a/hed/tools/remodeling/operations/summarize_column_names_op.py b/hed/tools/remodeling/operations/summarize_column_names_op.py index d454bcc16..da3be4e52 100644 --- a/hed/tools/remodeling/operations/summarize_column_names_op.py +++ b/hed/tools/remodeling/operations/summarize_column_names_op.py @@ -23,6 +23,7 @@ class SummarizeColumnNamesOp(BaseOp): "summary_filename": str }, "optional_parameters": { + "append_timecode": bool } } @@ -46,6 +47,7 @@ def __init__(self, parameters): super().__init__(self.PARAMS, parameters) self.summary_name = parameters['summary_name'] self.summary_filename = parameters['summary_filename'] + self.append_timecode = parameters.get('append_timecode', False) def do_op(self, dispatcher, df, name, sidecar=None): """ Create factor columns corresponding to values in a specified column. diff --git a/hed/tools/remodeling/operations/summarize_column_values_op.py b/hed/tools/remodeling/operations/summarize_column_values_op.py index a01dfc856..64d67445f 100644 --- a/hed/tools/remodeling/operations/summarize_column_values_op.py +++ b/hed/tools/remodeling/operations/summarize_column_values_op.py @@ -30,8 +30,9 @@ class SummarizeColumnValuesOp(BaseOp): "value_columns": list }, "optional_parameters": { - "values_per_line": int, - "max_categorical": int + "append_timecode": bool, + "max_categorical": int, + "values_per_line": int } } @@ -61,6 +62,7 @@ def __init__(self, parameters): self.summary_filename = parameters['summary_filename'] self.skip_columns = parameters['skip_columns'] self.value_columns = parameters['value_columns'] + self.append_timecode = parameters.get('append_timecode', False) self.max_categorical = parameters.get('max_categorical', float('inf')) self.values_per_line = parameters.get('values_per_line', self.VALUES_PER_LINE) @@ -125,7 +127,7 @@ def get_details_dict(self, summary): unique_counts = [(key, len(count_dict)) for key, count_dict in this_summary['Categorical columns'].items()] this_summary['Categorical counts'] = dict(unique_counts) for key, dict_entry in this_summary['Categorical columns'].items(): - num_disp, sorted_tuples = ColumnValueSummary.sort_dict(self, dict_entry, reverse=True) + num_disp, sorted_tuples = ColumnValueSummary.sort_dict(dict_entry, reverse=True) this_summary['Categorical columns'][key] = dict(sorted_tuples[:min(num_disp, self.op.max_categorical)]) return this_summary @@ -166,7 +168,7 @@ def _get_categorical_string(self, result, offset="", indent=" "): """ Return a string with the summary for a particular categorical dictionary. Parameters: - cat_dict (dict): Dictionary of summary information for a particular tabular file. + result (dict): Dictionary of summary information for a particular tabular file. offset (str): String of blanks used as offset for every item indent (str): String of blanks used as the additional amount to indent an item's for readability. @@ -230,7 +232,8 @@ def _get_categorical_col(self, entry, count_dict, offset="", indent=" "): """ Return a string with the summary for a particular categorical column. Parameters: - dict_entry(tuple): (Name of the column, summary dict for that column) + entry(tuple): (Name of the column, summary dict for that column) + count_dict (dict): Count of the total number of unique values indexed by the name offset(str): String of blanks used as offset for all items indent (str): String of blanks used as the additional amount to indent for this item's readability. @@ -273,6 +276,6 @@ def _get_value_string(val_dict, offset="", indent=""): return "\n".join(sum_list) @staticmethod - def sort_dict(self, count_dict, reverse=False): + def sort_dict(count_dict, reverse=False): sorted_tuples = sorted(count_dict.items(), key=lambda x: x[1][0], reverse=reverse) return len(sorted_tuples), sorted_tuples diff --git a/hed/tools/remodeling/operations/summarize_definitions_op.py b/hed/tools/remodeling/operations/summarize_definitions_op.py index 73c7d5957..aa880307f 100644 --- a/hed/tools/remodeling/operations/summarize_definitions_op.py +++ b/hed/tools/remodeling/operations/summarize_definitions_op.py @@ -24,6 +24,7 @@ class SummarizeDefinitionsOp(BaseOp): "summary_filename": str }, "optional_parameters": { + "append_timecode": bool } } @@ -49,6 +50,7 @@ def __init__(self, parameters): super().__init__(self.PARAMS, parameters) self.summary_name = parameters['summary_name'] self.summary_filename = parameters['summary_filename'] + self.append_timecode = parameters.get('append_timecode', False) def do_op(self, dispatcher, df, name, sidecar=None): """ Create factor columns corresponding to values in a specified column. @@ -178,6 +180,7 @@ def nested_dict_to_string(data, level=1): return nested_dict_to_string(summary_dict) + @staticmethod def remove_description(def_entry): def_group = def_entry.contents.copy() description = "" diff --git a/hed/tools/remodeling/operations/summarize_hed_tags_op.py b/hed/tools/remodeling/operations/summarize_hed_tags_op.py index f44c70f21..5d58b78fd 100644 --- a/hed/tools/remodeling/operations/summarize_hed_tags_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_tags_op.py @@ -34,6 +34,7 @@ class SummarizeHedTagsOp(BaseOp): "tags": dict }, "optional_parameters": { + "append_timecode": bool, "expand_context": bool } } @@ -59,6 +60,7 @@ def __init__(self, parameters): self.summary_name = parameters['summary_name'] self.summary_filename = parameters['summary_filename'] self.tags = parameters['tags'] + self.append_timecode = parameters.get('append_timecode', False) self.expand_context = parameters.get('expand_context', False) def do_op(self, dispatcher, df, name, sidecar=None): diff --git a/hed/tools/remodeling/operations/summarize_hed_type_op.py b/hed/tools/remodeling/operations/summarize_hed_type_op.py index 4cbb96675..1a8c51211 100644 --- a/hed/tools/remodeling/operations/summarize_hed_type_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_type_op.py @@ -31,6 +31,7 @@ class SummarizeHedTypeOp(BaseOp): "type_tag": str }, "optional_parameters": { + "append_timecode": bool } } @@ -55,6 +56,7 @@ def __init__(self, parameters): self.summary_name = parameters['summary_name'] self.summary_filename = parameters['summary_filename'] self.type_tag = parameters['type_tag'].lower() + self.append_timecode = parameters.get('append_timecode', False) def do_op(self, dispatcher, df, name, sidecar=None): """ Summarize a specified HED type variable such as Condition-variable . diff --git a/hed/tools/remodeling/operations/summarize_hed_validation_op.py b/hed/tools/remodeling/operations/summarize_hed_validation_op.py index f395e837e..0633c5b57 100644 --- a/hed/tools/remodeling/operations/summarize_hed_validation_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_validation_op.py @@ -24,10 +24,11 @@ class SummarizeHedValidationOp(BaseOp): "operation": "summarize_hed_validation", "required_parameters": { "summary_name": str, - "summary_filename": str, - "check_for_warnings": bool + "summary_filename": str }, "optional_parameters": { + "append_timecode": bool, + "check_for_warnings": bool } } @@ -51,7 +52,8 @@ def __init__(self, parameters): super().__init__(self.PARAMS, parameters) self.summary_name = parameters['summary_name'] self.summary_filename = parameters['summary_filename'] - self.check_for_warnings = parameters['check_for_warnings'] + self.append_timecode = parameters.get('append_timecode', False) + self.check_for_warnings = parameters.get('check_for_warnings', False) def do_op(self, dispatcher, df, name, sidecar=None): """ Validate the dataframe with the accompanying sidecar, if any. @@ -220,7 +222,7 @@ def format_error(error): HedValidationSummary.update_error_location(error_locations, "row", "ec_row", error) HedValidationSummary.update_error_location(error_locations, "column", "ec_column", error) HedValidationSummary.update_error_location(error_locations, "sidecar column", - "ec_sidecarColumnName", error) + "ec_sidecarColumnName", error) HedValidationSummary.update_error_location(error_locations, "sidecar key", "ec_sidecarKeyName", error) location_str = ",".join(error_locations) if location_str: diff --git a/hed/tools/remodeling/operations/summarize_sidecar_from_events_op.py b/hed/tools/remodeling/operations/summarize_sidecar_from_events_op.py index dc68bb065..b191f0525 100644 --- a/hed/tools/remodeling/operations/summarize_sidecar_from_events_op.py +++ b/hed/tools/remodeling/operations/summarize_sidecar_from_events_op.py @@ -28,6 +28,7 @@ class SummarizeSidecarFromEventsOp(BaseOp): "value_columns": list, }, "optional_parameters": { + "append_timecode": bool } } @@ -54,6 +55,7 @@ def __init__(self, parameters): self.summary_filename = parameters['summary_filename'] self.skip_columns = parameters['skip_columns'] self.value_columns = parameters['value_columns'] + self.append_timecode = parameters.get('append_timecode', False) def do_op(self, dispatcher, df, name, sidecar=None): """ Create factor columns corresponding to values in a specified column. diff --git a/tests/tools/remodeling/operations/test_base_summary.py b/tests/tools/remodeling/operations/test_base_summary.py index 3dc2d10fc..5faed6467 100644 --- a/tests/tools/remodeling/operations/test_base_summary.py +++ b/tests/tools/remodeling/operations/test_base_summary.py @@ -12,7 +12,9 @@ class TestOp(BaseOp): "summary_name": str, "summary_filename": str }, - "optional_parameters": {} + "optional_parameters": { + "append_timecode": bool + } } SUMMARY_TYPE = "test_sum" @@ -21,6 +23,7 @@ def __init__(self, parameters): super().__init__(self.PARAMS, parameters) self.summary_name = parameters['summary_name'] self.summary_filename = parameters['summary_filename'] + self.append_timecode = parameters.get('append_timecode', False) class TestSummary(BaseSummary):