Added alternate work directory location for remodeling

hed/tools/remodeling/backup_manager.py

@@ -10,16 +10,16 @@
 
 class BackupManager:
     DEFAULT_BACKUP_NAME = 'default_back'
-    RELATIVE_BACKUP_LOCATION = 'derivatives/remodel/backups'
+    RELATIVE_BACKUP_LOCATION = 'derivatives/remodel'
     BACKUP_DICTIONARY = 'backup_lock.json'
     BACKUP_ROOT = 'backup_root'
 
-    def __init__(self, data_root):
+    def __init__(self, data_root, backups_root=None):
         """ Constructor for the backup manager.
 
         Parameters:
-            data_root (str): full path of the root of the data directory.
-
+            data_root (str): Full path of the root of the data directory.
+            backups_root (str or None):  Full path to the root where backups subdirectory is located.
         Raises:
             - HedFileError:
                 - If the data_root does not correspond to a real directory.
@@ -28,11 +28,15 @@ def __init__(self, data_root):
         if not os.path.isdir(data_root):
             raise HedFileError('NonExistentData', f"{data_root} is not an existing directory", "")
         self.data_root = data_root
-        self.backups_root = os.path.join(data_root, self.RELATIVE_BACKUP_LOCATION)
-        os.makedirs(self.backups_root, exist_ok=True)
+        if backups_root:
+            self.backups_path = os.path.join(backups_root, 'backups')
+        else:
+            self.backups_path = os.path.join(data_root, self.RELATIVE_BACKUP_LOCATION, 'backups')
+        self.backups_path = os.path.realpath(self.backups_path)
+        os.makedirs(self.backups_path, exist_ok=True)
         self.backups_dict = self._get_backups()
 
-    def create_backup(self, file_list, backup_name=None, verbose=True):
+    def create_backup(self, file_list, backup_name=None, verbose=False):
         """ Create a new backup from file_list.
 
         Parameters:
@@ -46,7 +50,7 @@ def create_backup(self, file_list, backup_name=None, verbose=True):
         Raises:
             HedFileError   
                 - For missing or incorrect files.    
-                
+
             OS-related error   
                 - OS-related error when file copying occurs.    
 
@@ -59,7 +63,7 @@ def create_backup(self, file_list, backup_name=None, verbose=True):
         time_stamp = f"{str(datetime.now())}"
         if verbose:
             print(f"Creating backup {backup_name}")
-        backup_dir_path = os.path.realpath(os.path.join(self.backups_root, backup_name, BackupManager.BACKUP_ROOT))
+        backup_dir_path = os.path.realpath(os.path.join(self.backups_path, backup_name, BackupManager.BACKUP_ROOT))
         os.makedirs(backup_dir_path, exist_ok=True)
         for file in file_list:
             backup_file = self.get_backup_path(backup_name, file)
@@ -69,7 +73,7 @@ def create_backup(self, file_list, backup_name=None, verbose=True):
             shutil.copy2(file, backup_file)
             backup[self.get_file_key(file)] = time_stamp
         self.backups_dict[backup_name] = backup
-        backup_dict_path = os.path.realpath(os.path.join(self.backups_root, backup_name,
+        backup_dict_path = os.path.realpath(os.path.join(self.backups_path, backup_name,
                                                          self.BACKUP_DICTIONARY))
         with open(backup_dict_path, 'w') as fp:
             json.dump(backup, fp, indent=4)
@@ -83,11 +87,11 @@ def get_backup(self, backup_name):
 
         Returns:
             The dictionary with the backup info.
-            
+
         Notes:
-            - The dictionary with backup information has keys that are the paths of
-              the backed up files relative to the backup root. The values in this
-              dictionary are the dates on which the particular file was backed up.
+            The dictionary with backup information has keys that are the paths of
+            the backed up files relative to the backup root. The values in this
+            dictionary are the dates on which the particular file was backed up.
 
         """
         if backup_name not in self.backups_dict:
@@ -114,7 +118,7 @@ def get_backup_files(self, backup_name, original_paths=False):
             raise HedFileError("NoBackup", f"{backup_name} is not a valid backup", "")
         if original_paths:
             return [os.path.realpath(os.path.join(self.data_root, backup_key)) for backup_key in backup_dict.keys()]
-        return [os.path.realpath(os.path.join(self.backups_root, backup_name, self.BACKUP_ROOT, backup_key))
+        return [os.path.realpath(os.path.join(self.backups_path, backup_name, self.BACKUP_ROOT, backup_key))
                 for backup_key in backup_dict.keys()]
 
     def get_backup_path(self, backup_name, file_name):
@@ -128,7 +132,7 @@ def get_backup_path(self, backup_name, file_name):
             str:  Full path of the corresponding file in the backup.
 
         """
-        return os.path.realpath(os.path.join(self.backups_root, backup_name, self.BACKUP_ROOT,
+        return os.path.realpath(os.path.join(self.backups_path, backup_name, self.BACKUP_ROOT,
                                              self.get_file_key(file_name)))
 
     def get_file_key(self, file_name):
@@ -163,8 +167,8 @@ def _get_backups(self):
             HedFileError - if a backup is inconsistent for any reason.
         """
         backups = {}
-        for backup in os.listdir(self.backups_root):
-            backup_root = os.path.realpath(os.path.join(self.backups_root, backup))
+        for backup in os.listdir(self.backups_path):
+            backup_root = os.path.realpath(os.path.join(self.backups_path, backup))
             if not os.path.isdir(backup_root):
                 raise HedFileError('BadBackupPath', f"{backup_root} is not a backup directory.", "")
             if len(os.listdir(backup_root)) != 2:
@@ -195,12 +199,12 @@ def _check_backup_consistency(self, backup_name):
 
         """
 
-        backup_dict_path = os.path.realpath(os.path.join(self.backups_root, backup_name, self.BACKUP_DICTIONARY))
+        backup_dict_path = os.path.realpath(os.path.join(self.backups_path, backup_name, self.BACKUP_DICTIONARY))
         if not os.path.exists(backup_dict_path):
             raise HedFileError("BadBackupDictionaryPath",
                                f"Backup dictionary path {backup_dict_path} for backup "
                                f"{backup_name} does not exist so backup invalid", "")
-        backup_root_path = os.path.realpath(os.path.join(self.backups_root, backup_name, self.BACKUP_ROOT))
+        backup_root_path = os.path.realpath(os.path.join(self.backups_path, backup_name, self.BACKUP_ROOT))
         if not os.path.isdir(backup_root_path):
             raise HedFileError("BadBackupRootPath",
                                f"Backup root path {backup_root_path} for {backup_name} "

hed/tools/remodeling/cli/run_remodel.py

@@ -12,10 +12,10 @@
 
 def get_parser():
     """ Create a parser for the run_remodel command-line arguments. 
-    
+
     Returns:
         argparse.ArgumentParser:  A parser for parsing the command line arguments.
-        
+
     """
     parser = argparse.ArgumentParser(description="Converts event files based on a json file specifying operations.")
     parser.add_argument("data_dir", help="Full path of dataset root directory.")
@@ -33,13 +33,17 @@ def get_parser():
                         help="Optional path to JSON sidecar with HED information")
     parser.add_argument("-n", "--backup-name", default=BackupManager.DEFAULT_BACKUP_NAME, dest="backup_name",
                         help="Name of the default backup for remodeling")
+    parser.add_argument("-nb", "--no-backup", action='store_true', dest="no_backup",
+                        help="If present, the operations are run directly on the files with no backup.")
     parser.add_argument("-r", "--hed-versions", dest="hed_versions", nargs="*", default=[],
                         help="Optional list of HED schema versions used for annotation, include prefixes.")
     parser.add_argument("-s", "--save-formats", nargs="*", default=['.json', '.txt'], dest="save_formats",
                         help="Format for saving any summaries, if any. If empty, then no summaries are saved.")
     parser.add_argument("-t", "--task-names", dest="task_names", nargs="*", default=[], help="The names of the task.")
     parser.add_argument("-v", "--verbose", action='store_true',
                         help="If present, output informative messages as computation progresses.")
+    parser.add_argument("-w", "--work-dir", default="", dest="work_dir",
+                        help="If given, is the path to directory for saving, otherwise derivatives/remodel is used.")
     parser.add_argument("-x", "--exclude-dirs", nargs="*", default=[], dest="exclude_dirs",
                         help="Directories names to exclude from search for files.")
     return parser
@@ -148,19 +152,24 @@ def main(arg_list=None):
     args, operations = parse_arguments(arg_list)
     if not os.path.isdir(args.data_dir):
         raise HedFileError("DataDirectoryDoesNotExist", f"The root data directory {args.data_dir} does not exist", "")
-    if args.backup_name:
+    if args.no_backup:
+        backup_name = None
+    else:
         backup_man = BackupManager(args.data_dir)
         if not backup_man.get_backup(args.backup_name):
             raise HedFileError("BackupDoesNotExist", f"Backup {args.backup_name} does not exist. "
                                f"Please run_remodel_backup first", "")
         backup_man.restore_backup(args.backup_name, args.task_names, verbose=args.verbose)
-    dispatch = Dispatcher(operations, data_root=args.data_dir, backup_name=args.backup_name,
-                          hed_versions=args.hed_versions)
+        backup_name = args.backup_name
+    dispatch = Dispatcher(operations, data_root=args.data_dir, backup_name=backup_name, hed_versions=args.hed_versions)
     if args.use_bids:
         run_bids_ops(dispatch, args)
     else:
         run_direct_ops(dispatch, args)
-    dispatch.save_summaries(args.save_formats, individual_summaries=args.individual_summaries)
+    save_dir = None
+    if args.work_dir:
+        save_dir = os.path.realpath(os.path.join(args.work_dir, Dispatcher.REMODELING_SUMMARY_PATH))
+    dispatch.save_summaries(args.save_formats, individual_summaries=args.individual_summaries, summary_dir=save_dir)
 
 
 if __name__ == '__main__':

hed/tools/remodeling/cli/run_remodel_backup.py

@@ -21,13 +21,19 @@ def get_parser():
                         help="Filename suffix of files to be backed up. A * indicates all files allowed.")
     parser.add_argument("-n", "--backup_name", default=BackupManager.DEFAULT_BACKUP_NAME, dest="backup_name",
                         help="Name of the default backup for remodeling")
+    parser.add_argument("-p", "--path-work", default="", dest="path_work",
+                        help="The root path for remodeling work if given, " +
+                             "otherwise [data_root]/derivatives/remodel is used.")
     parser.add_argument("-t", "--task-names", dest="task_names", nargs="*", default=[], help="The name of the task.")
     parser.add_argument("-v", "--verbose", action='store_true',
                         help="If present, output informative messages as computation progresses.")
+    parser.add_argument("-w", "--work-dir", default="", dest="work_dir",
+                        help="If given, is the path to directory for saving, " +
+                             "otherwise [data_root]derivatives/remodel is used.")
     parser.add_argument("-x", "--exclude-dirs", nargs="*", default=['derivatives'], dest="exclude_dirs",
                         help="Directories names to exclude from search for files. " +
                              "If omitted, no directories except the backup directory will be excluded." +
-                             "Note data_dir/remodel/backup will always be excluded.")
+                             "Note [data_root]/derivatives/remodel will always be excluded.")
     return parser
 
 
@@ -55,7 +61,11 @@ def main(arg_list=None):
                               exclude_dirs=exclude_dirs)
     if args.task_names:
         file_list = get_filtered_by_element(file_list, args.task_names)
-    backup_man = BackupManager(args.data_dir)
+    if args.work_dir:
+        backups_root = args.work_dir
+    else:
+        backups_root = None
+    backup_man = BackupManager(args.data_dir, backups_root=backups_root)
     if backup_man.get_backup(args.backup_name):
         raise HedFileError("BackupExists", f"Backup {args.backup_name} already exists", "")
     else:

hed/tools/remodeling/cli/run_remodel_restore.py

@@ -16,9 +16,13 @@ def get_parser():
     parser.add_argument("data_dir", help="Full path of dataset root directory.")
     parser.add_argument("-n", "--backup_name", default=BackupManager.DEFAULT_BACKUP_NAME, dest="backup_name",
                         help="Name of the default backup for remodeling")
+
     parser.add_argument("-t", "--task-names", dest="task_names", nargs="*", default=[], help="The names of the task.")
     parser.add_argument("-v", "--verbose", action='store_true',
                         help="If present, output informative messages as computation progresses.")
+    parser.add_argument("-w", "--work_dir", default="", dest="work_dir",
+                        help="The root path for remodeling work if given, " +
+                             "otherwise [data_root]/derivatives/remodel is used.")
     return parser
 
 
@@ -36,7 +40,11 @@ def main(arg_list=None):
     """
     parser = get_parser()
     args = parser.parse_args(arg_list)
-    backup_man = BackupManager(args.data_dir)
+    if args.work_dir:
+        backups_root = args.work_dir
+    else:
+        backups_root = None
+    backup_man = BackupManager(args.data_dir, backups_root=backups_root)
     if not backup_man.get_backup(args.backup_name):
         raise HedFileError("BackupDoesNotExist", f"{args.backup_name}", "")
     backup_man.restore_backup(args.backup_name, task_names=args.task_names, verbose=args.verbose)

hed/tools/remodeling/dispatcher.py

@@ -14,7 +14,7 @@
 class Dispatcher:
     """ Controller for applying operations to tabular files and saving the results. """
 
-    REMODELING_SUMMARY_PATH = 'derivatives/remodel/summaries'
+    REMODELING_SUMMARY_PATH = 'remodel/summaries'
 
     def __init__(self, operation_list, data_root=None,
                  backup_name=BackupManager.DEFAULT_BACKUP_NAME, hed_versions=None):
@@ -128,7 +128,7 @@ def get_summary_save_dir(self):
         """
 
         if self.data_root:
-            return os.path.realpath(os.path.join(self.data_root, Dispatcher.REMODELING_SUMMARY_PATH))
+            return os.path.realpath(os.path.join(self.data_root, 'derivatives', Dispatcher.REMODELING_SUMMARY_PATH))
         raise HedFileError("NoDataRoot", f"Dispatcher must have a data root to produce directories", "")
 
     def run_operations(self, file_path, sidecar=None, verbose=False):
@@ -160,9 +160,10 @@ def save_summaries(self, save_formats=['.json', '.txt'], individual_summaries="s
         Parameters:
             save_formats (list):  A list of formats [".txt", ."json"]
             individual_summaries (str): If True, include summaries of individual files.
-            summary_dir (str or None): Directory for saving summaries
+            summary_dir (str or None): Directory for saving summaries.
 
-        The summaries are saved in the dataset derivatives/remodeling folder if no save_dir is provided.
+        Notes:
+            The summaries are saved in the dataset derivatives/remodeling folder if no save_dir is provided.
 
         """
         if not save_formats:

hed/tools/remodeling/operations/base_context.py

@@ -62,12 +62,12 @@ def get_summary(self, individual_summaries="separate"):
 
         Returns:
             dict - dictionary with "Dataset" and "Individual files" keys.
-            
+
         Notes: The individual_summaries value is processed as follows
            -  "separate" individual summaries are to be in separate files
            -  "consolidated" means that the individual summaries are in same file as overall summary
            -  "none" means that only the overall summary is produced.
-            
+
         """
         include_individual = individual_summaries == "separate" or individual_summaries == "consolidated"
         summary_details = self.get_summary_details(include_individual=include_individual)

hed/tools/remodeling/operations/summarize_definitions_op.py

@@ -105,6 +105,15 @@ def update_context(self, new_context):
         self.errors.update(errors)
 
     def _get_details_dict(self, summary):
+        """ Return the summary-specific information in a dictionary.
+
+        Parameters:
+            summary (?):  Contains the resolved dictionaries.
+
+        Returns:
+            dict: dictionary with the summary results.
+
+        """
         return None
 
     def _merge_all(self):

hed/tools/remodeling/operations/summarize_hed_tags_op.py

@@ -117,6 +117,15 @@ def update_context(self, new_context):
         self.summary_dict[new_context["name"]] = counts
 
     def _get_details_dict(self, merge_counts):
+        """ Return the summary-specific information in a dictionary.
+
+        Parameters:
+            merge_counts (HedTagCounts):  Contains the counts of tags in the dataset.
+
+        Returns:
+            dict: dictionary with the summary results.
+
+        """
         template, unmatched = merge_counts.organize_tags(self.tags)
         details = {}
         for key, key_list in self.tags.items():

hed/tools/remodeling/operations/summarize_hed_type_op.py

@@ -113,6 +113,15 @@ def update_context(self, new_context):
         self.summary_dict[new_context["name"]] = counts
 
     def _get_details_dict(self, counts):
+        """ Return the summary-specific information in a dictionary.
+
+        Parameters:
+            counts (HedTypeCounts):  Contains the counts of the events in which the type occurs.
+
+        Returns:
+            dict: dictionary with the summary results.
+
+        """
         return counts.get_summary()
 
     def _merge_all(self):

tests/tools/remodeling/cli/test_run_remodel.py

@@ -36,6 +36,9 @@ def setUp(self):
 
     def tearDown(self):
         shutil.rmtree(self.data_root)
+        work_path = os.path.realpath(os.path.join(self.extract_path, 'temp'))
+        if os.path.exists(work_path):
+            shutil.rmtree(work_path)
 
     @classmethod
     def tearDownClass(cls):
@@ -73,6 +76,15 @@ def test_main_bids(self):
             main(arg_list)
             self.assertFalse(fp.getvalue())
 
+    def test_main_bids_alt_path(self):
+        work_path = os.path.realpath(os.path.join(self.extract_path, 'temp'))
+        arg_list = [self.data_root, self.summary_model_path, '-x', 'derivatives', 'stimuli', '-r', '8.1.0',
+                   '-j', self.sidecar_path, '-w', work_path]
+
+        with patch('sys.stdout', new=io.StringIO()) as fp:
+            main(arg_list)
+            self.assertFalse(fp.getvalue())
+
     def test_main_bids_verbose_bad_task(self):
         arg_list = [self.data_root, self.model_path, '-x', 'derivatives', 'stimuli', '-b', '-t', 'junk', '-v']
         with patch('sys.stdout', new=io.StringIO()) as fp: