diff --git a/RLA/easy_log/log_tools.py b/RLA/easy_log/log_tools.py index 87654ef..a9b704e 100644 --- a/RLA/easy_log/log_tools.py +++ b/RLA/easy_log/log_tools.py @@ -161,7 +161,7 @@ def delete_related_log(self, skip_ask=False, delete_log_types=None): else: s = input("delete these files? (y/n)") if s == 'y': - print("do delete ...") + print("deleting ...") return self._delete_related_log(show=False, regex=self.regex, delete_log_types=delete_log_types) else: return 0 @@ -189,10 +189,11 @@ def delete_small_timestep_log(self, skip_ask=False): s = input("delete these files? (y/n)") if s == 'y' or skip_ask: for res in self.small_timestep_regs: - print("do delete: ", res[1]) + print("deleting: ", res[1]) log_found += self._delete_related_log(show=False, regex=res[0] + '*') return log_found + class ArchiveLogTool(BasicLogTool): def __init__(self, proj_root, task_table_name, regex, archive_table_name=ARCHIVED_TABLE, *args, **kwargs): self.proj_root = proj_root @@ -241,10 +242,64 @@ def archive_log(self, skip_ask=False): else: s = input("archive these files? (y/n) \n ") if s == 'y': - print("do archive ...") + print("archiving ...") + self._archive_log(show=False) + + + +class MoveLogTool(BasicLogTool): + def __init__(self, proj_root, task_table_name, regex, target_task_table_name, *args, **kwargs): + self.proj_root = proj_root + self.task_table_name = task_table_name + self.regex = regex + self.target_task_table_name = target_task_table_name + super(MoveLogTool, self).__init__(*args, **kwargs) + + def _archive_log(self, show=False): + for log_type in self.log_types: + root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex) + target_root_dir = osp.join(self.proj_root, log_type, self.target_task_table_name) + prefix_dir = osp.join(self.proj_root, log_type) + prefix_len = len(prefix_dir) + empty = True + # os.system("chmod +x -R \"{}\"".format(prefix_dir)) + for root_dir in glob.glob(root_dir_regex): + empty = False + if os.path.exists(root_dir): + # remove the overlapped path. + archiving_target = osp.join(target_root_dir, root_dir[prefix_len+1:]) + archiving_target_dir = '/'.join(archiving_target.split('/')[:-1]) + os.makedirs(archiving_target_dir, exist_ok=True) + if os.path.isdir(root_dir): + if not show: + # os.makedirs(archiving_target, exist_ok=True) + try: + shutil.copytree(root_dir, archiving_target) + except FileExistsError as e: + print(e) + + print("copy dir {}, to {}".format(root_dir, archiving_target)) + else: + if not show: + shutil.copy(root_dir, archiving_target) + print("copy file {}, to {}".format(root_dir, archiving_target)) + else: + print("no dir {}".format(root_dir)) + if empty: print("empty regex {}".format(root_dir_regex)) + pass + + def archive_log(self, skip_ask=False): + self._archive_log(show=True) + if skip_ask: + s = 'y' + else: + s = input("archive these files? (y/n) \n ") + if s == 'y': + print("moving ...") self._archive_log(show=False) + class ViewLogTool(BasicLogTool): def __init__(self, proj_root, task_table_name, regex, *args, **kwargs): self.proj_root = proj_root diff --git a/RLA/easy_plot/plot_util.py b/RLA/easy_plot/plot_util.py index c580b74..f281e29 100644 --- a/RLA/easy_plot/plot_util.py +++ b/RLA/easy_plot/plot_util.py @@ -222,10 +222,10 @@ def load_results(root_dir_or_dirs, names, x_bound, enable_progress=True, use_buf print("read buf: {}".format(buf_csv)) raw_df = read_csv(buf_csv) else: - reader = pd.read_csv(progcsv, chunksize=100000, quoting=csv.QUOTE_NONE, - encoding='utf-8', index_col=False, comment='#') - raw_df = pd.DataFrame() - + reader = pd.read_csv(progcsv, chunksize=5000, quoting=csv.QUOTE_NONE, + encoding='utf-8', index_col=False, comment='#', memory_map=True) + # raw_df = pd.DataFrame() + slim_chunk_list = [] for chunk in reader: slim_chunk = chunk # if set(names).issubset(slim_chunk.columns): @@ -244,7 +244,8 @@ def load_results(root_dir_or_dirs, names, x_bound, enable_progress=True, use_buf slim_chunk[x_bound[0]] > x_bound[1][0])] else: slim_chunk = slim_chunk[slim_chunk[x_bound[0]] < x_bound[1]] - raw_df = pd.concat([raw_df, slim_chunk], ignore_index=True) + slim_chunk_list.append(slim_chunk) + raw_df = pd.concat(slim_chunk_list, ignore_index=True) import csv raw_df.to_csv(buf_csv, index=False) result['progress'] = raw_df diff --git a/rla_scripts/archive_expt.py b/rla_scripts/archive_expt.py index 38df571..4607f62 100644 --- a/rla_scripts/archive_expt.py +++ b/rla_scripts/archive_expt.py @@ -9,7 +9,7 @@ from RLA.easy_log.log_tools import ArchiveLogTool import argparse -from config import * +from rla_script_config import * def argsparser(): parser = argparse.ArgumentParser("Archive Log") diff --git a/rla_scripts/config.py b/rla_scripts/config.py deleted file mode 100644 index 6899baf..0000000 --- a/rla_scripts/config.py +++ /dev/null @@ -1 +0,0 @@ -DATA_ROOT = '../example/simplest_code/' diff --git a/rla_scripts/delete_expt.py b/rla_scripts/delete_expt.py index c25c1f4..196c041 100644 --- a/rla_scripts/delete_expt.py +++ b/rla_scripts/delete_expt.py @@ -4,7 +4,7 @@ """ from RLA.easy_log.log_tools import DeleteLogTool, Filter import argparse -from config import * +from rla_script_config import * def argsparser(): parser = argparse.ArgumentParser("Delete Log") diff --git a/rla_scripts/migrate_expt.py b/rla_scripts/migrate_expt.py new file mode 100644 index 0000000..d5e66cc --- /dev/null +++ b/rla_scripts/migrate_expt.py @@ -0,0 +1,26 @@ +# Created by xionghuichen at 2022/12/6 +# Email: chenxh@lamda.nju.edu.cn +""" +A script to move some important experiments to another task_table + + +""" +from rla_script_config import * +from RLA.easy_log.log_tools import MoveLogTool +import argparse + +def argsparser(): + parser = argparse.ArgumentParser("Archive Log") + # reduce setting + parser.add_argument('--task_table', type=str) + parser.add_argument('--target_task_table', type=str) + parser.add_argument('--regex', type=str) + + args = parser.parse_args() + return args + +if __name__=='__main__': + args = argsparser() + dlt = MoveLogTool(proj_root=DATA_ROOT, task_table_name=args.task_table, regex=args.regex, + target_task_table_name=args.target_task_table) + dlt.archive_log() \ No newline at end of file diff --git a/rla_scripts/rla_script_config.py b/rla_scripts/rla_script_config.py new file mode 100755 index 0000000..c0e5249 --- /dev/null +++ b/rla_scripts/rla_script_config.py @@ -0,0 +1 @@ +DATA_ROOT = '../../out_debug' diff --git a/rla_scripts/start_pretty_plotter.py b/rla_scripts/start_pretty_plotter.py index 8217d2d..13d6b6d 100644 --- a/rla_scripts/start_pretty_plotter.py +++ b/rla_scripts/start_pretty_plotter.py @@ -7,7 +7,7 @@ from RLA.easy_log.log_tools import PrettyPlotterTool, Filter import argparse from RLA.rla_argparser import boolean_flag -from config import * +from rla_script_config import * from smart_logger.front_page.page import start_page_server import smart_logger.common.plot_config as plot_config diff --git a/rla_scripts/view_expt.py b/rla_scripts/view_expt.py index ab24250..3e4ca91 100644 --- a/rla_scripts/view_expt.py +++ b/rla_scripts/view_expt.py @@ -4,7 +4,7 @@ from RLA.easy_log.log_tools import ViewLogTool import argparse -from config import * +from rla_script_config import * def argsparser(): parser = argparse.ArgumentParser("View Log") diff --git a/setup.py b/setup.py index b8dc602..676248d 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='RLA', - version="0.6.0", + version="0.6.1", description=( 'RL assistant' ),