Sync chenxh

README.md

@@ -213,8 +213,8 @@ Result visualization:
       We can view the results in tensorboard by: `tensorboard --logdir ${data_root}/log/${task_name}`.
       For example, lanuch tensorboard by `tensorboard --logdir ./example/simplest_code/log/demo_task/2022/03`. We can see results:
         ![img.png](resource/demo-tb-res.png)
-   2. Easy_plot toolkit: The intermediate scalar variables are saved in a CSV file in `${data_root}/log/${task_name}/${index_name}/progress.csv`. 
-      We develop high-level APIs to load the CSV files from multiple experiments and group the lines by custom keys. We give an example to use easy_plot toolkit in https://github.com/xionghuichen/RLAssistant/blob/main/example/plot_res.ipynb and more user cases in https://github.com/xionghuichen/RLAssistant/blob/main/test/test_plot.py
+   2. Easy_plot toolkit: 
+       **We recommend the users maintain their research projects via some jupyter notebooks. You can record your ideas, surmises, related empirical evidence, and benchmark results in a notebook together.**  We develop high-level APIs to load the CSV files of the experiments (stored in `${data_root}/log/${task_name}/${index_name}/progress.csv`) and group the curves by custom keys.    We give common user cases of the plotter in https://github.com/xionghuichen/RLAssistant/blob/main/test/test_plot.ipynb
       The result will be something like this:
         ![img.png](resource/demo-easy-to-plot-res.png)
    3. View data in "results" directory directly: other type of data are stored in `${data_root}/results/${task_name}/${index_name}`

RLA/easy_log/logger.py

@@ -178,6 +178,7 @@ def writekvs(self, kvs):
             lines = self.file.readlines()
             self.file = open(self.filename, 'w+t')
             self.file.seek(0)
+
             for (i, key) in enumerate(self.keys):
                 if i > 0:
                     self.file.write(',')

RLA/easy_log/time_used_recorder.py

@@ -3,9 +3,78 @@
 from RLA.easy_log import logger
 import time
 
+class SingleTimeTracker:
+    def __init__(self,name:str='untitled') -> None:
+        self.name=name
+        self.t = 0.0
+        self.call_time=0
+        self.time_cost = 0.0
 
-rc_start_time = {}
+    def __enter__(self):
+        # trace time
+        self.call_time+=1            
+        self.t = time.perf_counter()
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.time_cost += time.perf_counter() - self.t
+
+
+class TimeTracker:
+    def __init__(self):
+        self.t0=time.time()#to calc total time
+        self.time_dict=dict()
+
+    def add(self,name='untitled'):
+        """
+        :param name: specify the SingleTimeTracker in the time_dict, recommend use
+        line num in the scripts, like 'xxx.py Line xxx', can be easily got in python scripts using 
+        `os.path.basename(__file__)+'  line '+str(sys._getframe().f_lineno)`  
+        """
+        if name not in self.time_dict.keys():
+            self.time_dict.update({name:SingleTimeTracker(name)})
+        return self.time_dict[name]
+
+    def __call__(self, name:str):
+        return self.time_dict[name]
+
+    def clear(self):
+        self.time_dict=dict()
 
+    def statistic_entry(self,name:str):
+        """
+        calc total calls of the 
+        """
+        assert name in self.time_dict.keys()
+
+        t1=time.time()
+        t_passed=t1-self.t0
+        return {
+            'total calls/'+name:self.time_dict[name].call_time,
+            'total time cost/'+name:self.time_dict[name].time_cost,
+            'average time cost/'+name:self.time_dict[name].time_cost/(1e-6+self.time_dict[name].call_time),
+            'time cost percentage/'+name:self.time_dict[name].time_cost/(1e-6+t_passed)
+        }
+
+    def get_info(self):
+        info={}
+        for k in self.time_dict.keys():
+            for entry_k,entry_v in self.statistic_entry(k).items():
+                info[entry_k]=entry_v
+        return info
+
+    def log(self,exclude_lst=['csv']):
+        logger.info('---------time dashboard---------')
+        for k in self.time_dict.keys():
+            for entry_k,entry_v in self.statistic_entry(k).items():
+                logger.record_tabular('time_used/'+entry_k,entry_v,exclude=exclude_lst)
+                logger.info(f"[{entry_k}]: {entry_v}")
+            logger.info('')
+        logger.info('---------dashboard end---------')
+
+
+rc_start_time = {}
 
 def time_record(name):
     """

RLA/easy_plot/plot_func_v2.py

@@ -18,9 +18,12 @@
 
 
 
-def default_key_to_legend(parse_dict, split_keys, y_name):
+def default_key_to_legend(parse_dict, split_keys, y_name, use_y_name=True):
     task_split_key = '.'.join(f'{k}={parse_dict[k]}' for k in split_keys)
-    return task_split_key + ' eval:' + y_name
+    if use_y_name:
+        return task_split_key + ' eval:' + y_name
+    else:
+        return task_split_key
 
 
 def plot_func(data_root:str, task_table_name:str, regs:list, split_keys:list, metrics:list,
@@ -29,6 +32,7 @@ def plot_func(data_root:str, task_table_name:str, regs:list, split_keys:list, me
               xlabel: Optional[str] = DEFAULT_X_NAME, ylabel: Optional[str] = None,
               scale_dict: Optional[dict] = None, regs2legends: Optional[list] = None,
               key_to_legend_fn: Optional[Callable] = default_key_to_legend,
+              split_by_metrics=True,
               save_name: Optional[str] = None, *args, **kwargs):
     """
     A high-level matplotlib plotter.
@@ -68,8 +72,11 @@ def plot_func(data_root:str, task_table_name:str, regs:list, split_keys:list, me
     :param key_to_legend_fn: we give a default function to stringify the k-v pairs. you can customize your own function in key_to_legend_fn.
     See default_key_to_legend for the detault way and test/test_plot/test_customize_legend_name_mode for details.
     :type key_to_legend_fn: Optional[Callable] = default_key_to_legend
+    :param split_by_metrics: you can plot figure with multiple metrics together.
+    By default, we will split the curves with the metric and merge them into a group figure.
+    If you would like to print multiple metrics in single figure, please set the parameter to False.
+    :type split_by_metrics: Optional[bool]
     :param args/kwargs: send other parameters to plot_util.plot_results
-
     :return:
     :rtype:
     """
@@ -112,10 +119,12 @@ def plot_func(data_root:str, task_table_name:str, regs:list, split_keys:list, me
         group_fn = lambda r: split_by_reg(taskpath=r, reg_group=reg_group, y_names=y_names)
     else:
         group_fn = lambda r: picture_split(taskpath=r, split_keys=split_keys, y_names=y_names,
-                                           key_to_legend_fn=key_to_legend_fn)
+                                           key_to_legend_fn=lambda parse_dict, split_keys, y_name:
+                                           key_to_legend_fn(parse_dict, split_keys, y_name, not split_by_metrics))
     _, _, lgd, texts, g2lf, score_results = \
         plot_util.plot_results(results, xy_fn= lambda r, y_names: csv_to_xy(r, DEFAULT_X_NAME, y_names, final_scale_dict),
-                           group_fn=group_fn, average_group=True, ylabel=ylabel, xlabel=xlabel, regs2legends=regs2legends, *args, **kwargs)
+                           group_fn=group_fn, average_group=True, ylabel=ylabel, xlabel=xlabel, metrics=metrics,
+                               split_by_metrics=split_by_metrics, regs2legends=regs2legends, *args, **kwargs)
     print("--- complete process ---")
     if save_name is not None:
         import os
@@ -141,7 +150,6 @@ def split_by_reg(taskpath, reg_group, y_names):
     assert len(y_names) == 1
     return task_split_key, y_names
 
-
 def split_by_task(taskpath, split_keys, y_names, key_to_legend_fn):
     pair_delimiter = '&'
     kv_delimiter = '='