Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion domainlab/exp/exp_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from domainlab.tasks.zoo_tasks import TaskChainNodeGetter
from domainlab.utils.logger import Logger
from domainlab.utils.sanity_check import SanityCheck

os.environ["CUDA_LAUNCH_BLOCKING"] = "1" # debug


Expand Down Expand Up @@ -57,6 +56,7 @@ def __init__(self, args, task=None, model=None, observer=None, visitor=AggWriter
self.trainer.init_business(self.model, self.task, observer, device, args)
self.visitor = visitor(self) # visitor depends on task initialization first
# visitor must be initialized last after trainer is initialized
self.experiment_duration = None
self.model.set_saver(self.visitor)

def execute(self, num_epochs=None):
Expand Down Expand Up @@ -96,6 +96,7 @@ def execute(self, num_epochs=None):
f"Experiment finished at epoch: {self.epoch_counter} "
f"with time: {t_c - t_0} at {t_c}"
)
self.experiment_duration = t_c - t_0
self.trainer.post_tr()

def clean_up(self):
Expand Down
5 changes: 5 additions & 0 deletions domainlab/exp/exp_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,11 @@ def _gen_line(self, dict_metric):
confmat_filename = dict_cols.get("mname", None) # return None if not found
# @FIXME: strong dependency on host attribute name
dict_cols.update({epos_name: self.host.epoch_counter})
if self.host.experiment_duration is not None:
dict_cols.update({"experiment_duration": self.host.experiment_duration})
else:
print("Does not have attribute")
print(self.host)
if not self.has_first_line:
self.first_line(dict_cols)
list_str = [str(dict_cols[key]) for key in self.list_cols]
Expand Down