From 4e98ae2b4a8830685e4ca5115e94d59cdb2e4875 Mon Sep 17 00:00:00 2001 From: Jake Tae Date: Sat, 14 Aug 2021 03:00:51 +0900 Subject: [PATCH 1/5] feat: allow tracker pause and restart --- codecarbon/emissions_tracker.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index bd9cfa84d..ea0909ae7 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -302,7 +302,7 @@ def start(self) -> None: self._scheduler.start() @suppress(Exception) - def stop(self) -> Optional[float]: + def _stop(self, destroy: bool) -> Optional[float]: """ Stops tracking the experiment :return: CO2 emissions in kgs @@ -311,7 +311,10 @@ def stop(self) -> Optional[float]: logger.error("Need to first start the tracker") return None - self._scheduler.shutdown() + if destroy: + self._scheduler.shutdown() + else: + self._scheduler.pause() # Run to calculate the power used from last # scheduled measurement to shutdown @@ -328,6 +331,32 @@ def stop(self) -> Optional[float]: self.final_emissions = emissions_data.emissions return emissions_data.emissions + @suppress(Exception) + def pause(self): + """ + Temporarily halts tracking the experiment. + Logs intermediate experiment tracking results. + :return: CO2 emissions in kgs + """ + return self._stop(destroy=False) + + @suppress(Exception) + def restart(self): + """ + Restarts + """ + logger.info("Restarting tracker") + self._last_measured_time = self._start_time = time.time() + self._scheduler.resume() + + @suppress(Exception) + def stop(self) -> Optional[float]: + """ + Stops tracking the experiment + :return: CO2 emissions in kgs + """ + return self._stop(destroy=True) + def _prepare_emissions_data(self, delta=False) -> EmissionsData: """ :delta: True to return only the delta comsumption since last call From 58ea1145765d16e148b2d45fa94205322c822676 Mon Sep 17 00:00:00 2001 From: Jake Tae Date: Sat, 14 Aug 2021 04:00:45 +0900 Subject: [PATCH 2/5] fix: prevent csv files from being overrideen --- codecarbon/emissions_tracker.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index ea0909ae7..48f0ed7e3 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -214,6 +214,7 @@ def __init__( self._measure_occurrence: int = 0 self._cloud = None self._previous_emissions = None + self.pause_count = 0 if isinstance(self._gpu_ids, str): self._gpu_ids = parse_gpu_ids(self._gpu_ids) @@ -338,10 +339,20 @@ def pause(self): Logs intermediate experiment tracking results. :return: CO2 emissions in kgs """ + # check if csv already exists + if self._save_to_file and os.path.exists( + os.path.join(self._output_dir, self._output_file) + ): + self.pause_count += 1 + filename, extension = os.path.splitext(self._output_file) + new_filename = f"{filename}_{self.pause_count}" + self.persistence_objs[0] = FileOutput( + os.path.join(self._output_dir, f"{new_filename}{extension}") + ) return self._stop(destroy=False) @suppress(Exception) - def restart(self): + def resume(self): """ Restarts """ From 484981b3ed662f4ef0a238da01278b92be0c4097 Mon Sep 17 00:00:00 2001 From: Jake Tae Date: Sat, 14 Aug 2021 04:06:53 +0900 Subject: [PATCH 3/5] fix: prevent `.stop()` from overriding first dump --- codecarbon/emissions_tracker.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 48f0ed7e3..2458b6d9e 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -317,6 +317,16 @@ def _stop(self, destroy: bool) -> Optional[float]: else: self._scheduler.pause() + if self._save_to_file and os.path.exists( + os.path.join(self._output_dir, self._output_file) + ): + self.pause_count += 1 + filename, extension = os.path.splitext(self._output_file) + new_filename = f"{filename}_{self.pause_count}" + self.persistence_objs[0] = FileOutput( + os.path.join(self._output_dir, f"{new_filename}{extension}") + ) + # Run to calculate the power used from last # scheduled measurement to shutdown self._measure_power() @@ -340,15 +350,6 @@ def pause(self): :return: CO2 emissions in kgs """ # check if csv already exists - if self._save_to_file and os.path.exists( - os.path.join(self._output_dir, self._output_file) - ): - self.pause_count += 1 - filename, extension = os.path.splitext(self._output_file) - new_filename = f"{filename}_{self.pause_count}" - self.persistence_objs[0] = FileOutput( - os.path.join(self._output_dir, f"{new_filename}{extension}") - ) return self._stop(destroy=False) @suppress(Exception) From 4cdda1907c8540b188c59bc7836586c3eb9d9368 Mon Sep 17 00:00:00 2001 From: Jake Tae Date: Sat, 14 Aug 2021 04:14:09 +0900 Subject: [PATCH 4/5] docs: complete docstring sentence --- codecarbon/emissions_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 2458b6d9e..ceb3adbbb 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -355,7 +355,7 @@ def pause(self): @suppress(Exception) def resume(self): """ - Restarts + Resumes previously paused tracking """ logger.info("Restarting tracker") self._last_measured_time = self._start_time = time.time() From ccac4eb252f5cc5f44f5bd35ad54f7708b4015bb Mon Sep 17 00:00:00 2001 From: Jake Tae Date: Sat, 14 Aug 2021 15:44:36 +0900 Subject: [PATCH 5/5] fix: remove unnecessary csv dumping --- codecarbon/emissions_tracker.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index ceb3adbbb..9d57f0307 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -214,7 +214,6 @@ def __init__( self._measure_occurrence: int = 0 self._cloud = None self._previous_emissions = None - self.pause_count = 0 if isinstance(self._gpu_ids, str): self._gpu_ids = parse_gpu_ids(self._gpu_ids) @@ -317,16 +316,6 @@ def _stop(self, destroy: bool) -> Optional[float]: else: self._scheduler.pause() - if self._save_to_file and os.path.exists( - os.path.join(self._output_dir, self._output_file) - ): - self.pause_count += 1 - filename, extension = os.path.splitext(self._output_file) - new_filename = f"{filename}_{self.pause_count}" - self.persistence_objs[0] = FileOutput( - os.path.join(self._output_dir, f"{new_filename}{extension}") - ) - # Run to calculate the power used from last # scheduled measurement to shutdown self._measure_power() @@ -349,7 +338,6 @@ def pause(self): Logs intermediate experiment tracking results. :return: CO2 emissions in kgs """ - # check if csv already exists return self._stop(destroy=False) @suppress(Exception)