From acc9f47f6a6a7118eb2d1c728f0ede81dc668704 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Dec 2020 18:41:09 +0800 Subject: [PATCH 1/2] [DLMED] enhance save_final in CheckpointSaver Signed-off-by: Nic Ma --- monai/handlers/checkpoint_saver.py | 10 ++++++++++ tests/test_handler_checkpoint_saver.py | 1 + 2 files changed, 11 insertions(+) diff --git a/monai/handlers/checkpoint_saver.py b/monai/handlers/checkpoint_saver.py index 57d8728cd4..f0b0cf7ea8 100644 --- a/monai/handlers/checkpoint_saver.py +++ b/monai/handlers/checkpoint_saver.py @@ -188,6 +188,12 @@ def attach(self, engine: Engine) -> None: else: engine.add_event_handler(Events.ITERATION_COMPLETED(every=self.save_interval), self.interval_completed) + def _delete_previous_final_ckpt(self): + saved = self._final_checkpoint._saved + if len(saved) > 0: + item = saved.pop(0) + self._final_checkpoint.save_handler.remove(item.filename) + def completed(self, engine: Engine) -> None: """Callback for train or validation/evaluation completed Event. Save final checkpoint if configure save_final is True. @@ -196,6 +202,8 @@ def completed(self, engine: Engine) -> None: engine: Ignite Engine, it can be a trainer, validator or evaluator. """ assert callable(self._final_checkpoint), "Error: _final_checkpoint function not specified." + # delete previous saved final checkpoint if existing + self._delete_previous_final_ckpt() self._final_checkpoint(engine) assert self.logger is not None assert hasattr(self.logger, "info"), "Error, provided logger has not info attribute." @@ -211,6 +219,8 @@ def exception_raised(self, engine: Engine, e: Exception) -> None: e: the exception caught in Ignite during engine.run(). """ assert callable(self._final_checkpoint), "Error: _final_checkpoint function not specified." + # delete previous saved final checkpoint if existing + self._delete_previous_final_ckpt() self._final_checkpoint(engine) assert self.logger is not None assert hasattr(self.logger, "info"), "Error, provided logger has not info attribute." diff --git a/tests/test_handler_checkpoint_saver.py b/tests/test_handler_checkpoint_saver.py index 8513b6625e..2df36d9720 100644 --- a/tests/test_handler_checkpoint_saver.py +++ b/tests/test_handler_checkpoint_saver.py @@ -117,6 +117,7 @@ def _train_func(engine, batch): n_saved, ) handler.attach(engine) + engine.run(data, max_epochs=2) engine.run(data, max_epochs=5) for filename in filenames: self.assertTrue(os.path.exists(os.path.join(tempdir, filename))) From 5d1293ff27d5ac62158859d96737650de3b5adb4 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Thu, 10 Dec 2020 20:02:17 +0800 Subject: [PATCH 2/2] [DLMED] add log message Signed-off-by: Nic Ma --- monai/handlers/checkpoint_saver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monai/handlers/checkpoint_saver.py b/monai/handlers/checkpoint_saver.py index f0b0cf7ea8..0cc05b2dc4 100644 --- a/monai/handlers/checkpoint_saver.py +++ b/monai/handlers/checkpoint_saver.py @@ -193,6 +193,7 @@ def _delete_previous_final_ckpt(self): if len(saved) > 0: item = saved.pop(0) self._final_checkpoint.save_handler.remove(item.filename) + self.logger.info(f"Deleted previous saved final checkpoint: {item.filename}") def completed(self, engine: Engine) -> None: """Callback for train or validation/evaluation completed Event.