From abb3b9c270d231a1c7ca4ec6f303977b7d454662 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Thu, 1 Apr 2021 21:40:34 -0400 Subject: [PATCH 1/8] Implement garbage collector handler Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- monai/handlers/__init__.py | 1 + monai/handlers/garbage_collector.py | 76 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 monai/handlers/garbage_collector.py diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py index 5669e8a9ee..970d9f0350 100644 --- a/monai/handlers/__init__.py +++ b/monai/handlers/__init__.py @@ -13,6 +13,7 @@ from .checkpoint_saver import CheckpointSaver from .classification_saver import ClassificationSaver from .confusion_matrix import ConfusionMatrix +from .garbage_collector import GarbageCollector from .hausdorff_distance import HausdorffDistance from .iteration_metric import IterationMetric from .lr_schedule_handler import LrScheduleHandler diff --git a/monai/handlers/garbage_collector.py b/monai/handlers/garbage_collector.py new file mode 100644 index 0000000000..744de85b5b --- /dev/null +++ b/monai/handlers/garbage_collector.py @@ -0,0 +1,76 @@ +# Copyright 2020 - 2021 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +from typing import TYPE_CHECKING + +from monai.utils import exact_version, optional_import + +if TYPE_CHECKING: + from ignite.engine import Engine, Events +else: + Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine") + Event, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events") + + +class GarbageCollector: + """ + Run garbage collector after each epoch + + Args: + trigger_event: the event that trigger a call to this handler. + - "epoch", after completion of each epoch (equivalent of ignite.engine.Events.EPOCH_COMPLETED) + - "iteration", after completion of each iteration (equivalent of ignite.engine.Events.ITERATION_COMPLETED) + - any ignite built-in event from ignite.engine.Events. + Defaults to "epoch". + log_level: log level (integer) for some garbage collection information as below. Defaults to 10 (DEBUG). + - 50 (CRITICAL) + - 40 (ERROR) + - 30 (WARNING) + - 20 (INFO) + - 10 (DEBUG) + - 0 (NOTSET) + """ + + def __init__(self, trigger_event: str = "epoch", log_level: int = 10): + if trigger_event == "epoch": + self.trigger_event = Events.EPOCH_COMPLETED + elif trigger_event == "iteration": + self.trigger_event = Events.ITERATION_COMPLETED + elif isinstance(trigger_event, Events): + self.trigger_event = trigger_event + else: + raise ValueError( + f"'trigger_event' should be either epoch, iteration, or an ignite built-in event from" + f" ignite.engine.Events, '{self.trigger_event}' was given." + ) + + self.log_level = log_level + + def attach(self, engine: Engine) -> None: + if not engine.has_event_handler(self, self.trigger_event): + engine.add_event_handler(self.trigger_event, self) + + def __call__(self, engine: Engine) -> None: + """ + This method calls python garbage collector. + + Args: + engine: Ignite Engine, it should be either a trainer or validator. + """ + engine.logger.log(self.log_level, "Collecting garbages....") + pre_count = gc.get_count() + unreachable = gc.collect() + after_count = gc.get_count() + engine.logger.log( + self.log_level, + f"Garbage Count: [before: {pre_count}] -> [after: {after_count}] (unreachable : {unreachable})", + ) From e4ea62e0de09c2720e61939c0be88d11a97993d6 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Fri, 2 Apr 2021 10:40:32 -0400 Subject: [PATCH 2/8] Make trigger_event lower case Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- monai/handlers/garbage_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monai/handlers/garbage_collector.py b/monai/handlers/garbage_collector.py index 744de85b5b..2d9cf491ee 100644 --- a/monai/handlers/garbage_collector.py +++ b/monai/handlers/garbage_collector.py @@ -18,7 +18,7 @@ from ignite.engine import Engine, Events else: Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine") - Event, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events") + Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events") class GarbageCollector: @@ -41,9 +41,9 @@ class GarbageCollector: """ def __init__(self, trigger_event: str = "epoch", log_level: int = 10): - if trigger_event == "epoch": + if trigger_event.lower() == "epoch": self.trigger_event = Events.EPOCH_COMPLETED - elif trigger_event == "iteration": + elif trigger_event.lower() == "iteration": self.trigger_event = Events.ITERATION_COMPLETED elif isinstance(trigger_event, Events): self.trigger_event = trigger_event From e8c3ba84500e67232c6f53765bada2ee5db36ff8 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Sun, 4 Apr 2021 16:19:38 -0400 Subject: [PATCH 3/8] Add unittest for garbage collector Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- monai/handlers/garbage_collector.py | 7 +-- tests/test_handler_garbage_collector.py | 77 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 tests/test_handler_garbage_collector.py diff --git a/monai/handlers/garbage_collector.py b/monai/handlers/garbage_collector.py index 2d9cf491ee..92679a0b42 100644 --- a/monai/handlers/garbage_collector.py +++ b/monai/handlers/garbage_collector.py @@ -41,12 +41,12 @@ class GarbageCollector: """ def __init__(self, trigger_event: str = "epoch", log_level: int = 10): - if trigger_event.lower() == "epoch": + if isinstance(trigger_event, Events): + self.trigger_event = trigger_event + elif trigger_event.lower() == "epoch": self.trigger_event = Events.EPOCH_COMPLETED elif trigger_event.lower() == "iteration": self.trigger_event = Events.ITERATION_COMPLETED - elif isinstance(trigger_event, Events): - self.trigger_event = trigger_event else: raise ValueError( f"'trigger_event' should be either epoch, iteration, or an ignite built-in event from" @@ -66,7 +66,6 @@ def __call__(self, engine: Engine) -> None: Args: engine: Ignite Engine, it should be either a trainer or validator. """ - engine.logger.log(self.log_level, "Collecting garbages....") pre_count = gc.get_count() unreachable = gc.collect() after_count = gc.get_count() diff --git a/tests/test_handler_garbage_collector.py b/tests/test_handler_garbage_collector.py new file mode 100644 index 0000000000..5e6bd7275c --- /dev/null +++ b/tests/test_handler_garbage_collector.py @@ -0,0 +1,77 @@ +# Copyright 2020 - 2021 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest +from unittest import skipUnless + +import torch +from ignite.engine import Engine +from parameterized import parameterized + +from monai.data import Dataset +from monai.handlers import GarbageCollector +from monai.utils import exact_version, optional_import + +Events, has_ignite = optional_import("ignite.engine", "0.4.4", exact_version, "Events") + + +TEST_CASE_0 = [[0, 1, 2], "epoch"] + +TEST_CASE_1 = [[0, 1, 2], "iteration"] + +TEST_CASE_2 = [[0, 1, 2], Events.EPOCH_COMPLETED] + + +class TestHandlerGarbageCollector(unittest.TestCase): + @skipUnless(has_ignite, "Requires ignite") + @parameterized.expand( + [ + TEST_CASE_0, + TEST_CASE_1, + TEST_CASE_2, + ] + ) + def test_content(self, data, trigger_event): + # set up engine + gb_count_dict = {} + + def _train_func(engine, batch): + # store garbage collection counts + if trigger_event == Events.EPOCH_COMPLETED or trigger_event.lower() == "epoch": + if engine.state.iteration % engine.state.epoch_length == 1: + gb_count_dict[engine.state.epoch] = gc.get_count() + elif trigger_event.lower() == "iteration": + gb_count_dict[engine.state.iteration] = gc.get_count() + + engine = Engine(_train_func) + + # set up testing handler + dataset = Dataset(data, transform=None) + data_loader = torch.utils.data.DataLoader(dataset, batch_size=1) + GarbageCollector(trigger_event=trigger_event, log_level=30).attach(engine) + + engine.run(data_loader, max_epochs=5) + print(gb_count_dict) + + first_count = 0 + for epoch, gb_count in gb_count_dict.items(): + # At least one zero-generation object + self.assertGreater(gb_count[0], 0) + if epoch == 1: + first_count = gb_count[0] + else: + # The should be less number of collected objects in the next calls. + self.assertLess(gb_count[0], first_count) + + +if __name__ == "__main__": + unittest.main() From d411d53e26976ed70bf8ab9dd007429e7737843f Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Sun, 4 Apr 2021 19:42:42 -0400 Subject: [PATCH 4/8] Update docs Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- docs/source/handlers.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index 080e7e138c..869467c496 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -115,3 +115,8 @@ EarlyStop handler ----------------- .. autoclass:: EarlyStopHandler :members: + +GarbageCollector handler +------------------------ +.. autoclass:: GarbageCollector + :members: From ddd71f4a6469e0d3f514da72c58a5fc8bbf1b696 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Sun, 4 Apr 2021 19:44:16 -0400 Subject: [PATCH 5/8] Exclude from min test Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- tests/min_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/min_tests.py b/tests/min_tests.py index abb5b73764..82eb0e8b83 100644 --- a/tests/min_tests.py +++ b/tests/min_tests.py @@ -42,6 +42,7 @@ def run_testsuit(): "test_handler_confusion_matrix", "test_handler_confusion_matrix_dist", "test_handler_hausdorff_distance", + "test_handler_hausdorff_garbage_collector", "test_handler_mean_dice", "test_handler_prob_map_producer", "test_handler_rocauc", From f4d229515ad14fb25b889fb9b8eae4986f001304 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Sun, 4 Apr 2021 20:04:58 -0400 Subject: [PATCH 6/8] Fix a typo Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- tests/min_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/min_tests.py b/tests/min_tests.py index 82eb0e8b83..91f649cfc5 100644 --- a/tests/min_tests.py +++ b/tests/min_tests.py @@ -42,7 +42,7 @@ def run_testsuit(): "test_handler_confusion_matrix", "test_handler_confusion_matrix_dist", "test_handler_hausdorff_distance", - "test_handler_hausdorff_garbage_collector", + "test_handler_garbage_collector", "test_handler_mean_dice", "test_handler_prob_map_producer", "test_handler_rocauc", From 0cd8506bbd3fa3a7a452d5c332867a6a307d40ff Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Sun, 4 Apr 2021 20:16:35 -0400 Subject: [PATCH 7/8] Fix a bug Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- monai/handlers/garbage_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/garbage_collector.py b/monai/handlers/garbage_collector.py index 92679a0b42..7581551498 100644 --- a/monai/handlers/garbage_collector.py +++ b/monai/handlers/garbage_collector.py @@ -50,7 +50,7 @@ def __init__(self, trigger_event: str = "epoch", log_level: int = 10): else: raise ValueError( f"'trigger_event' should be either epoch, iteration, or an ignite built-in event from" - f" ignite.engine.Events, '{self.trigger_event}' was given." + f" ignite.engine.Events, '{trigger_event}' was given." ) self.log_level = log_level From cf10114894c02ae5c63d5cbc9367d847a73d547c Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 5 Apr 2021 10:14:38 -0400 Subject: [PATCH 8/8] Add another call to gc.collect Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- monai/handlers/garbage_collector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/monai/handlers/garbage_collector.py b/monai/handlers/garbage_collector.py index 7581551498..7bb59c9049 100644 --- a/monai/handlers/garbage_collector.py +++ b/monai/handlers/garbage_collector.py @@ -66,8 +66,13 @@ def __call__(self, engine: Engine) -> None: Args: engine: Ignite Engine, it should be either a trainer or validator. """ + # get count before garbage collection pre_count = gc.get_count() + # fits call to garbage collector + gc.collect() + # second call to garbage collector unreachable = gc.collect() + # get count after garbage collection after_count = gc.get_count() engine.logger.log( self.log_level,