From d4b52d12b646a41b6a1f93d530dcf409cbad40d2 Mon Sep 17 00:00:00 2001 From: skinan Date: Thu, 9 Feb 2023 10:23:41 +0000 Subject: [PATCH 01/32] ClearML Integration (with Tensorboard) --- monai/handlers/__init__.py | 1 + monai/handlers/clearml_handlers.py | 52 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 monai/handlers/clearml_handlers.py diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py index 51aa4f7df9..880483708c 100644 --- a/monai/handlers/__init__.py +++ b/monai/handlers/__init__.py @@ -39,5 +39,6 @@ from .stats_handler import StatsHandler from .surface_distance import SurfaceDistance from .tensorboard_handlers import TensorBoardHandler, TensorBoardImageHandler, TensorBoardStatsHandler +from .clearml_handlers import ClearMLStatsHandler, ClearMLImageHandler from .utils import from_engine, ignore_data, stopping_fn_from_loss, stopping_fn_from_metric, write_metrics_reports from .validation_handler import ValidationHandler diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py new file mode 100644 index 0000000000..d9a7c75993 --- /dev/null +++ b/monai/handlers/clearml_handlers.py @@ -0,0 +1,52 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from .tensorboard_handlers import TensorBoardHandler, TensorBoardImageHandler, TensorBoardStatsHandler + + +class ClearMLStatsHandler(TensorBoardStatsHandler): + """ + Class to write tensorboard stats into ClearML WebUI. + """ + + def __init__(self, project_name ="Muhammad/monai", task_name="unet_training_dict", output_uri=True, *args, **kwargs): + try: + from clearml import Task + except ModuleNotFoundError: + raise "Please install ClearML with 'pip install clearml' before using it." + + if Task.current_task(): + self.clearml_task = Task.current_task() + else: + self.clearml_task = Task.init(project_name=project_name, task_name=task_name, output_uri=output_uri) + + super().__init__(*args, **kwargs) + + +class ClearMLImageHandler(TensorBoardImageHandler): + """ + Class to write tensorboard image stats into ClearML WebUI. + """ + + def __init__(self, project_name ="Muhammad/monai", task_name="unet_training_dict", output_uri=True, *args, **kwargs): + try: + from clearml import Task + except ModuleNotFoundError: + raise "Please install ClearML with 'pip install clearml' before using it." + + if Task.current_task(): + self.clearml_task = Task.current_task() + else: + self.clearml_task = Task.init(project_name=project_name, task_name=task_name, output_uri=output_uri) + + super().__init__(*args, **kwargs) From 2296a34069fa80ef27d642865c2c89963fd24734 Mon Sep 17 00:00:00 2001 From: skinan Date: Sat, 11 Feb 2023 17:49:26 +0000 Subject: [PATCH 02/32] Updates on ClearML Integration --- monai/handlers/__init__.py | 2 +- monai/handlers/clearml_handlers.py | 140 ++++++++++++++++++++++++----- 2 files changed, 121 insertions(+), 21 deletions(-) diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py index 880483708c..42224bfac2 100644 --- a/monai/handlers/__init__.py +++ b/monai/handlers/__init__.py @@ -39,6 +39,6 @@ from .stats_handler import StatsHandler from .surface_distance import SurfaceDistance from .tensorboard_handlers import TensorBoardHandler, TensorBoardImageHandler, TensorBoardStatsHandler -from .clearml_handlers import ClearMLStatsHandler, ClearMLImageHandler +from .clearml_handlers import ClearMLHandler, ClearMLStatsHandler, ClearMLImageHandler from .utils import from_engine, ignore_data, stopping_fn_from_loss, stopping_fn_from_metric, write_metrics_reports from .validation_handler import ValidationHandler diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index d9a7c75993..62841ca10e 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -10,43 +10,143 @@ # limitations under the License. from __future__ import annotations +from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler -from .tensorboard_handlers import TensorBoardHandler, TensorBoardImageHandler, TensorBoardStatsHandler - -class ClearMLStatsHandler(TensorBoardStatsHandler): +class ClearMLHandler: """ - Class to write tensorboard stats into ClearML WebUI. + + Base class for the handlers to log everything to ClearML WebUI. + + Args: + project_name: The name of the project in which the experiment will be created. If the project does not + exist, it is created. If project_name is None, the repository name is used. (Optional) + task_name: ClearML task name. Default set to 'monai clearml example'. + output_uri: The default location for output models and other artifacts. + If True, the default files_server will be used for model storage. In the default location, ClearML creates a subfolder for the output. + The subfolder structure is the following: / / .. Default set to 'True' + tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']. Default set to empty Python List. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, and the same project and Task name. Default set to 'True'. + conitnue_last_task: Continue the execution of a previously executed Task (experiment). Default set to 'False'. + auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to + serialize plots, graphs, and the model location to the ClearML Server (backend), in addition to original output destination. + + For more details of ClearML usage, please refer to: https://clear.ml/docs/latest/docs/references/sdk/task + """ - def __init__(self, project_name ="Muhammad/monai", task_name="unet_training_dict", output_uri=True, *args, **kwargs): + def __init__(self, project_name:str, task_name:str, output_uri:str, tags:List[str], reuse_last_task_id:bool, continue_last_task:bool, auto_connect_frameworks): try: from clearml import Task except ModuleNotFoundError: - raise "Please install ClearML with 'pip install clearml' before using it." + raise ModuleNotFoundError(f"Please install ClearML with 'pip install clearml' before using it.") if Task.current_task(): self.clearml_task = Task.current_task() else: - self.clearml_task = Task.init(project_name=project_name, task_name=task_name, output_uri=output_uri) + self.clearml_task = Task.init( + project_name=project_name, + task_name=task_name, + output_uri=output_uri, + tags=tags, + reuse_last_task_id=reuse_last_task_id, + continue_last_task=continue_last_task, + auto_connect_frameworks=auto_connect_frameworks + ) + + +class ClearMLStatsHandler(ClearMLHandler,TensorBoardStatsHandler): + """ + + Class to write tensorboard stats by inheriting TensorBoardStatsHandler class. + Everything from Tensorboard is logged automatically to ClearML WebUI. + + """ + + def __init__( + self, + project_name:str = None, + task_name:str = None, + output_uri:str = True, + tags:List[str] = [], + reuse_last_task_id:bool = True, + continue_last_task:bool = False, + auto_connect_frameworks = True, + *args, + **kwargs + ): - super().__init__(*args, **kwargs) + """ + Args: + project_name: The name of the project in which the experiment will be created. If the project does not + exist, it is created. If project_name is None, the repository name is used. (Optional) + task_name: ClearML task name. Default set to 'monai clearml example'. + output_uri: The default location for output models and other artifacts. + If True, the default files_server will be used for model storage. In the default location, ClearML creates a subfolder for the output. + The subfolder structure is the following: / / .. Default set to 'True'. + tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']. Default set to empty Python List. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, and the same project and Task name. Default set to 'True'. + conitnue_last_task: Continue the execution of a previously executed Task (experiment). Default set to 'False'. + auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to + serialize plots, graphs, and the model location to the ClearML Server (backend), in addition to original output destination. + """ + + ClearMLHandler.__init__(self, + project_name=project_name, + task_name=task_name, + output_uri=output_uri, + tags=tags, + reuse_last_task_id=reuse_last_task_id, + continue_last_task=continue_last_task, + auto_connect_frameworks=auto_connect_frameworks + ) + TensorBoardStatsHandler.__init__(self, *args, **kwargs) -class ClearMLImageHandler(TensorBoardImageHandler): +class ClearMLImageHandler(ClearMLHandler,TensorBoardImageHandler): """ - Class to write tensorboard image stats into ClearML WebUI. + + This class inherits all functionality from TensorBoardImageHandler class. + Everything from Tensorboard is logged automatically to ClearML WebUI. + """ - def __init__(self, project_name ="Muhammad/monai", task_name="unet_training_dict", output_uri=True, *args, **kwargs): - try: - from clearml import Task - except ModuleNotFoundError: - raise "Please install ClearML with 'pip install clearml' before using it." + def __init__( + self, + project_name:str = None, + task_name:str = None, + output_uri:str = True, + tags:List[str] = [], + reuse_last_task_id:bool = True, + continue_last_task:bool = False, + auto_connect_frameworks = True, + *args, + **kwargs + ): + + """ + Args: + project_name: The name of the project in which the experiment will be created. If the project does not + exist, it is created. If project_name is None, the repository name is used. (Optional) + task_name: ClearML task name. Default set to 'monai clearml example'. + output_uri: The default location for output models and other artifacts. + If True, the default files_server will be used for model storage. In the default location, ClearML creates a subfolder for the output. + The subfolder structure is the following: / / .. Default set to 'True'. + tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']. Default set to empty Python List. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, and the same project and Task name. Default set to 'True'. + conitnue_last_task: Continue the execution of a previously executed Task (experiment). Default set to 'False'. + auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to + serialize plots, graphs, and the model location to the ClearML Server (backend), in addition to original output destination. + """ - if Task.current_task(): - self.clearml_task = Task.current_task() - else: - self.clearml_task = Task.init(project_name=project_name, task_name=task_name, output_uri=output_uri) + ClearMLHandler.__init__(self, + project_name=project_name, + task_name=task_name, + output_uri=output_uri, + tags=tags, + reuse_last_task_id=reuse_last_task_id, + continue_last_task=continue_last_task, + auto_connect_frameworks=auto_connect_frameworks + ) - super().__init__(*args, **kwargs) + TensorBoardImageHandler.__init__(self, *args, **kwargs) From 3dbcd27d381f06558009debfc7f86e277f95710e Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 13 Feb 2023 15:14:15 +0000 Subject: [PATCH 03/32] default_tag --- monai/handlers/clearml_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 62841ca10e..b4c0f1edbd 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -68,7 +68,7 @@ def __init__( project_name:str = None, task_name:str = None, output_uri:str = True, - tags:List[str] = [], + tags:List[str] = ['default_tag_1', 'default_tag_2'], reuse_last_task_id:bool = True, continue_last_task:bool = False, auto_connect_frameworks = True, @@ -116,7 +116,7 @@ def __init__( project_name:str = None, task_name:str = None, output_uri:str = True, - tags:List[str] = [], + tags:List[str] = ['default_tag_1', 'default_tag_2'], reuse_last_task_id:bool = True, continue_last_task:bool = False, auto_connect_frameworks = True, From 6bcd8c6e1a7a59ddef331ee00c56dd60a7755182 Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 13 Feb 2023 16:20:12 +0000 Subject: [PATCH 04/32] Resolve type-formatting issues and others --- monai/handlers/clearml_handlers.py | 226 +++++++++++++++++------------ 1 file changed, 133 insertions(+), 93 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index b4c0f1edbd..aefaa3c955 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -10,6 +10,7 @@ # limitations under the License. from __future__ import annotations + from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler @@ -19,134 +20,173 @@ class ClearMLHandler: Base class for the handlers to log everything to ClearML WebUI. Args: - project_name: The name of the project in which the experiment will be created. If the project does not - exist, it is created. If project_name is None, the repository name is used. (Optional) + project_name: The name of the project in which the experiment will be created. + If the project does not exist, it is created. If project_name is + None, the repository name is used. (Optional) task_name: ClearML task name. Default set to 'monai clearml example'. - output_uri: The default location for output models and other artifacts. - If True, the default files_server will be used for model storage. In the default location, ClearML creates a subfolder for the output. - The subfolder structure is the following: / / .. Default set to 'True' - tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']. Default set to empty Python List. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, and the same project and Task name. Default set to 'True'. - conitnue_last_task: Continue the execution of a previously executed Task (experiment). Default set to 'False'. - auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to - serialize plots, graphs, and the model location to the ClearML Server (backend), in addition to original output destination. + output_uri: The default location for output models and other artifacts. + If True, the default files_server will be used for model storage. + In the default location, ClearML creates a subfolder for the output. + The subfolder structure is the following: + / / .. + Default set to 'True' + tags: Add a list of tags (str) to the created Task. + For example: tags=['512x512', 'yolov3']. + Default set to None. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, + and the same project and Task name. Default set to 'True'. + continue_last_task: Continue the execution of a previously executed Task (experiment). + Default set to 'False'. + auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, + XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to + serialize plots, graphs, and the model location + to the ClearML Server (backend), in addition to original output destination. For more details of ClearML usage, please refer to: https://clear.ml/docs/latest/docs/references/sdk/task - + """ - def __init__(self, project_name:str, task_name:str, output_uri:str, tags:List[str], reuse_last_task_id:bool, continue_last_task:bool, auto_connect_frameworks): + def __init__( + self, + project_name: str, + task_name: str, + output_uri: str, + tags, + reuse_last_task_id: bool, + continue_last_task: bool, + auto_connect_frameworks, + ): try: from clearml import Task - except ModuleNotFoundError: - raise ModuleNotFoundError(f"Please install ClearML with 'pip install clearml' before using it.") + except ModuleNotFoundError as exc: + raise ModuleNotFoundError("Please install ClearML with 'pip install clearml' before using it.") from exc if Task.current_task(): self.clearml_task = Task.current_task() else: self.clearml_task = Task.init( - project_name=project_name, - task_name=task_name, - output_uri=output_uri, - tags=tags, - reuse_last_task_id=reuse_last_task_id, - continue_last_task=continue_last_task, - auto_connect_frameworks=auto_connect_frameworks - ) + project_name=project_name, + task_name=task_name, + output_uri=output_uri, + tags=tags, + reuse_last_task_id=reuse_last_task_id, + continue_last_task=continue_last_task, + auto_connect_frameworks=auto_connect_frameworks, + ) -class ClearMLStatsHandler(ClearMLHandler,TensorBoardStatsHandler): +class ClearMLStatsHandler(ClearMLHandler, TensorBoardStatsHandler): """ - Class to write tensorboard stats by inheriting TensorBoardStatsHandler class. + Class to write tensorboard stats by inheriting TensorBoardStatsHandler class. Everything from Tensorboard is logged automatically to ClearML WebUI. """ def __init__( - self, - project_name:str = None, - task_name:str = None, - output_uri:str = True, - tags:List[str] = ['default_tag_1', 'default_tag_2'], - reuse_last_task_id:bool = True, - continue_last_task:bool = False, - auto_connect_frameworks = True, - *args, - **kwargs - ): - - """ + self, + project_name: str = None, + task_name: str = None, + output_uri: str = True, + tags=None, + reuse_last_task_id: bool = True, + continue_last_task: bool = False, + auto_connect_frameworks=True, + *args, + **kwargs, + ): + """ Args: - project_name: The name of the project in which the experiment will be created. If the project does not - exist, it is created. If project_name is None, the repository name is used. (Optional) + project_name: The name of the project in which the experiment will be created. + If the project does not exist, it is created. If project_name is + None, the repository name is used. (Optional) task_name: ClearML task name. Default set to 'monai clearml example'. - output_uri: The default location for output models and other artifacts. - If True, the default files_server will be used for model storage. In the default location, ClearML creates a subfolder for the output. - The subfolder structure is the following: / / .. Default set to 'True'. - tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']. Default set to empty Python List. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, and the same project and Task name. Default set to 'True'. - conitnue_last_task: Continue the execution of a previously executed Task (experiment). Default set to 'False'. - auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to - serialize plots, graphs, and the model location to the ClearML Server (backend), in addition to original output destination. + output_uri: The default location for output models and other artifacts. + If True, the default files_server will be used for model storage. + In the default location, ClearML creates a subfolder for the output. + The subfolder structure is the following: + / / .. + Default set to 'True' + tags: Add a list of tags (str) to the created Task. + For example: tags=['512x512', 'yolov3']. + Default set to None. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, + and the same project and Task name. Default set to 'True'. + continue_last_task: Continue the execution of a previously executed Task (experiment). + Default set to 'False'. + auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, + XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to + serialize plots, graphs, and the model location + to the ClearML Server (backend), in addition to original output destination. """ - ClearMLHandler.__init__(self, - project_name=project_name, - task_name=task_name, - output_uri=output_uri, - tags=tags, - reuse_last_task_id=reuse_last_task_id, - continue_last_task=continue_last_task, - auto_connect_frameworks=auto_connect_frameworks - ) + ClearMLHandler.__init__( + self, + project_name=project_name, + task_name=task_name, + output_uri=output_uri, + tags=tags, + reuse_last_task_id=reuse_last_task_id, + continue_last_task=continue_last_task, + auto_connect_frameworks=auto_connect_frameworks, + ) TensorBoardStatsHandler.__init__(self, *args, **kwargs) -class ClearMLImageHandler(ClearMLHandler,TensorBoardImageHandler): +class ClearMLImageHandler(ClearMLHandler, TensorBoardImageHandler): """ - - This class inherits all functionality from TensorBoardImageHandler class. + + This class inherits all functionality from TensorBoardImageHandler class. Everything from Tensorboard is logged automatically to ClearML WebUI. - + """ def __init__( - self, - project_name:str = None, - task_name:str = None, - output_uri:str = True, - tags:List[str] = ['default_tag_1', 'default_tag_2'], - reuse_last_task_id:bool = True, - continue_last_task:bool = False, - auto_connect_frameworks = True, - *args, - **kwargs - ): - + self, + project_name: str = None, + task_name: str = None, + output_uri: str = True, + tags=None, + reuse_last_task_id: bool = True, + continue_last_task: bool = False, + auto_connect_frameworks=True, + *args, + **kwargs, + ): """ Args: - project_name: The name of the project in which the experiment will be created. If the project does not - exist, it is created. If project_name is None, the repository name is used. (Optional) - task_name: ClearML task name. Default set to 'monai clearml example'. - output_uri: The default location for output models and other artifacts. - If True, the default files_server will be used for model storage. In the default location, ClearML creates a subfolder for the output. - The subfolder structure is the following: / / .. Default set to 'True'. - tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']. Default set to empty Python List. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, and the same project and Task name. Default set to 'True'. - conitnue_last_task: Continue the execution of a previously executed Task (experiment). Default set to 'False'. - auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to - serialize plots, graphs, and the model location to the ClearML Server (backend), in addition to original output destination. + project_name: The name of the project in which the experiment will be created. + If the project does not exist, it is created. If project_name is + None, the repository name is used. (Optional) + task_name: ClearML task name. Default set to 'monai clearml example'. + output_uri: The default location for output models and other artifacts. + If True, the default files_server will be used for model storage. + In the default location, ClearML creates a subfolder for the output. + The subfolder structure is the following: + / / .. + Default set to 'True' + tags: Add a list of tags (str) to the created Task. + For example: tags=['512x512', 'yolov3']. + Default set to None. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, + and the same project and Task name. Default set to 'True'. + continue_last_task: Continue the execution of a previously executed Task (experiment). + Default set to 'False'. + auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, + XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to + serialize plots, graphs, and the model location + to the ClearML Server (backend), in addition to original output destination. """ - ClearMLHandler.__init__(self, - project_name=project_name, - task_name=task_name, - output_uri=output_uri, - tags=tags, - reuse_last_task_id=reuse_last_task_id, - continue_last_task=continue_last_task, - auto_connect_frameworks=auto_connect_frameworks - ) - + ClearMLHandler.__init__( + self, + project_name=project_name, + task_name=task_name, + output_uri=output_uri, + tags=tags, + reuse_last_task_id=reuse_last_task_id, + continue_last_task=continue_last_task, + auto_connect_frameworks=auto_connect_frameworks, + ) + TensorBoardImageHandler.__init__(self, *args, **kwargs) From acec1ebc5828ec8816b96d4c7ad711d96cc790c5 Mon Sep 17 00:00:00 2001 From: skinan Date: Fri, 17 Feb 2023 06:44:57 +0000 Subject: [PATCH 05/32] Resolved reviews on pull-request --- monai/handlers/clearml_handlers.py | 104 ++++++++++------------------- 1 file changed, 34 insertions(+), 70 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index aefaa3c955..de7d6e3942 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,38 +11,23 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +from monai.utils import optional_import + from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler +if TYPE_CHECKING: + from clearml import Task +else: + Task = optional_import("clearml.Task", name="Task") + class ClearMLHandler: """ - Base class for the handlers to log everything to ClearML WebUI. - - Args: - project_name: The name of the project in which the experiment will be created. - If the project does not exist, it is created. If project_name is - None, the repository name is used. (Optional) - task_name: ClearML task name. Default set to 'monai clearml example'. - output_uri: The default location for output models and other artifacts. - If True, the default files_server will be used for model storage. - In the default location, ClearML creates a subfolder for the output. - The subfolder structure is the following: - / / .. - Default set to 'True' - tags: Add a list of tags (str) to the created Task. - For example: tags=['512x512', 'yolov3']. - Default set to None. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, - and the same project and Task name. Default set to 'True'. - continue_last_task: Continue the execution of a previously executed Task (experiment). - Default set to 'False'. - auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, - XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to - serialize plots, graphs, and the model location - to the ClearML Server (backend), in addition to original output destination. - - For more details of ClearML usage, please refer to: https://clear.ml/docs/latest/docs/references/sdk/task + For more details of ClearML usage, please refer to: + https://clear.ml/docs/latest/docs/references/sdk/task """ @@ -56,10 +41,16 @@ def __init__( continue_last_task: bool, auto_connect_frameworks, ): - try: - from clearml import Task - except ModuleNotFoundError as exc: - raise ModuleNotFoundError("Please install ClearML with 'pip install clearml' before using it.") from exc + """ + Args: + project_name: ClearML project name. + task_name: ClearML task name. + output_uri: The default location for output models and other artifacts. + tags: A list of tags (str) to the created Task. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID. + continue_last_task: Continue the execution of a previously executed Task (experiment). + auto_connect_frameworks: Automatically connect frameworks. + """ if Task.current_task(): self.clearml_task = Task.current_task() @@ -97,27 +88,13 @@ def __init__( ): """ Args: - project_name: The name of the project in which the experiment will be created. - If the project does not exist, it is created. If project_name is - None, the repository name is used. (Optional) - task_name: ClearML task name. Default set to 'monai clearml example'. + project_name: ClearML project name. + task_name: ClearML task name. output_uri: The default location for output models and other artifacts. - If True, the default files_server will be used for model storage. - In the default location, ClearML creates a subfolder for the output. - The subfolder structure is the following: - / / .. - Default set to 'True' - tags: Add a list of tags (str) to the created Task. - For example: tags=['512x512', 'yolov3']. - Default set to None. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, - and the same project and Task name. Default set to 'True'. + tags: A list of tags (str) to the created Task. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID. continue_last_task: Continue the execution of a previously executed Task (experiment). - Default set to 'False'. - auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, - XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to - serialize plots, graphs, and the model location - to the ClearML Server (backend), in addition to original output destination. + auto_connect_frameworks: Automatically connect frameworks. """ ClearMLHandler.__init__( @@ -155,27 +132,14 @@ def __init__( ): """ Args: - project_name: The name of the project in which the experiment will be created. - If the project does not exist, it is created. If project_name is - None, the repository name is used. (Optional) - task_name: ClearML task name. Default set to 'monai clearml example'. - output_uri: The default location for output models and other artifacts. - If True, the default files_server will be used for model storage. - In the default location, ClearML creates a subfolder for the output. - The subfolder structure is the following: - / / .. - Default set to 'True' - tags: Add a list of tags (str) to the created Task. - For example: tags=['512x512', 'yolov3']. - Default set to None. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, - and the same project and Task name. Default set to 'True'. - continue_last_task: Continue the execution of a previously executed Task (experiment). - Default set to 'False'. - auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, - XGBoost, scikit-learn, Keras callbacks, and TensorBoard/X to - serialize plots, graphs, and the model location - to the ClearML Server (backend), in addition to original output destination. + project_name: ClearML project name. + task_name: ClearML task name. + output_uri: The default location for output models and other artifacts. + tags: A list of tags (str) to the created Task. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID. + continue_last_task: Continue the execution of a previously executed Task (experiment). + auto_connect_frameworks: Automatically connect frameworks. + """ ClearMLHandler.__init__( From ef42a1cdef17a7d11623a49cd3e1c0b0751c3820 Mon Sep 17 00:00:00 2001 From: skinan Date: Fri, 24 Feb 2023 18:07:48 +0000 Subject: [PATCH 06/32] Fixed typehints based pull-request review. --- monai/handlers/clearml_handlers.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index de7d6e3942..1d61365b78 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -12,7 +12,7 @@ from __future__ import annotations from typing import TYPE_CHECKING - +from collections.abc import Mapping, Sequence, Optional, Union from monai.utils import optional_import from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler @@ -33,13 +33,13 @@ class ClearMLHandler: def __init__( self, - project_name: str, - task_name: str, - output_uri: str, - tags, + project_name: str | None, + task_name: str | None, + output_uri: str | None, + tags: Optional[Sequence[str]] | None, reuse_last_task_id: bool, continue_last_task: bool, - auto_connect_frameworks, + auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] | bool, ): """ Args: @@ -76,13 +76,13 @@ class ClearMLStatsHandler(ClearMLHandler, TensorBoardStatsHandler): def __init__( self, - project_name: str = None, - task_name: str = None, + project_name: str | None = "MONAI", + task_name: str | None = "Default Task", output_uri: str = True, - tags=None, + tags: Optional[Sequence[str]] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks=True, + auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] | bool = True, *args, **kwargs, ): @@ -120,13 +120,13 @@ class ClearMLImageHandler(ClearMLHandler, TensorBoardImageHandler): def __init__( self, - project_name: str = None, - task_name: str = None, - output_uri: str = True, - tags=None, + project_name: str | None = "MONAI", + task_name: str | None = "Default Task", + output_uri: str | bool = True, + tags: Optional[Sequence[str]] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks=True, + auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] | bool = True, *args, **kwargs, ): From 302d81e9ad4c1fce83d41cef788b10bc4fa1663e Mon Sep 17 00:00:00 2001 From: skinan Date: Tue, 28 Feb 2023 16:34:54 +0000 Subject: [PATCH 07/32] Formatting, Unit Test added, type hinting fixed --- monai/handlers/__init__.py | 2 +- monai/handlers/clearml_handlers.py | 6 ++-- tests/test_handler_clearml_image.py | 43 +++++++++++++++++++++++++++++ tests/test_handler_clearml_stats.py | 43 +++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 tests/test_handler_clearml_image.py create mode 100644 tests/test_handler_clearml_stats.py diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py index 42224bfac2..f032191043 100644 --- a/monai/handlers/__init__.py +++ b/monai/handlers/__init__.py @@ -14,6 +14,7 @@ from .checkpoint_loader import CheckpointLoader from .checkpoint_saver import CheckpointSaver from .classification_saver import ClassificationSaver +from .clearml_handlers import ClearMLHandler, ClearMLImageHandler, ClearMLStatsHandler from .confusion_matrix import ConfusionMatrix from .decollate_batch import DecollateBatch from .earlystop_handler import EarlyStopHandler @@ -39,6 +40,5 @@ from .stats_handler import StatsHandler from .surface_distance import SurfaceDistance from .tensorboard_handlers import TensorBoardHandler, TensorBoardImageHandler, TensorBoardStatsHandler -from .clearml_handlers import ClearMLHandler, ClearMLStatsHandler, ClearMLImageHandler from .utils import from_engine, ignore_data, stopping_fn_from_loss, stopping_fn_from_metric, write_metrics_reports from .validation_handler import ValidationHandler diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 1d61365b78..7740f20c7f 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,8 +11,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING -from collections.abc import Mapping, Sequence, Optional, Union +from typing import TYPE_CHECKING, Mapping, Optional, Sequence, Union + from monai.utils import optional_import from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler @@ -20,7 +20,7 @@ if TYPE_CHECKING: from clearml import Task else: - Task = optional_import("clearml.Task", name="Task") + Task, _ = optional_import("clearml", name="Task") class ClearMLHandler: diff --git a/tests/test_handler_clearml_image.py b/tests/test_handler_clearml_image.py new file mode 100644 index 0000000000..2d5ce0a35c --- /dev/null +++ b/tests/test_handler_clearml_image.py @@ -0,0 +1,43 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import unittest + +from monai.handlers import ClearMLImageHandler +from monai.utils import optional_import + +Task, has_clearml = optional_import("clearml", name="Task") + + +@unittest.skipUnless(has_clearml, "Requires 'clearml' installation") +class TestHandlerClearMLImageHandler(unittest.TestCase): + def test_task_init(self): + Task.set_offline(offline_mode=True) + try: + ClearMLImageHandler( + project_name="MONAI", + task_name="Default Task", + output_uri=True, + tags=None, + reuse_last_task_id=True, + continue_last_task=False, + auto_connect_frameworks=True, + ) + except Exception as exc: + self.fail(exc) + self.assertEqual(Task.current_task().name, "Default Task") + self.assertEqual(Task.current_task()._project_name[1], "MONAI") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_handler_clearml_stats.py b/tests/test_handler_clearml_stats.py new file mode 100644 index 0000000000..0df2728d8c --- /dev/null +++ b/tests/test_handler_clearml_stats.py @@ -0,0 +1,43 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import unittest + +from monai.handlers import ClearMLStatsHandler +from monai.utils import optional_import + +Task, has_clearml = optional_import("clearml", name="Task") + + +@unittest.skipUnless(has_clearml, "Requires 'clearml' installation") +class TestHandlerClearMLStatsHandler(unittest.TestCase): + def test_task_init(self): + Task.set_offline(offline_mode=True) + try: + ClearMLStatsHandler( + project_name="MONAI", + task_name="Default Task", + output_uri=True, + tags=None, + reuse_last_task_id=True, + continue_last_task=False, + auto_connect_frameworks=True, + ) + except Exception as exc: + self.fail(exc) + self.assertEqual(Task.current_task().name, "Default Task") + self.assertEqual(Task.current_task()._project_name[1], "MONAI") + + +if __name__ == "__main__": + unittest.main() From 1971492dac90362890e49fbb9960a1f19a3fc41a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:36:11 +0000 Subject: [PATCH 08/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/handlers/clearml_handlers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 7740f20c7f..eaeb06e73f 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Mapping, Optional, Sequence, Union +from typing import TYPE_CHECKING, Mapping, Sequence from monai.utils import optional_import @@ -36,10 +36,10 @@ def __init__( project_name: str | None, task_name: str | None, output_uri: str | None, - tags: Optional[Sequence[str]] | None, + tags: Sequence[str] | None | None, reuse_last_task_id: bool, continue_last_task: bool, - auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] | bool, + auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool, ): """ Args: @@ -79,10 +79,10 @@ def __init__( project_name: str | None = "MONAI", task_name: str | None = "Default Task", output_uri: str = True, - tags: Optional[Sequence[str]] | None = None, + tags: Sequence[str] | None | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] | bool = True, + auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, *args, **kwargs, ): @@ -123,10 +123,10 @@ def __init__( project_name: str | None = "MONAI", task_name: str | None = "Default Task", output_uri: str | bool = True, - tags: Optional[Sequence[str]] | None = None, + tags: Sequence[str] | None | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] | bool = True, + auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, *args, **kwargs, ): From 331953077f5e50c9d6bdc31c30d0a304b18dbef8 Mon Sep 17 00:00:00 2001 From: skinan Date: Tue, 28 Feb 2023 16:38:54 +0000 Subject: [PATCH 09/32] "output_uri" type hinting fixed --- monai/handlers/clearml_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 7740f20c7f..102b6700cf 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -35,7 +35,7 @@ def __init__( self, project_name: str | None, task_name: str | None, - output_uri: str | None, + output_uri: str | bool, tags: Optional[Sequence[str]] | None, reuse_last_task_id: bool, continue_last_task: bool, @@ -78,7 +78,7 @@ def __init__( self, project_name: str | None = "MONAI", task_name: str | None = "Default Task", - output_uri: str = True, + output_uri: str | bool = True, tags: Optional[Sequence[str]] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, From 89fb3c04ea26f0976c2db091b9bc43f23b61df9c Mon Sep 17 00:00:00 2001 From: skinan Date: Tue, 28 Feb 2023 16:43:40 +0000 Subject: [PATCH 10/32] type hinting fixed --- monai/handlers/clearml_handlers.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 3cfd0b445f..ad22292ccd 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Mapping, Sequence +from typing import TYPE_CHECKING, Mapping, Sequence, Optional from monai.utils import optional_import @@ -37,8 +37,6 @@ def __init__( task_name: str | None, output_uri: str | bool, tags: Optional[Sequence[str]] | None, - output_uri: str | None, - tags: Sequence[str] | None | None, reuse_last_task_id: bool, continue_last_task: bool, auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool, @@ -82,8 +80,6 @@ def __init__( task_name: str | None = "Default Task", output_uri: str | bool = True, tags: Optional[Sequence[str]] | None = None, - output_uri: str = True, - tags: Sequence[str] | None | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, @@ -127,7 +123,7 @@ def __init__( project_name: str | None = "MONAI", task_name: str | None = "Default Task", output_uri: str | bool = True, - tags: Sequence[str] | None | None = None, + tags: Optional[Sequence[str]] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, From 96f38fcfa77bccfb9fcc1138ba6cd2be47ff56bf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:47:52 +0000 Subject: [PATCH 11/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/handlers/clearml_handlers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index ad22292ccd..f5af8575f8 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Mapping, Sequence, Optional +from typing import TYPE_CHECKING, Mapping, Sequence from monai.utils import optional_import @@ -36,7 +36,7 @@ def __init__( project_name: str | None, task_name: str | None, output_uri: str | bool, - tags: Optional[Sequence[str]] | None, + tags: Sequence[str] | None | None, reuse_last_task_id: bool, continue_last_task: bool, auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool, @@ -79,7 +79,7 @@ def __init__( project_name: str | None = "MONAI", task_name: str | None = "Default Task", output_uri: str | bool = True, - tags: Optional[Sequence[str]] | None = None, + tags: Sequence[str] | None | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, @@ -123,7 +123,7 @@ def __init__( project_name: str | None = "MONAI", task_name: str | None = "Default Task", output_uri: str | bool = True, - tags: Optional[Sequence[str]] | None = None, + tags: Sequence[str] | None | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, From 5dd1c095b3b98d7effeac3942a4c81cd18ef38c8 Mon Sep 17 00:00:00 2001 From: skinan Date: Tue, 28 Feb 2023 16:55:26 +0000 Subject: [PATCH 12/32] Type hinting --- monai/handlers/clearml_handlers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index ad22292ccd..63ab08e67c 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Mapping, Sequence, Optional +from typing import TYPE_CHECKING, Mapping, Sequence, Union from monai.utils import optional_import @@ -36,10 +36,10 @@ def __init__( project_name: str | None, task_name: str | None, output_uri: str | bool, - tags: Optional[Sequence[str]] | None, + tags: Sequence[str] | None, reuse_last_task_id: bool, continue_last_task: bool, - auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool, + auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]], ): """ Args: @@ -79,10 +79,10 @@ def __init__( project_name: str | None = "MONAI", task_name: str | None = "Default Task", output_uri: str | bool = True, - tags: Optional[Sequence[str]] | None = None, + tags: Sequence[str] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, + auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] = True, *args, **kwargs, ): @@ -123,10 +123,10 @@ def __init__( project_name: str | None = "MONAI", task_name: str | None = "Default Task", output_uri: str | bool = True, - tags: Optional[Sequence[str]] | None = None, + tags: Sequence[str] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks: bool | Mapping[str, bool | str | list] | bool = True, + auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] = True, *args, **kwargs, ): From 87ff034a1461945523143f0d374e9bd45a61ed30 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:57:20 +0000 Subject: [PATCH 13/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/handlers/clearml_handlers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 63ab08e67c..604f6a7154 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Mapping, Sequence, Union +from typing import TYPE_CHECKING, Mapping, Sequence from monai.utils import optional_import @@ -39,7 +39,7 @@ def __init__( tags: Sequence[str] | None, reuse_last_task_id: bool, continue_last_task: bool, - auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]], + auto_connect_frameworks: bool | Mapping[str, bool | str | list], ): """ Args: @@ -82,7 +82,7 @@ def __init__( tags: Sequence[str] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] = True, + auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, *args, **kwargs, ): @@ -126,7 +126,7 @@ def __init__( tags: Sequence[str] | None = None, reuse_last_task_id: bool = True, continue_last_task: bool = False, - auto_connect_frameworks: Union[bool, Mapping[str, Union[bool, str, list]]] = True, + auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, *args, **kwargs, ): From 77146d6db397935e5533c63519ea805a634bf49d Mon Sep 17 00:00:00 2001 From: skinan Date: Wed, 1 Mar 2023 07:15:26 +0000 Subject: [PATCH 14/32] review changes --- docs/requirements.txt | 1 + docs/source/handlers.rst | 11 +++ docs/source/installation.md | 93 ++++++++++++++++------- monai/bundle/scripts.py | 2 + monai/bundle/utils.py | 111 +++++++++++++++++++++++++++- monai/handlers/clearml_handlers.py | 52 ++++++------- requirements-dev.txt | 1 + tests/test_handler_clearml_image.py | 4 +- tests/test_handler_clearml_stats.py | 4 +- 9 files changed, 219 insertions(+), 60 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index e5888b85db..40d476aa5c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -22,6 +22,7 @@ pandas einops transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 mlflow>=1.28.0 +clearml >=1.9.1 tensorboardX imagecodecs; platform_system == "Linux" or platform_system == "Darwin" tifffile; platform_system == "Linux" or platform_system == "Darwin" diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index 3e635319d1..f70f28b45e 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -189,6 +189,17 @@ MLFlow handler .. autoclass:: MLFlowHandler :members: +ClearML handlers +-------------------- +.. autoclass:: ClearMLHandler + :members: + +.. autoclass:: ClearMLStatsHandler + :members: + +.. autoclass:: ClearMLImageHandler + :members: + NVTX Handlers ------------- .. automodule:: monai.handlers.nvtx_handlers diff --git a/docs/source/installation.md b/docs/source/installation.md index 5d071d0b8b..939eaefdca 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -1,18 +1,21 @@ # Installation Guide ## Table of Contents -1. [From PyPI](#from-pypi) - 1. [Milestone release](#milestone-release) - 2. [Weekly preview release](#weekly-preview-release) - 3. [Uninstall the packages](#uninstall-the-packages) -1. [From conda-forge](#from-conda-forge) -2. [From GitHub](#from-github) - 1. [System-wide](#milestone-release) - 2. [Editable](#weekly-preview-release) -3. [Validating the install](#validating-the-install) -4. [MONAI version string](#monai-version-string) -5. [From DockerHub](#from-dockerhub) -6. [Installing the recommended dependencies](#installing-the-recommended-dependencies) + +- [Installation Guide](#installation-guide) + - [Table of Contents](#table-of-contents) + - [From PyPI](#from-pypi) + - [Milestone release](#milestone-release) + - [Weekly preview release](#weekly-preview-release) + - [Uninstall the packages](#uninstall-the-packages) + - [From conda-forge](#from-conda-forge) + - [From GitHub](#from-github) + - [Option 1 (as a part of your system-wide module):](#option-1-as-a-part-of-your-system-wide-module) + - [Option 2 (editable installation):](#option-2-editable-installation) + - [Validating the install](#validating-the-install) + - [MONAI version string](#monai-version-string) + - [From DockerHub](#from-dockerhub) + - [Installing the recommended dependencies](#installing-the-recommended-dependencies) --- @@ -24,32 +27,39 @@ and the Python package index (PyPI). The pre-built Docker images are made availa To install optional features such as handling the NIfTI files using [Nibabel](https://nipy.org/nibabel/), or building workflows using [Pytorch Ignite](https://pytorch.org/ignite/), please follow the instructions: + - [Installing the recommended dependencies](#installing-the-recommended-dependencies) The installation commands bellow usually end up installing CPU variant of PyTorch. To install GPU-enabled PyTorch: + 1. Install the latest NVIDIA driver. 1. Check [PyTorch Official Guide](https://pytorch.org/get-started/locally/) for the recommended CUDA versions. For Pip package, the user needs to download the CUDA manually, install it on the system, and ensure CUDA_PATH is set properly. 1. Continue to follow the guide and install PyTorch. 1. Install MONAI using one the ways described below. ---- +--- ## From PyPI ### Milestone release + To install the [current milestone release](https://pypi.org/project/monai/): + ```bash pip install monai ``` ### Weekly preview release + To install the [weekly preview release](https://pypi.org/project/monai-weekly/): + ```bash pip install monai-weekly ``` The weekly build is released to PyPI every Sunday with a pre-release build number `dev[%y%U]`. To report any issues on the weekly preview, please include the version and commit information: + ```bash python -c "import monai; print(monai.__version__); print(monai.__commit_id__)" ``` @@ -61,7 +71,9 @@ without uninstalling the existing one first. To address this issue, please uninstall both packages, and retry the installation. ### Uninstall the packages + The packages installed using `pip install` could be removed by: + ```bash pip uninstall -y monai pip uninstall -y monai-weekly @@ -70,51 +82,64 @@ pip uninstall -y monai-weekly ## From conda-forge To install the [current milestone release](https://anaconda.org/conda-forge/monai): + ```bash conda install -c conda-forge monai ``` ## From GitHub + (_If you have installed the -PyPI release version using ``pip install monai``, please run ``pip uninstall -monai`` before using the commands from this section. Because ``pip`` by +PyPI release version using `pip install monai`, please run `pip uninstall +monai` before using the commands from this section. Because `pip` by default prefers the milestone release_.) -The milestone versions are currently planned and released every few months. As the +The milestone versions are currently planned and released every few months. As the codebase is under active development, you may want to install MONAI from GitHub for the latest features: ### Option 1 (as a part of your system-wide module): + ```bash pip install git+https://github.com/Project-MONAI/MONAI#egg=monai ``` + or, to build with MONAI C++/CUDA extensions: + ```bash BUILD_MONAI=1 pip install git+https://github.com/Project-MONAI/MONAI#egg=monai ``` To build the extensions, if the system environment already has a version of Pytorch installed, `--no-build-isolation` might be preferred: + ```bash BUILD_MONAI=1 pip install --no-build-isolation git+https://github.com/Project-MONAI/MONAI#egg=monai ``` + this command will download and install the current `dev` branch of [MONAI from GitHub](https://github.com/Project-MONAI/MONAI). This documentation website by default shows the information for the latest version. ### Option 2 (editable installation): + To install an editable version of MONAI, it is recommended to clone the codebase directly: + ```bash git clone https://github.com/Project-MONAI/MONAI.git ``` -This command will create a ``MONAI/`` folder in your current directory. + +This command will create a `MONAI/` folder in your current directory. You can install it by running: + ```bash cd MONAI/ python setup.py develop ``` + or, to build with MONAI C++/CUDA extensions and install: + ```bash cd MONAI/ BUILD_MONAI=1 python setup.py develop @@ -123,6 +148,7 @@ BUILD_MONAI=1 CC=clang CXX=clang++ python setup.py develop ``` To uninstall the package please run: + ```bash cd MONAI/ python setup.py develop --uninstall @@ -131,37 +157,41 @@ python setup.py develop --uninstall ./runtests.sh --clean ``` -Alternatively, simply adding the root directory of the cloned source code (e.g., ``/workspace/Documents/MONAI``) to your ``$PYTHONPATH`` +Alternatively, simply adding the root directory of the cloned source code (e.g., `/workspace/Documents/MONAI`) to your `$PYTHONPATH` and the codebase is ready to use (without the additional features of MONAI C++/CUDA extensions). > The C++/CUDA extension features are currently experimental, a pre-compiled version is made available via > [the recent docker image releases](https://hub.docker.com/r/projectmonai/monai). > Building the extensions from source may require [Ninja](https://ninja-build.org/) and [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit). > By default, CUDA extension is built if `torch.cuda.is_available()`. It's possible to force building by -> setting ``FORCE_CUDA=1`` environment variable. - +> setting `FORCE_CUDA=1` environment variable. ## Validating the install + You can verify the installation by: + ```bash python -c "import monai; monai.config.print_config()" ``` + If the installation is successful, this command will print out the MONAI version information, and this confirms the core modules of MONAI are ready-to-use. - ## MONAI version string + The MONAI version string shows the current status of your local installation. For example: + ``` MONAI version: 0.1.0+144.g52c763d.dirty ``` -- ``0.1.0`` indicates that your installation is based on the ``0.1.0`` milestone release. -- ``+144`` indicates that your installation is 144 git commits ahead of the milestone release. -- ``g52c763d`` indicates that your installation corresponds to the git commit hash ``52c763d``. -- ``dirty`` indicates that you have modified the codebase locally, and the codebase is inconsistent with ``52c763d``. +- `0.1.0` indicates that your installation is based on the `0.1.0` milestone release. +- `+144` indicates that your installation is 144 git commits ahead of the milestone release. +- `g52c763d` indicates that your installation corresponds to the git commit hash `52c763d`. +- `dirty` indicates that you have modified the codebase locally, and the codebase is inconsistent with `52c763d`. ## From DockerHub + Make sure you have installed the NVIDIA driver and Docker 19.03+ for your Linux distribution. Note that you do not need to install the CUDA toolkit on the host, but the driver needs to be installed. Please find out more information on [nvidia-docker](https://github.com/NVIDIA/nvidia-docker). @@ -169,20 +199,24 @@ Please find out more information on [nvidia-docker](https://github.com/NVIDIA/nv Assuming that you have the Nvidia driver and Docker 19.03+ installed, running the following command will download and start a container with the latest version of MONAI. The latest `dev` branch of MONAI from GitHub is included in the image. + ```bash docker run --gpus all --rm -ti --ipc=host projectmonai/monai:latest ``` You can also run a milestone release docker image by specifying the image tag, for example: + ``` docker run --gpus all --rm -ti --ipc=host projectmonai/monai:0.1.0 ``` ## Installing the recommended dependencies + By default, the installation steps will only download and install the minimal requirements of MONAI. Optional dependencies can be installed using [the extras syntax](https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras) to support additional features. For example, to install MONAI with Nibabel and Scikit-image support: + ```bash git clone https://github.com/Project-MONAI/MONAI.git cd MONAI/ @@ -190,6 +224,7 @@ pip install -e '.[nibabel,skimage]' ``` Alternatively, to install all optional dependencies: + ```bash git clone https://github.com/Project-MONAI/MONAI.git cd MONAI/ @@ -197,6 +232,7 @@ pip install -e '.[all]' ``` To install all optional dependencies with `pip` based on MONAI development environment settings: + ```bash git clone https://github.com/Project-MONAI/MONAI.git cd MONAI/ @@ -205,6 +241,7 @@ pip install -r requirements-dev.txt To install all optional dependencies with `conda` based on MONAI development environment settings (`environment-dev.yml`; this will install PyTorch as well as `pytorch-cuda`, please follow https://pytorch.org/get-started/locally/#start-locally for more details about installing PyTorch): + ```bash git clone https://github.com/Project-MONAI/MONAI.git cd MONAI/ @@ -215,10 +252,12 @@ conda env update -n -f environment-dev.yml Since MONAI v0.2.0, the extras syntax such as `pip install 'monai[nibabel]'` is available via PyPI. - The options are + ``` -[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml, fire, jsonschema, ninja, pynrrd, pydicom, h5py, nni, optuna] +[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, clearml, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml, fire, jsonschema, ninja, pynrrd, pydicom, h5py, nni, optuna] ``` + which correspond to `nibabel`, `scikit-image`, `pillow`, `tensorboard`, -`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, `fire`, `jsonschema`, `ninja`, `pynrrd`, `pydicom`, `h5py`, `nni`, `optuna`, respectively. +`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `clearml`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, `fire`, `jsonschema`, `ninja`, `pynrrd`, `pydicom`, `h5py`, `nni`, `optuna`, respectively. - `pip install 'monai[all]'` installs all the optional dependencies. diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 3d4b7288d4..c323164cd5 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -650,6 +650,8 @@ def run( Default to "configs/logging.conf", which is commonly used for bundles in MONAI model zoo. tracking: enable the experiment tracking feature at runtime with optionally configurable and extensible. if "mlflow", will add `MLFlowHandler` to the parsed bundle with default logging settings, + if "clearml_stats", will add `ClearMLStatsHandler` to the parsed bundle with default logging settings, + if "clearml_image", will add `ClearMLImageHandler` to the parsed bundle with default logging settings, if other string, treat it as file path to load the logging settings, if `dict`, treat it as logging settings, otherwise, use all the default settings. will patch the target config content with `tracking handlers` and the top-level items of `configs`. diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 0961493b20..abea51c2ad 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -21,7 +21,16 @@ yaml, _ = optional_import("yaml") -__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY", "DEFAULT_MLFLOW_SETTINGS", "DEFAULT_EXP_MGMT_SETTINGS"] +__all__ = [ + "ID_REF_KEY", + "ID_SEP_KEY", + "EXPR_KEY", + "MACRO_KEY", + "DEFAULT_MLFLOW_SETTINGS", + "DEFAULT_CLEARML_STATS_SETTINGS", + "DEFAULT_CLEARML_IMAGE_SETTINGS", + "DEFAULT_EXP_MGMT_SETTINGS", +] ID_REF_KEY = "@" # start of a reference to a ConfigItem ID_SEP_KEY = "#" # separator for the ID of a ConfigItem @@ -156,15 +165,109 @@ } DEFAULT_CLEARML_STATS_SETTINGS = { - + "handlers_id": DEFAULT_HANDLERS_ID, + "configs": { + "experiment_name": "monai_experiment", + # may fill it at runtime + "execute_config": None, + "is_not_rank0": ( + "$torch.distributed.is_available() \ + and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0" + ), + # ClearMLStatsHandler config for the trainer + "trainer": { + "_target_": "ClearMLStatsHandler", + "_disabled_": "@is_not_rank0", + "project_name": "MONAI", + "task_name": "@experiment_name", + "output_uri:": "True", + "tags": "None", + "reuse_last_task_id": "True", + "continue_last_task": "False", + "auto_connect_frameworks": "True", + }, + # ClearMLStatsHandler config for the validator + "validator": { + "_target_": "ClearMLStatsHandler", + "_disabled_": "@is_not_rank0", + "project_name": "MONAI", + "task_name": "@experiment_name", + "output_uri:": "True", + "tags": "None", + "reuse_last_task_id": "True", + "continue_last_task": "False", + "auto_connect_frameworks": "True", + }, + # ClearMLStatsHandler config for the evaluator + "evaluator": { + "_target_": "ClearMLStatsHandler", + "_disabled_": "@is_not_rank0", + "project_name": "MONAI", + "task_name": "@experiment_name", + "output_uri:": "True", + "tags": "None", + "reuse_last_task_id": "True", + "continue_last_task": "False", + "auto_connect_frameworks": "True", + }, + }, } DEFAULT_CLEARML_IMAGE_SETTINGS = { - + "handlers_id": DEFAULT_HANDLERS_ID, + "configs": { + "experiment_name": "monai_experiment", + # may fill it at runtime + "execute_config": None, + "is_not_rank0": ( + "$torch.distributed.is_available() \ + and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0" + ), + # ClearMLImageHandler config for the trainer + "trainer": { + "_target_": "ClearMLImageHandler", + "_disabled_": "@is_not_rank0", + "project_name": "MONAI", + "task_name": "@experiment_name", + "output_uri:": "True", + "tags": "None", + "reuse_last_task_id": "True", + "continue_last_task": "False", + "auto_connect_frameworks": "True", + }, + # ClearMLImageHandler config for the validator + "validator": { + "_target_": "ClearMLImageHandler", + "_disabled_": "@is_not_rank0", + "project_name": "MONAI", + "task_name": "@experiment_name", + "output_uri:": "True", + "tags": "None", + "reuse_last_task_id": "True", + "continue_last_task": "False", + "auto_connect_frameworks": "True", + }, + # ClearMLImageHandler config for the evaluator + "evaluator": { + "_target_": "ClearMLImageHandler", + "_disabled_": "@is_not_rank0", + "project_name": "MONAI", + "task_name": "@experiment_name", + "output_uri:": "True", + "tags": "None", + "reuse_last_task_id": "True", + "continue_last_task": "False", + "auto_connect_frameworks": "True", + }, + }, } -DEFAULT_EXP_MGMT_SETTINGS = {"mlflow": DEFAULT_MLFLOW_SETTINGS} # default experiment management settings +DEFAULT_EXP_MGMT_SETTINGS = { + "mlflow": DEFAULT_MLFLOW_SETTINGS, + "clearml_stats": DEFAULT_CLEARML_STATS_SETTINGS, + "clearml_image": DEFAULT_CLEARML_IMAGE_SETTINGS, +} # default experiment management settings def load_bundle_config(bundle_path: str, *config_names: str, **load_kw_args: Any) -> Any: diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 604f6a7154..93fb9a9b32 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -40,16 +40,17 @@ def __init__( reuse_last_task_id: bool, continue_last_task: bool, auto_connect_frameworks: bool | Mapping[str, bool | str | list], - ): + ) -> None: """ Args: - project_name: ClearML project name. - task_name: ClearML task name. - output_uri: The default location for output models and other artifacts. - tags: A list of tags (str) to the created Task. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID. - continue_last_task: Continue the execution of a previously executed Task (experiment). - auto_connect_frameworks: Automatically connect frameworks. + project_name: ClearML project name, default to 'MONAI'. + task_name: ClearML task name, default to 'monai_experiment'. + output_uri: The default location for output models and other artifacts, default to 'True'. + tags: A list of tags (str) to the created Task, default to 'None'. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, default to 'True'. + continue_last_task: Continue the execution of a previously executed Task (experiment), default to 'False'. + auto_connect_frameworks: Automatically connect frameworks, default to 'True'. + """ if Task.current_task(): @@ -77,7 +78,7 @@ class ClearMLStatsHandler(ClearMLHandler, TensorBoardStatsHandler): def __init__( self, project_name: str | None = "MONAI", - task_name: str | None = "Default Task", + task_name: str | None = "monai_experiment", output_uri: str | bool = True, tags: Sequence[str] | None = None, reuse_last_task_id: bool = True, @@ -85,16 +86,17 @@ def __init__( auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, *args, **kwargs, - ): + ) -> None: """ Args: - project_name: ClearML project name. - task_name: ClearML task name. - output_uri: The default location for output models and other artifacts. - tags: A list of tags (str) to the created Task. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID. - continue_last_task: Continue the execution of a previously executed Task (experiment). - auto_connect_frameworks: Automatically connect frameworks. + project_name: ClearML project name, default to 'MONAI'. + task_name: ClearML task name, default to 'monai_experiment'. + output_uri: The default location for output models and other artifacts, default to 'True'. + tags: A list of tags (str) to the created Task, default to 'None'. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, default to 'True'. + continue_last_task: Continue the execution of a previously executed Task (experiment), default to 'False'. + auto_connect_frameworks: Automatically connect frameworks, default to 'True'. + """ ClearMLHandler.__init__( @@ -121,7 +123,7 @@ class ClearMLImageHandler(ClearMLHandler, TensorBoardImageHandler): def __init__( self, project_name: str | None = "MONAI", - task_name: str | None = "Default Task", + task_name: str | None = "monai_experiment", output_uri: str | bool = True, tags: Sequence[str] | None = None, reuse_last_task_id: bool = True, @@ -132,13 +134,13 @@ def __init__( ): """ Args: - project_name: ClearML project name. - task_name: ClearML task name. - output_uri: The default location for output models and other artifacts. - tags: A list of tags (str) to the created Task. - reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID. - continue_last_task: Continue the execution of a previously executed Task (experiment). - auto_connect_frameworks: Automatically connect frameworks. + project_name: ClearML project name, default to 'MONAI'. + task_name: ClearML task name, default to 'monai_experiment'. + output_uri: The default location for output models and other artifacts, default to 'True'. + tags: A list of tags (str) to the created Task, default to 'None'. + reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, default to 'True'. + continue_last_task: Continue the execution of a previously executed Task (experiment), default to 'False'. + auto_connect_frameworks: Automatically connect frameworks, default to 'True'. """ diff --git a/requirements-dev.txt b/requirements-dev.txt index f8de589eef..59f704b19c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -36,6 +36,7 @@ requests einops transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 mlflow>=1.28.0 +clearml >=1.9.1 matplotlib!=3.5.0 tensorboardX types-PyYAML diff --git a/tests/test_handler_clearml_image.py b/tests/test_handler_clearml_image.py index 2d5ce0a35c..2ba736e878 100644 --- a/tests/test_handler_clearml_image.py +++ b/tests/test_handler_clearml_image.py @@ -26,7 +26,7 @@ def test_task_init(self): try: ClearMLImageHandler( project_name="MONAI", - task_name="Default Task", + task_name="monai_experiment", output_uri=True, tags=None, reuse_last_task_id=True, @@ -35,7 +35,7 @@ def test_task_init(self): ) except Exception as exc: self.fail(exc) - self.assertEqual(Task.current_task().name, "Default Task") + self.assertEqual(Task.current_task().name, "monai_experiment") self.assertEqual(Task.current_task()._project_name[1], "MONAI") diff --git a/tests/test_handler_clearml_stats.py b/tests/test_handler_clearml_stats.py index 0df2728d8c..947361e17e 100644 --- a/tests/test_handler_clearml_stats.py +++ b/tests/test_handler_clearml_stats.py @@ -26,7 +26,7 @@ def test_task_init(self): try: ClearMLStatsHandler( project_name="MONAI", - task_name="Default Task", + task_name="monai_experiment", output_uri=True, tags=None, reuse_last_task_id=True, @@ -35,7 +35,7 @@ def test_task_init(self): ) except Exception as exc: self.fail(exc) - self.assertEqual(Task.current_task().name, "Default Task") + self.assertEqual(Task.current_task().name, "monai_experiment") self.assertEqual(Task.current_task()._project_name[1], "MONAI") From 956a195c37b93ce7f0cdf0fe45f4bba28ee5db71 Mon Sep 17 00:00:00 2001 From: Muhammad Sakib Khan Inan Date: Fri, 3 Mar 2023 19:40:15 +0600 Subject: [PATCH 15/32] Update docs/source/installation.md Co-authored-by: Wenqi Li <831580+wyli@users.noreply.github.com> Signed-off-by: Muhammad Sakib Khan Inan --- docs/source/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/installation.md b/docs/source/installation.md index 939eaefdca..ffa9cdf091 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -10,8 +10,8 @@ - [Uninstall the packages](#uninstall-the-packages) - [From conda-forge](#from-conda-forge) - [From GitHub](#from-github) - - [Option 1 (as a part of your system-wide module):](#option-1-as-a-part-of-your-system-wide-module) - - [Option 2 (editable installation):](#option-2-editable-installation) + - [Option 1 (as a part of your system-wide module)](#option-1-as-a-part-of-your-system-wide-module) + - [Option 2 (editable installation)](#option-2-editable-installation) - [Validating the install](#validating-the-install) - [MONAI version string](#monai-version-string) - [From DockerHub](#from-dockerhub) From 5dff6e416083abf9507a7c2f885d1a0d11640662 Mon Sep 17 00:00:00 2001 From: skinan Date: Fri, 3 Mar 2023 13:59:49 +0000 Subject: [PATCH 16/32] updated optional dependency configs --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 2fe9b464b6..5374408c7a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,6 +67,7 @@ all = einops transformers<4.22 mlflow>=1.28.0 + clearml>=1.9.1 matplotlib tensorboardX pyyaml From a06afbe23a9f8906699037e2cd594cd1594b098f Mon Sep 17 00:00:00 2001 From: skinan Date: Fri, 3 Mar 2023 14:06:40 +0000 Subject: [PATCH 17/32] Update requirements-dev.txt --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 59f704b19c..573178c849 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -36,7 +36,7 @@ requests einops transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 mlflow>=1.28.0 -clearml >=1.9.1 +clearml>=1.9.1 matplotlib!=3.5.0 tensorboardX types-PyYAML From 4533232759b3c53b4a6557a8ddb6e78684a32baf Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 06:37:44 +0000 Subject: [PATCH 18/32] tracking removed clearml --- monai/bundle/scripts.py | 2 - monai/bundle/utils.py | 106 +--------------------------------------- 2 files changed, 1 insertion(+), 107 deletions(-) diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index c323164cd5..3d4b7288d4 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -650,8 +650,6 @@ def run( Default to "configs/logging.conf", which is commonly used for bundles in MONAI model zoo. tracking: enable the experiment tracking feature at runtime with optionally configurable and extensible. if "mlflow", will add `MLFlowHandler` to the parsed bundle with default logging settings, - if "clearml_stats", will add `ClearMLStatsHandler` to the parsed bundle with default logging settings, - if "clearml_image", will add `ClearMLImageHandler` to the parsed bundle with default logging settings, if other string, treat it as file path to load the logging settings, if `dict`, treat it as logging settings, otherwise, use all the default settings. will patch the target config content with `tracking handlers` and the top-level items of `configs`. diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index abea51c2ad..4f15514f9a 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -27,8 +27,6 @@ "EXPR_KEY", "MACRO_KEY", "DEFAULT_MLFLOW_SETTINGS", - "DEFAULT_CLEARML_STATS_SETTINGS", - "DEFAULT_CLEARML_IMAGE_SETTINGS", "DEFAULT_EXP_MGMT_SETTINGS", ] @@ -164,110 +162,8 @@ }, } -DEFAULT_CLEARML_STATS_SETTINGS = { - "handlers_id": DEFAULT_HANDLERS_ID, - "configs": { - "experiment_name": "monai_experiment", - # may fill it at runtime - "execute_config": None, - "is_not_rank0": ( - "$torch.distributed.is_available() \ - and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0" - ), - # ClearMLStatsHandler config for the trainer - "trainer": { - "_target_": "ClearMLStatsHandler", - "_disabled_": "@is_not_rank0", - "project_name": "MONAI", - "task_name": "@experiment_name", - "output_uri:": "True", - "tags": "None", - "reuse_last_task_id": "True", - "continue_last_task": "False", - "auto_connect_frameworks": "True", - }, - # ClearMLStatsHandler config for the validator - "validator": { - "_target_": "ClearMLStatsHandler", - "_disabled_": "@is_not_rank0", - "project_name": "MONAI", - "task_name": "@experiment_name", - "output_uri:": "True", - "tags": "None", - "reuse_last_task_id": "True", - "continue_last_task": "False", - "auto_connect_frameworks": "True", - }, - # ClearMLStatsHandler config for the evaluator - "evaluator": { - "_target_": "ClearMLStatsHandler", - "_disabled_": "@is_not_rank0", - "project_name": "MONAI", - "task_name": "@experiment_name", - "output_uri:": "True", - "tags": "None", - "reuse_last_task_id": "True", - "continue_last_task": "False", - "auto_connect_frameworks": "True", - }, - }, -} - -DEFAULT_CLEARML_IMAGE_SETTINGS = { - "handlers_id": DEFAULT_HANDLERS_ID, - "configs": { - "experiment_name": "monai_experiment", - # may fill it at runtime - "execute_config": None, - "is_not_rank0": ( - "$torch.distributed.is_available() \ - and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0" - ), - # ClearMLImageHandler config for the trainer - "trainer": { - "_target_": "ClearMLImageHandler", - "_disabled_": "@is_not_rank0", - "project_name": "MONAI", - "task_name": "@experiment_name", - "output_uri:": "True", - "tags": "None", - "reuse_last_task_id": "True", - "continue_last_task": "False", - "auto_connect_frameworks": "True", - }, - # ClearMLImageHandler config for the validator - "validator": { - "_target_": "ClearMLImageHandler", - "_disabled_": "@is_not_rank0", - "project_name": "MONAI", - "task_name": "@experiment_name", - "output_uri:": "True", - "tags": "None", - "reuse_last_task_id": "True", - "continue_last_task": "False", - "auto_connect_frameworks": "True", - }, - # ClearMLImageHandler config for the evaluator - "evaluator": { - "_target_": "ClearMLImageHandler", - "_disabled_": "@is_not_rank0", - "project_name": "MONAI", - "task_name": "@experiment_name", - "output_uri:": "True", - "tags": "None", - "reuse_last_task_id": "True", - "continue_last_task": "False", - "auto_connect_frameworks": "True", - }, - }, -} - -DEFAULT_EXP_MGMT_SETTINGS = { - "mlflow": DEFAULT_MLFLOW_SETTINGS, - "clearml_stats": DEFAULT_CLEARML_STATS_SETTINGS, - "clearml_image": DEFAULT_CLEARML_IMAGE_SETTINGS, -} # default experiment management settings +DEFAULT_EXP_MGMT_SETTINGS = {"mlflow": DEFAULT_MLFLOW_SETTINGS} # default experiment management settings def load_bundle_config(bundle_path: str, *config_names: str, **load_kw_args: Any) -> Any: From f2cfd605a39aeb884b0477e626da1a7e7b2e5e2a Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 07:17:27 +0000 Subject: [PATCH 19/32] Utils updated removed clearml and spaces --- monai/bundle/utils.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 4f15514f9a..64e4901a8c 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -21,14 +21,7 @@ yaml, _ = optional_import("yaml") -__all__ = [ - "ID_REF_KEY", - "ID_SEP_KEY", - "EXPR_KEY", - "MACRO_KEY", - "DEFAULT_MLFLOW_SETTINGS", - "DEFAULT_EXP_MGMT_SETTINGS", -] +__all__ = ["ID_REF_KEY","ID_SEP_KEY","EXPR_KEY","MACRO_KEY","DEFAULT_MLFLOW_SETTINGS","DEFAULT_EXP_MGMT_SETTINGS"] ID_REF_KEY = "@" # start of a reference to a ConfigItem ID_SEP_KEY = "#" # separator for the ID of a ConfigItem @@ -162,10 +155,8 @@ }, } - DEFAULT_EXP_MGMT_SETTINGS = {"mlflow": DEFAULT_MLFLOW_SETTINGS} # default experiment management settings - def load_bundle_config(bundle_path: str, *config_names: str, **load_kw_args: Any) -> Any: """ Load the metadata and nominated configuration files from a MONAI bundle without loading the network itself. From 03b8377727f10bee2d4e53910fed2af21319a7ef Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 07:23:47 +0000 Subject: [PATCH 20/32] Formaitting --- monai/bundle/utils.py | 3 ++- monai/handlers/clearml_handlers.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py index 64e4901a8c..62d4975d94 100644 --- a/monai/bundle/utils.py +++ b/monai/bundle/utils.py @@ -21,7 +21,7 @@ yaml, _ = optional_import("yaml") -__all__ = ["ID_REF_KEY","ID_SEP_KEY","EXPR_KEY","MACRO_KEY","DEFAULT_MLFLOW_SETTINGS","DEFAULT_EXP_MGMT_SETTINGS"] +__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY", "DEFAULT_MLFLOW_SETTINGS", "DEFAULT_EXP_MGMT_SETTINGS"] ID_REF_KEY = "@" # start of a reference to a ConfigItem ID_SEP_KEY = "#" # separator for the ID of a ConfigItem @@ -157,6 +157,7 @@ DEFAULT_EXP_MGMT_SETTINGS = {"mlflow": DEFAULT_MLFLOW_SETTINGS} # default experiment management settings + def load_bundle_config(bundle_path: str, *config_names: str, **load_kw_args: Any) -> Any: """ Load the metadata and nominated configuration files from a MONAI bundle without loading the network itself. diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 93fb9a9b32..d72419666a 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -29,6 +29,9 @@ class ClearMLHandler: For more details of ClearML usage, please refer to: https://clear.ml/docs/latest/docs/references/sdk/task + Usage example is available in the tutorial: + https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/unet_segmentation_3d_ignite.ipynb. + """ def __init__( @@ -73,6 +76,9 @@ class ClearMLStatsHandler(ClearMLHandler, TensorBoardStatsHandler): Class to write tensorboard stats by inheriting TensorBoardStatsHandler class. Everything from Tensorboard is logged automatically to ClearML WebUI. + Usage example is available in the tutorial: + https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/unet_segmentation_3d_ignite.ipynb. + """ def __init__( @@ -118,6 +124,9 @@ class ClearMLImageHandler(ClearMLHandler, TensorBoardImageHandler): This class inherits all functionality from TensorBoardImageHandler class. Everything from Tensorboard is logged automatically to ClearML WebUI. + Usage example is available in the tutorial: + https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/unet_segmentation_3d_ignite.ipynb. + """ def __init__( From ffbbf325dab0a85867188df8f418634c4da2c707 Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 07:47:15 +0000 Subject: [PATCH 21/32] length of "-" to align with words "ClearML handlers". --- docs/source/handlers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index f70f28b45e..7da7f7f50d 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -190,7 +190,7 @@ MLFlow handler :members: ClearML handlers --------------------- +---------------- .. autoclass:: ClearMLHandler :members: From e50f1cb95aa13deb64d65f60647b24335d13f8e1 Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 15:44:25 +0000 Subject: [PATCH 22/32] skinan Signed-off-by: skinan --- monai/handlers/clearml_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index d72419666a..b18c4c2cd1 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -140,7 +140,7 @@ def __init__( auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, *args, **kwargs, - ): + ) -> None: """ Args: project_name: ClearML project name, default to 'MONAI'. From 4cc3f04d618dd92d2d26af9a6a2e891ea76ce42b Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 15:53:02 +0000 Subject: [PATCH 23/32] DCO Remediation Commit for skinan I, skinan , hereby add my Signed-off-by to this commit: d4b52d12b646a41b6a1f93d530dcf409cbad40d2 I, skinan , hereby add my Signed-off-by to this commit: 2296a34069fa80ef27d642865c2c89963fd24734 I, skinan , hereby add my Signed-off-by to this commit: 3dbcd27d381f06558009debfc7f86e277f95710e I, skinan , hereby add my Signed-off-by to this commit: 6bcd8c6e1a7a59ddef331ee00c56dd60a7755182 I, skinan , hereby add my Signed-off-by to this commit: acec1ebc5828ec8816b96d4c7ad711d96cc790c5 I, skinan , hereby add my Signed-off-by to this commit: ef42a1cdef17a7d11623a49cd3e1c0b0751c3820 I, skinan , hereby add my Signed-off-by to this commit: 302d81e9ad4c1fce83d41cef788b10bc4fa1663e I, skinan , hereby add my Signed-off-by to this commit: 331953077f5e50c9d6bdc31c30d0a304b18dbef8 I, skinan , hereby add my Signed-off-by to this commit: 89fb3c04ea26f0976c2db091b9bc43f23b61df9c I, skinan , hereby add my Signed-off-by to this commit: 5dd1c095b3b98d7effeac3942a4c81cd18ef38c8 I, skinan , hereby add my Signed-off-by to this commit: 77146d6db397935e5533c63519ea805a634bf49d I, skinan , hereby add my Signed-off-by to this commit: 5dff6e416083abf9507a7c2f885d1a0d11640662 I, skinan , hereby add my Signed-off-by to this commit: a06afbe23a9f8906699037e2cd594cd1594b098f I, skinan , hereby add my Signed-off-by to this commit: 4533232759b3c53b4a6557a8ddb6e78684a32baf I, skinan , hereby add my Signed-off-by to this commit: f2cfd605a39aeb884b0477e626da1a7e7b2e5e2a I, skinan , hereby add my Signed-off-by to this commit: 03b8377727f10bee2d4e53910fed2af21319a7ef I, skinan , hereby add my Signed-off-by to this commit: ffbbf325dab0a85867188df8f418634c4da2c707 Signed-off-by: skinan --- monai/handlers/clearml_handlers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index b18c4c2cd1..9b647931aa 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -25,7 +25,7 @@ class ClearMLHandler: """ - Base class for the handlers to log everything to ClearML WebUI. + Base class for the handlers to log everything to ClearML. For more details of ClearML usage, please refer to: https://clear.ml/docs/latest/docs/references/sdk/task @@ -74,7 +74,7 @@ class ClearMLStatsHandler(ClearMLHandler, TensorBoardStatsHandler): """ Class to write tensorboard stats by inheriting TensorBoardStatsHandler class. - Everything from Tensorboard is logged automatically to ClearML WebUI. + Everything from Tensorboard is logged automatically to ClearML. Usage example is available in the tutorial: https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/unet_segmentation_3d_ignite.ipynb. @@ -122,7 +122,7 @@ class ClearMLImageHandler(ClearMLHandler, TensorBoardImageHandler): """ This class inherits all functionality from TensorBoardImageHandler class. - Everything from Tensorboard is logged automatically to ClearML WebUI. + Everything from Tensorboard is logged automatically to ClearML. Usage example is available in the tutorial: https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/unet_segmentation_3d_ignite.ipynb. From 0b52e16b7c90e7785d9320a683074067adf28ec6 Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 16:29:17 +0000 Subject: [PATCH 24/32] arg kwargs typehint added Signed-off-by: skinan --- monai/handlers/clearml_handlers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 9b647931aa..1657f5f4cc 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Mapping, Sequence +from typing import TYPE_CHECKING, Mapping, Sequence, Any from monai.utils import optional_import @@ -90,8 +90,8 @@ def __init__( reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, - *args, - **kwargs, + *args: Any, + **kwargs: Any, ) -> None: """ Args: @@ -138,8 +138,8 @@ def __init__( reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, - *args, - **kwargs, + *args: Any, + **kwargs: Any, ) -> None: """ Args: From 9b48bc1a1894c6290c5c1e0965f6700c25efb327 Mon Sep 17 00:00:00 2001 From: skinan Date: Mon, 6 Mar 2023 16:41:16 +0000 Subject: [PATCH 25/32] codeformatted Signed-off-by: skinan --- monai/handlers/clearml_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 1657f5f4cc..8ca4282dc3 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Mapping, Sequence, Any +from typing import TYPE_CHECKING, Any, Mapping, Sequence from monai.utils import optional_import From ce0ac23428f24e5e72217282dfe177cab332dd14 Mon Sep 17 00:00:00 2001 From: Victor Sonck Date: Tue, 7 Mar 2023 09:57:19 +0100 Subject: [PATCH 26/32] Add tensorboard check and skip --- tests/test_handler_clearml_image.py | 2 ++ tests/test_handler_clearml_stats.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/test_handler_clearml_image.py b/tests/test_handler_clearml_image.py index 2ba736e878..d03cfc74d6 100644 --- a/tests/test_handler_clearml_image.py +++ b/tests/test_handler_clearml_image.py @@ -17,9 +17,11 @@ from monai.utils import optional_import Task, has_clearml = optional_import("clearml", name="Task") +_, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter") @unittest.skipUnless(has_clearml, "Requires 'clearml' installation") +@unittest.skipUnless(has_tb, "Requires SummaryWriter installation") class TestHandlerClearMLImageHandler(unittest.TestCase): def test_task_init(self): Task.set_offline(offline_mode=True) diff --git a/tests/test_handler_clearml_stats.py b/tests/test_handler_clearml_stats.py index 947361e17e..0f02a6ad40 100644 --- a/tests/test_handler_clearml_stats.py +++ b/tests/test_handler_clearml_stats.py @@ -17,9 +17,11 @@ from monai.utils import optional_import Task, has_clearml = optional_import("clearml", name="Task") +_, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter") @unittest.skipUnless(has_clearml, "Requires 'clearml' installation") +@unittest.skipUnless(has_tb, "Requires SummaryWriter installation") class TestHandlerClearMLStatsHandler(unittest.TestCase): def test_task_init(self): Task.set_offline(offline_mode=True) From 619dbf2373a0dffb8ac0ec8f2c8c41fe71648274 Mon Sep 17 00:00:00 2001 From: skinan Date: Wed, 8 Mar 2023 17:10:44 +0000 Subject: [PATCH 27/32] close clearml tasks Signed-off-by: skinan --- tests/test_handler_clearml_image.py | 2 ++ tests/test_handler_clearml_stats.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/test_handler_clearml_image.py b/tests/test_handler_clearml_image.py index d03cfc74d6..4741b7de7a 100644 --- a/tests/test_handler_clearml_image.py +++ b/tests/test_handler_clearml_image.py @@ -39,6 +39,8 @@ def test_task_init(self): self.fail(exc) self.assertEqual(Task.current_task().name, "monai_experiment") self.assertEqual(Task.current_task()._project_name[1], "MONAI") + # Close ClearML Task + Task.current_task().close() if __name__ == "__main__": diff --git a/tests/test_handler_clearml_stats.py b/tests/test_handler_clearml_stats.py index 0f02a6ad40..9d860bafe8 100644 --- a/tests/test_handler_clearml_stats.py +++ b/tests/test_handler_clearml_stats.py @@ -39,6 +39,8 @@ def test_task_init(self): self.fail(exc) self.assertEqual(Task.current_task().name, "monai_experiment") self.assertEqual(Task.current_task()._project_name[1], "MONAI") + # Close ClearML Task + Task.current_task().close() if __name__ == "__main__": From 07692df6654588750f48ef41ef61c325e560c058 Mon Sep 17 00:00:00 2001 From: skinan Date: Thu, 9 Mar 2023 06:42:49 +0000 Subject: [PATCH 28/32] auto_connect_arg_parse=False Signed-off-by: skinan --- monai/handlers/clearml_handlers.py | 8 ++++++++ tests/test_handler_clearml_image.py | 1 + tests/test_handler_clearml_stats.py | 1 + 3 files changed, 10 insertions(+) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 8ca4282dc3..4db9efcba4 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -43,6 +43,7 @@ def __init__( reuse_last_task_id: bool, continue_last_task: bool, auto_connect_frameworks: bool | Mapping[str, bool | str | list], + auto_connect_arg_parser: bool | Mapping[str, bool], ) -> None: """ Args: @@ -53,6 +54,7 @@ def __init__( reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, default to 'True'. continue_last_task: Continue the execution of a previously executed Task (experiment), default to 'False'. auto_connect_frameworks: Automatically connect frameworks, default to 'True'. + auto_connect_arg_parser: Automatically connect an argparse object to the Task, default to 'True'. """ @@ -90,6 +92,7 @@ def __init__( reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, + auto_connect_arg_parser: bool | Mapping[str, bool] = True, *args: Any, **kwargs: Any, ) -> None: @@ -102,6 +105,7 @@ def __init__( reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, default to 'True'. continue_last_task: Continue the execution of a previously executed Task (experiment), default to 'False'. auto_connect_frameworks: Automatically connect frameworks, default to 'True'. + auto_connect_arg_parser: Automatically connect an argparse object to the Task, default to 'True'. """ @@ -114,6 +118,7 @@ def __init__( reuse_last_task_id=reuse_last_task_id, continue_last_task=continue_last_task, auto_connect_frameworks=auto_connect_frameworks, + auto_connect_arg_parser=auto_connect_arg_parser, ) TensorBoardStatsHandler.__init__(self, *args, **kwargs) @@ -138,6 +143,7 @@ def __init__( reuse_last_task_id: bool = True, continue_last_task: bool = False, auto_connect_frameworks: bool | Mapping[str, bool | str | list] = True, + auto_connect_arg_parser: bool | Mapping[str, bool] = True, *args: Any, **kwargs: Any, ) -> None: @@ -150,6 +156,7 @@ def __init__( reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID, default to 'True'. continue_last_task: Continue the execution of a previously executed Task (experiment), default to 'False'. auto_connect_frameworks: Automatically connect frameworks, default to 'True'. + auto_connect_arg_parser: Automatically connect an argparse object to the Task, default to 'True'. """ @@ -162,6 +169,7 @@ def __init__( reuse_last_task_id=reuse_last_task_id, continue_last_task=continue_last_task, auto_connect_frameworks=auto_connect_frameworks, + auto_connect_arg_parser=auto_connect_arg_parser, ) TensorBoardImageHandler.__init__(self, *args, **kwargs) diff --git a/tests/test_handler_clearml_image.py b/tests/test_handler_clearml_image.py index 4741b7de7a..6aedaed73b 100644 --- a/tests/test_handler_clearml_image.py +++ b/tests/test_handler_clearml_image.py @@ -34,6 +34,7 @@ def test_task_init(self): reuse_last_task_id=True, continue_last_task=False, auto_connect_frameworks=True, + auto_connect_arg_parser=False, ) except Exception as exc: self.fail(exc) diff --git a/tests/test_handler_clearml_stats.py b/tests/test_handler_clearml_stats.py index 9d860bafe8..3db324a33d 100644 --- a/tests/test_handler_clearml_stats.py +++ b/tests/test_handler_clearml_stats.py @@ -34,6 +34,7 @@ def test_task_init(self): reuse_last_task_id=True, continue_last_task=False, auto_connect_frameworks=True, + auto_connect_arg_parser=False, ) except Exception as exc: self.fail(exc) From c57884aadf181dd3827fe093f9659335cd49c04c Mon Sep 17 00:00:00 2001 From: skinan Date: Sun, 12 Mar 2023 06:43:53 +0000 Subject: [PATCH 29/32] clearml version updated Signed-off-by: skinan --- docs/requirements.txt | 2 +- requirements-dev.txt | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 40d476aa5c..a89961826f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -22,7 +22,7 @@ pandas einops transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 mlflow>=1.28.0 -clearml >=1.9.1 +clearml >=1.10.0rc0 tensorboardX imagecodecs; platform_system == "Linux" or platform_system == "Darwin" tifffile; platform_system == "Linux" or platform_system == "Darwin" diff --git a/requirements-dev.txt b/requirements-dev.txt index 573178c849..8b4433b39d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -36,7 +36,7 @@ requests einops transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 mlflow>=1.28.0 -clearml>=1.9.1 +clearml>=1.10.0rc0 matplotlib!=3.5.0 tensorboardX types-PyYAML diff --git a/setup.cfg b/setup.cfg index 5374408c7a..379a031b89 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ all = einops transformers<4.22 mlflow>=1.28.0 - clearml>=1.9.1 + clearml>=1.10.0rc0 matplotlib tensorboardX pyyaml From 41a58534f1406d3e66d3f684fa2c123c1517ec01 Mon Sep 17 00:00:00 2001 From: skinan Date: Sun, 12 Mar 2023 09:42:00 +0000 Subject: [PATCH 30/32] Change the position of optional import Task Signed-off-by: skinan --- monai/handlers/clearml_handlers.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 4db9efcba4..e6f8db3c4a 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -17,11 +17,6 @@ from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler -if TYPE_CHECKING: - from clearml import Task -else: - Task, _ = optional_import("clearml", name="Task") - class ClearMLHandler: """ @@ -58,6 +53,11 @@ def __init__( """ + if TYPE_CHECKING: + from clearml import Task + else: + Task, _ = optional_import("clearml", name="Task") + if Task.current_task(): self.clearml_task = Task.current_task() else: @@ -69,6 +69,7 @@ def __init__( reuse_last_task_id=reuse_last_task_id, continue_last_task=continue_last_task, auto_connect_frameworks=auto_connect_frameworks, + auto_connect_arg_parser=auto_connect_arg_parser, ) From 453d51e60de481031b46d8b2f3cc4c1231b6fd17 Mon Sep 17 00:00:00 2001 From: Victor Sonck Date: Sun, 12 Mar 2023 11:39:59 +0100 Subject: [PATCH 31/32] DCO Remediation Commit for Victor Sonck I, Victor Sonck , hereby add my Signed-off-by to this commit: ce0ac23428f24e5e72217282dfe177cab332dd14 Signed-off-by: Victor Sonck --- monai/handlers/clearml_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index e6f8db3c4a..6d73dad739 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -58,6 +58,8 @@ def __init__( else: Task, _ = optional_import("clearml", name="Task") + # Always check if the user didn't already add a `task.init`` in before + # if so, use that task, otherwise create a new one. if Task.current_task(): self.clearml_task = Task.current_task() else: From 109499f77d1499b5eb2423c4bb14c07eefef44d3 Mon Sep 17 00:00:00 2001 From: skinan Date: Sun, 12 Mar 2023 10:51:54 +0000 Subject: [PATCH 32/32] type hinting fixed Signed-off-by: skinan --- monai/handlers/clearml_handlers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monai/handlers/clearml_handlers.py b/monai/handlers/clearml_handlers.py index 6d73dad739..f4d6f197d2 100644 --- a/monai/handlers/clearml_handlers.py +++ b/monai/handlers/clearml_handlers.py @@ -54,16 +54,16 @@ def __init__( """ if TYPE_CHECKING: - from clearml import Task + import clearml else: - Task, _ = optional_import("clearml", name="Task") + clearml, _ = optional_import("clearml") # Always check if the user didn't already add a `task.init`` in before # if so, use that task, otherwise create a new one. - if Task.current_task(): - self.clearml_task = Task.current_task() + if clearml.Task.current_task(): + self.clearml_task = clearml.Task.current_task() else: - self.clearml_task = Task.init( + self.clearml_task = clearml.Task.init( project_name=project_name, task_name=task_name, output_uri=output_uri,