From b87ae5e67182518e267c37644461179abd0d3c90 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:43:59 +0800 Subject: [PATCH 01/10] workaround for #7575 Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 58 ++++++++++++---------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 44b3c24256..0308bcb1d6 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -22,6 +22,7 @@ from monai.apps.nnunet.utils import analyze_data, create_new_data_copy, create_new_dataset_json from monai.bundle import ConfigParser from monai.utils import ensure_tuple, optional_import +from monai.utils.misc import run_cmd load_pickle, _ = optional_import("batchgenerators.utilities.file_and_folder_operations", name="load_pickle") join, _ = optional_import("batchgenerators.utilities.file_and_folder_operations", name="join") @@ -522,38 +523,33 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int kwargs.pop("export_validation_probabilities") logger.warning("please specify the `export_validation_probabilities` in the __init__ of `nnUNetV2Runner`.") + # from nnunetv2.run.run_training import run_training + cmd = self.train_single_model_command(config, fold,gpu_id, kwargs) + run_cmd(cmd, shell=True) + + def train_single_model_command(self, config, fold, gpu_id, kwargs): if isinstance(gpu_id, (tuple, list)): if len(gpu_id) > 1: gpu_ids_str = "" for _i in range(len(gpu_id)): gpu_ids_str += f"{gpu_id[_i]}," - os.environ["CUDA_VISIBLE_DEVICES"] = gpu_ids_str[:-1] + set_visible_device = f"CUDA_VISIBLE_DEVICES={gpu_ids_str[:-1]}" else: - os.environ["CUDA_VISIBLE_DEVICES"] = f"{gpu_id[0]}" + set_visible_device = f"CUDA_VISIBLE_DEVICES={gpu_id[0]}" else: - os.environ["CUDA_VISIBLE_DEVICES"] = f"{gpu_id}" - - from nnunetv2.run.run_training import run_training + set_visible_device = f"CUDA_VISIBLE_DEVICES={gpu_id}" + num_gpus = 1 if isinstance(gpu_id, int) or len(gpu_id) == 1 else len(gpu_id) - if isinstance(gpu_id, int) or len(gpu_id) == 1: - run_training( - dataset_name_or_id=self.dataset_name_or_id, - configuration=config, - fold=fold, - trainer_class_name=self.trainer_class_name, - export_validation_probabilities=self.export_validation_probabilities, - **kwargs, - ) - else: - run_training( - dataset_name_or_id=self.dataset_name_or_id, - configuration=config, - fold=fold, - num_gpus=len(gpu_id), - trainer_class_name=self.trainer_class_name, - export_validation_probabilities=self.export_validation_probabilities, - **kwargs, - ) + cmd = ( + f"{set_visible_device} nnUNetv2_train " + + f"{self.dataset_name_or_id} {config} {fold} " + + f"-tr {self.trainer_class_name} -num_gpus {num_gpus}" + ) + if self.export_validation_probabilities: + cmd += " --npz" + for _key, _value in kwargs.items(): + cmd += f" --{_key} {_value}" + return cmd def train( self, @@ -637,15 +633,7 @@ def train_parallel_cmd( if _config in ensure_tuple(configs): for _i in range(self.num_folds): the_device = gpu_id_for_all[_index % n_devices] # type: ignore - cmd = ( - "python -m monai.apps.nnunet nnUNetV2Runner train_single_model " - + f"--input_config '{self.input_config_or_dict}' --work_dir '{self.work_dir}' " - + f"--config '{_config}' --fold {_i} --gpu_id {the_device} " - + f"--trainer_class_name {self.trainer_class_name} " - + f"--export_validation_probabilities {self.export_validation_probabilities}" - ) - for _key, _value in kwargs.items(): - cmd += f" --{_key} {_value}" + cmd = self.train_single_model_command(_config, _i, the_device, kwargs) all_cmds[-1][the_device].append(cmd) _index += 1 return all_cmds @@ -686,7 +674,9 @@ def train_parallel( continue cmd_str = "; ".join(stage[device_id]) logger.info(f"Current running command on GPU device {device_id}:\n{cmd_str}\n") - processes.append(subprocess.Popen(cmd_str, shell=True, stdout=subprocess.DEVNULL)) + processes.append(subprocess.Popen( + cmd_str, shell=True, stdout=subprocess.DEVNULL) + ) # finish this stage first for p in processes: p.wait() From 9319a3cd250d1c7734872dbf186e75a339409525 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:32:22 +0800 Subject: [PATCH 02/10] workaround for #7575 Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 8 ++++---- tests/test_set_visible_devices.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 0308bcb1d6..25038238ac 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -533,15 +533,15 @@ def train_single_model_command(self, config, fold, gpu_id, kwargs): gpu_ids_str = "" for _i in range(len(gpu_id)): gpu_ids_str += f"{gpu_id[_i]}," - set_visible_device = f"CUDA_VISIBLE_DEVICES={gpu_ids_str[:-1]}" + device_setting = f"CUDA_VISIBLE_DEVICES={gpu_ids_str[:-1]}" else: - set_visible_device = f"CUDA_VISIBLE_DEVICES={gpu_id[0]}" + device_setting = f"CUDA_VISIBLE_DEVICES={gpu_id[0]}" else: - set_visible_device = f"CUDA_VISIBLE_DEVICES={gpu_id}" + device_setting = f"CUDA_VISIBLE_DEVICES={gpu_id}" num_gpus = 1 if isinstance(gpu_id, int) or len(gpu_id) == 1 else len(gpu_id) cmd = ( - f"{set_visible_device} nnUNetv2_train " + f"{device_setting} nnUNetv2_train " + f"{self.dataset_name_or_id} {config} {fold} " + f"-tr {self.trainer_class_name} -num_gpus {num_gpus}" ) diff --git a/tests/test_set_visible_devices.py b/tests/test_set_visible_devices.py index 7860656b3d..b4f44957a2 100644 --- a/tests/test_set_visible_devices.py +++ b/tests/test_set_visible_devices.py @@ -14,7 +14,7 @@ import os import unittest -from tests.utils import skip_if_no_cuda +from tests.utils import SkipIfAtLeastPyTorchVersion, skip_if_no_cuda class TestVisibleDevices(unittest.TestCase): @@ -25,6 +25,7 @@ def run_process_and_get_exit_code(code_to_execute): return int(bin(value).replace("0b", "").rjust(16, "0")[:8], 2) @skip_if_no_cuda + @SkipIfAtLeastPyTorchVersion((2, 2, 1)) def test_visible_devices(self): num_gpus_before = self.run_process_and_get_exit_code( 'python -c "import os; import torch; ' From e26cab4cd1e340b839bbcfd1f2e4906599a3242f Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:34:04 +0800 Subject: [PATCH 03/10] fix flake8 Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 25038238ac..e761971ca3 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -524,7 +524,7 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int logger.warning("please specify the `export_validation_probabilities` in the __init__ of `nnUNetV2Runner`.") # from nnunetv2.run.run_training import run_training - cmd = self.train_single_model_command(config, fold,gpu_id, kwargs) + cmd = self.train_single_model_command(config, fold, gpu_id, kwargs) run_cmd(cmd, shell=True) def train_single_model_command(self, config, fold, gpu_id, kwargs): @@ -674,9 +674,7 @@ def train_parallel( continue cmd_str = "; ".join(stage[device_id]) logger.info(f"Current running command on GPU device {device_id}:\n{cmd_str}\n") - processes.append(subprocess.Popen( - cmd_str, shell=True, stdout=subprocess.DEVNULL) - ) + processes.append(subprocess.Popen(cmd_str, shell=True, stdout=subprocess.DEVNULL)) # finish this stage first for p in processes: p.wait() From aa6cb345392ac04f17565331940f092cc8fdab12 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:20:07 +0800 Subject: [PATCH 04/10] address comments Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index e761971ca3..9bb3c7b00a 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -523,7 +523,6 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int kwargs.pop("export_validation_probabilities") logger.warning("please specify the `export_validation_probabilities` in the __init__ of `nnUNetV2Runner`.") - # from nnunetv2.run.run_training import run_training cmd = self.train_single_model_command(config, fold, gpu_id, kwargs) run_cmd(cmd, shell=True) From 30ee4d35584af9ee83df30eb02a7e79d57fa9d2d Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:36:07 +0800 Subject: [PATCH 05/10] address comments Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 9bb3c7b00a..2b348d5178 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -496,17 +496,16 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int fold: fold of the 5-fold cross-validation. Should be an int between 0 and 4. gpu_id: an integer to select the device to use, or a tuple/list of GPU device indices used for multi-GPU training (e.g., (0,1)). Default: 0. - from nnunetv2.run.run_training import run_training kwargs: this optional parameter allows you to specify additional arguments in ``nnunetv2.run.run_training.run_training``. Currently supported args are - - plans_identifier: custom plans identifier. Default: "nnUNetPlans". + - p: custom plans identifier. Default: "nnUNetPlans". - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be used when actually training. Beta. Use with caution. Default: False. - - use_compressed_data: True to use compressed data for training. Reading compressed data is much + - use_compressed: True to use compressed data for training. Reading compressed data is much more CPU and (potentially) RAM intensive and should only be used if you know what you are doing. Default: False. - - continue_training: continue training from latest checkpoint. Default: False. - - only_run_validation: True to run the validation only. Requires training to have finished. + - c: continue training from latest checkpoint. Default: False. + - val: True to run the validation only. Requires training to have finished. Default: False. - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you don't want to flood your hard drive with checkpoints. Default: False. @@ -515,12 +514,12 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int kwargs.pop("num_gpus") logger.warning("please use gpu_id to set the GPUs to use") - if "trainer_class_name" in kwargs: - kwargs.pop("trainer_class_name") + if "tr" in kwargs: + kwargs.pop("tr") logger.warning("please specify the `trainer_class_name` in the __init__ of `nnUNetV2Runner`.") - if "export_validation_probabilities" in kwargs: - kwargs.pop("export_validation_probabilities") + if "npz" in kwargs: + kwargs.pop("npz") logger.warning("please specify the `export_validation_probabilities` in the __init__ of `nnUNetV2Runner`.") cmd = self.train_single_model_command(config, fold, gpu_id, kwargs) @@ -547,7 +546,10 @@ def train_single_model_command(self, config, fold, gpu_id, kwargs): if self.export_validation_probabilities: cmd += " --npz" for _key, _value in kwargs.items(): - cmd += f" --{_key} {_value}" + if _key == "p" or _key == "pretrained_weights": + cmd += f" -{_key} {_value}" + else: + cmd += f" --{_key} {_value}" return cmd def train( From b70cb32b947a1bd27a01ee0c3c17500d8189e73c Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:45:15 +0800 Subject: [PATCH 06/10] fix ci Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 2b348d5178..85d2fc7e37 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -498,17 +498,17 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int training (e.g., (0,1)). Default: 0. kwargs: this optional parameter allows you to specify additional arguments in ``nnunetv2.run.run_training.run_training``. Currently supported args are - - p: custom plans identifier. Default: "nnUNetPlans". - - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be - used when actually training. Beta. Use with caution. Default: False. - - use_compressed: True to use compressed data for training. Reading compressed data is much - more CPU and (potentially) RAM intensive and should only be used if you know what you are - doing. Default: False. - - c: continue training from latest checkpoint. Default: False. - - val: True to run the validation only. Requires training to have finished. - Default: False. - - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you - don't want to flood your hard drive with checkpoints. Default: False. + - p: custom plans identifier. Default: "nnUNetPlans". + - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be + used when actually training. Beta. Use with caution. Default: False. + - use_compressed: True to use compressed data for training. Reading compressed data is much + more CPU and (potentially) RAM intensive and should only be used if you know what you are + doing. Default: False. + - c: continue training from latest checkpoint. Default: False. + - val: True to run the validation only. Requires training to have finished. + Default: False. + - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you + don't want to flood your hard drive with checkpoints. Default: False. """ if "num_gpus" in kwargs: kwargs.pop("num_gpus") From f7e8c2980e5b0027ae3111de314f4fa142fde454 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 28 Mar 2024 23:25:29 +0800 Subject: [PATCH 07/10] address comments Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 85d2fc7e37..4b74f9fcb4 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -497,18 +497,19 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int gpu_id: an integer to select the device to use, or a tuple/list of GPU device indices used for multi-GPU training (e.g., (0,1)). Default: 0. kwargs: this optional parameter allows you to specify additional arguments in - ``nnunetv2.run.run_training.run_training``. Currently supported args are - - p: custom plans identifier. Default: "nnUNetPlans". - - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be - used when actually training. Beta. Use with caution. Default: False. - - use_compressed: True to use compressed data for training. Reading compressed data is much - more CPU and (potentially) RAM intensive and should only be used if you know what you are - doing. Default: False. - - c: continue training from latest checkpoint. Default: False. - - val: True to run the validation only. Requires training to have finished. - Default: False. - - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you - don't want to flood your hard drive with checkpoints. Default: False. + ``nnunetv2.run.run_training.run_training_entry``. + Currently supported args are: + - p: custom plans identifier. Default: "nnUNetPlans". + - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be + used when actually training. Beta. Use with caution. Default: False. + - use_compressed: True to use compressed data for training. Reading compressed data is much + more CPU and (potentially) RAM intensive and should only be used if you know what you are + doing. Default: False. + - c: continue training from latest checkpoint. Default: False. + - val: True to run the validation only. Requires training to have finished. + Default: False. + - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you + don't want to flood your hard drive with checkpoints. Default: False. """ if "num_gpus" in kwargs: kwargs.pop("num_gpus") From d86d45118132658db113f64998ee06169de868fa Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 28 Mar 2024 23:44:47 +0800 Subject: [PATCH 08/10] minor fix Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 4b74f9fcb4..2bfaecaeb4 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -499,17 +499,17 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int kwargs: this optional parameter allows you to specify additional arguments in ``nnunetv2.run.run_training.run_training_entry``. Currently supported args are: - - p: custom plans identifier. Default: "nnUNetPlans". - - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be - used when actually training. Beta. Use with caution. Default: False. - - use_compressed: True to use compressed data for training. Reading compressed data is much - more CPU and (potentially) RAM intensive and should only be used if you know what you are - doing. Default: False. - - c: continue training from latest checkpoint. Default: False. - - val: True to run the validation only. Requires training to have finished. - Default: False. - - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you - don't want to flood your hard drive with checkpoints. Default: False. + - p: custom plans identifier. Default: "nnUNetPlans". + - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be + used when actually training. Beta. Use with caution. Default: False. + - use_compressed: True to use compressed data for training. Reading compressed data is much + more CPU and (potentially) RAM intensive and should only be used if you know what you are + doing. Default: False. + - c: continue training from latest checkpoint. Default: False. + - val: True to run the validation only. Requires training to have finished. + Default: False. + - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you + don't want to flood your hard drive with checkpoints. Default: False. """ if "num_gpus" in kwargs: kwargs.pop("num_gpus") From bde9b8ba7987a0437f3ca9bc788459a7031bd883 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 28 Mar 2024 23:54:02 +0800 Subject: [PATCH 09/10] fix ci Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 2bfaecaeb4..9700e2873d 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -499,16 +499,16 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int kwargs: this optional parameter allows you to specify additional arguments in ``nnunetv2.run.run_training.run_training_entry``. Currently supported args are: - - p: custom plans identifier. Default: "nnUNetPlans". - - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be + p: custom plans identifier. Default: "nnUNetPlans". + pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be used when actually training. Beta. Use with caution. Default: False. - - use_compressed: True to use compressed data for training. Reading compressed data is much + use_compressed: True to use compressed data for training. Reading compressed data is much more CPU and (potentially) RAM intensive and should only be used if you know what you are doing. Default: False. - - c: continue training from latest checkpoint. Default: False. - - val: True to run the validation only. Requires training to have finished. + c: continue training from latest checkpoint. Default: False. + val: True to run the validation only. Requires training to have finished. Default: False. - - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you + disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you don't want to flood your hard drive with checkpoints. Default: False. """ if "num_gpus" in kwargs: From 16b01b8af22cda98f50d28ca51119f3d5b7fecf6 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Fri, 29 Mar 2024 00:10:45 +0800 Subject: [PATCH 10/10] fix ci Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- monai/apps/nnunet/nnunetv2_runner.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/monai/apps/nnunet/nnunetv2_runner.py b/monai/apps/nnunet/nnunetv2_runner.py index 9700e2873d..8a10849904 100644 --- a/monai/apps/nnunet/nnunetv2_runner.py +++ b/monai/apps/nnunet/nnunetv2_runner.py @@ -498,17 +498,19 @@ def train_single_model(self, config: Any, fold: int, gpu_id: tuple | list | int training (e.g., (0,1)). Default: 0. kwargs: this optional parameter allows you to specify additional arguments in ``nnunetv2.run.run_training.run_training_entry``. + Currently supported args are: - p: custom plans identifier. Default: "nnUNetPlans". - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be + + - p: custom plans identifier. Default: "nnUNetPlans". + - pretrained_weights: path to nnU-Net checkpoint file to be used as pretrained model. Will only be used when actually training. Beta. Use with caution. Default: False. - use_compressed: True to use compressed data for training. Reading compressed data is much + - use_compressed: True to use compressed data for training. Reading compressed data is much more CPU and (potentially) RAM intensive and should only be used if you know what you are doing. Default: False. - c: continue training from latest checkpoint. Default: False. - val: True to run the validation only. Requires training to have finished. + - c: continue training from latest checkpoint. Default: False. + - val: True to run the validation only. Requires training to have finished. Default: False. - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you + - disable_checkpointing: True to disable checkpointing. Ideal for testing things out and you don't want to flood your hard drive with checkpoints. Default: False. """ if "num_gpus" in kwargs: