From efc48a64aeefde192f1856c9eb9db5869a49c854 Mon Sep 17 00:00:00 2001 From: heyufan1995 Date: Fri, 21 Apr 2023 09:40:00 -0400 Subject: [PATCH 1/5] Ensemble issue fix Signed-off-by: heyufan1995 --- monai/apps/auto3dseg/ensemble_builder.py | 10 ++++++++-- monai/apps/auto3dseg/utils.py | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/monai/apps/auto3dseg/ensemble_builder.py b/monai/apps/auto3dseg/ensemble_builder.py index 6f26b9a777..ecd2b049f9 100644 --- a/monai/apps/auto3dseg/ensemble_builder.py +++ b/monai/apps/auto3dseg/ensemble_builder.py @@ -175,8 +175,14 @@ def __call__(self, pred_param: dict | None = None) -> list: pred = infer_instance.predict(predict_files=[file], predict_params=param) preds.append(pred[0]) if "image_save_func" in param: - res = img_saver(self.ensemble_pred(preds, sigmoid=sigmoid)) + try: + ensemble_preds = self.ensemble_pred(preds, sigmoid=sigmoid) + except: + ensemble_preds = self.ensemble_pred([_.to('cpu') for _ in preds], sigmoid=sigmoid) + _ = img_saver(ensemble_preds) + res = None else: + warn('Prediction returned in list instead of disk, provide image_save_func to avoid out of memory.') res = self.ensemble_pred(preds, sigmoid=sigmoid) outputs.append(res) return outputs @@ -483,7 +489,7 @@ def ensemble(self): if history_untrained: if self.rank == 0: warn( - f"Ensembling step will skip {[h['name'] for h in history_untrained]} untrained algos." + f"Ensembling step will skip {[h[AlgoKeys.ID] for h in history_untrained]} untrained algos." "Generally it means these algos did not complete training." ) history = [h for h in history if h[AlgoKeys.IS_TRAINED]] diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 90de5e8f75..abca4616d9 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -50,6 +50,12 @@ def import_bundle_algo_history( algo.template_path = algo_meta_data["template_path"] best_metric = algo_meta_data.get(AlgoKeys.SCORE, None) + if best_metric is None: + try: + best_metric = algo.get_score() + except: + pass + is_trained = best_metric is not None if (only_trained and is_trained) or not only_trained: From 587d0dbd4126d6dd71395b4504f11d2ddcbd6ac6 Mon Sep 17 00:00:00 2001 From: heyufan1995 Date: Fri, 21 Apr 2023 10:28:23 -0400 Subject: [PATCH 2/5] Change Algo ensemble call to return save path Signed-off-by: heyufan1995 --- monai/apps/auto3dseg/ensemble_builder.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/monai/apps/auto3dseg/ensemble_builder.py b/monai/apps/auto3dseg/ensemble_builder.py index ecd2b049f9..4d13b59b7d 100644 --- a/monai/apps/auto3dseg/ensemble_builder.py +++ b/monai/apps/auto3dseg/ensemble_builder.py @@ -179,8 +179,13 @@ def __call__(self, pred_param: dict | None = None) -> list: ensemble_preds = self.ensemble_pred(preds, sigmoid=sigmoid) except: ensemble_preds = self.ensemble_pred([_.to('cpu') for _ in preds], sigmoid=sigmoid) - _ = img_saver(ensemble_preds) - res = None + res = img_saver(ensemble_preds) + # res is the path to the saved results + if hasattr(res,'meta') and 'saved_to' in res.meta.keys(): + res = res.meta['saved_to'] + else: + warn('Image save path not returned.') + res = None else: warn('Prediction returned in list instead of disk, provide image_save_func to avoid out of memory.') res = self.ensemble_pred(preds, sigmoid=sigmoid) @@ -457,6 +462,7 @@ def set_image_save_transform(self, **kwargs): "output_dtype": output_dtype, "resample": resample, "print_log": False, + "savepath_in_metadict": True } if kwargs: self.save_image.update(kwargs) From fff52942b55c91d9ac53a0808e249265657ea779 Mon Sep 17 00:00:00 2001 From: heyufan1995 Date: Fri, 21 Apr 2023 11:03:19 -0400 Subject: [PATCH 3/5] Add even_divisible to partition dataset to avoid hang Signed-off-by: heyufan1995 --- monai/apps/auto3dseg/ensemble_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/auto3dseg/ensemble_builder.py b/monai/apps/auto3dseg/ensemble_builder.py index 4d13b59b7d..9e4f8923af 100644 --- a/monai/apps/auto3dseg/ensemble_builder.py +++ b/monai/apps/auto3dseg/ensemble_builder.py @@ -509,7 +509,7 @@ def ensemble(self): builder.set_ensemble_method(self.ensemble_method) self.ensembler = builder.get_ensemble() infer_files = self.ensembler.infer_files - infer_files = partition_dataset(data=infer_files, shuffle=False, num_partitions=self.world_size)[self.rank] + infer_files = partition_dataset(data=infer_files, shuffle=False, num_partitions=self.world_size, even_divisible=True)[self.rank] # TO DO: Add some function in ensembler for infer_files update? self.ensembler.infer_files = infer_files # self.kwargs has poped out args for set_image_save_transform From b643c821447b24fcb100b6e61e9c7d66dbf5cb73 Mon Sep 17 00:00:00 2001 From: heyufan1995 Date: Fri, 21 Apr 2023 23:08:01 -0400 Subject: [PATCH 4/5] Change unit test cases Signed-off-by: heyufan1995 --- tests/test_auto3dseg_hpo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py index f8b9ebb2fb..08a51752e5 100644 --- a/tests/test_auto3dseg_hpo.py +++ b/tests/test_auto3dseg_hpo.py @@ -177,9 +177,8 @@ def test_get_history(self) -> None: obj_filename = nni_gen.get_obj_filename() NNIGen().run_algo(obj_filename, self.work_dir) - history = import_bundle_algo_history(self.work_dir, only_trained=True) - assert len(history) == 1 + assert len(history) == 3 def tearDown(self) -> None: self.test_dir.cleanup() From fe135b57df07ff89e86f07c60d3d48fa2e97e9ea Mon Sep 17 00:00:00 2001 From: monai-bot Date: Sat, 22 Apr 2023 12:47:09 +0000 Subject: [PATCH 5/5] [MONAI] code formatting Signed-off-by: monai-bot --- monai/apps/auto3dseg/ensemble_builder.py | 18 ++++++++++-------- monai/apps/auto3dseg/utils.py | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/monai/apps/auto3dseg/ensemble_builder.py b/monai/apps/auto3dseg/ensemble_builder.py index 9e4f8923af..0ac7ed4930 100644 --- a/monai/apps/auto3dseg/ensemble_builder.py +++ b/monai/apps/auto3dseg/ensemble_builder.py @@ -177,17 +177,17 @@ def __call__(self, pred_param: dict | None = None) -> list: if "image_save_func" in param: try: ensemble_preds = self.ensemble_pred(preds, sigmoid=sigmoid) - except: - ensemble_preds = self.ensemble_pred([_.to('cpu') for _ in preds], sigmoid=sigmoid) + except BaseException: + ensemble_preds = self.ensemble_pred([_.to("cpu") for _ in preds], sigmoid=sigmoid) res = img_saver(ensemble_preds) # res is the path to the saved results - if hasattr(res,'meta') and 'saved_to' in res.meta.keys(): - res = res.meta['saved_to'] + if hasattr(res, "meta") and "saved_to" in res.meta.keys(): + res = res.meta["saved_to"] else: - warn('Image save path not returned.') + warn("Image save path not returned.") res = None else: - warn('Prediction returned in list instead of disk, provide image_save_func to avoid out of memory.') + warn("Prediction returned in list instead of disk, provide image_save_func to avoid out of memory.") res = self.ensemble_pred(preds, sigmoid=sigmoid) outputs.append(res) return outputs @@ -462,7 +462,7 @@ def set_image_save_transform(self, **kwargs): "output_dtype": output_dtype, "resample": resample, "print_log": False, - "savepath_in_metadict": True + "savepath_in_metadict": True, } if kwargs: self.save_image.update(kwargs) @@ -509,7 +509,9 @@ def ensemble(self): builder.set_ensemble_method(self.ensemble_method) self.ensembler = builder.get_ensemble() infer_files = self.ensembler.infer_files - infer_files = partition_dataset(data=infer_files, shuffle=False, num_partitions=self.world_size, even_divisible=True)[self.rank] + infer_files = partition_dataset( + data=infer_files, shuffle=False, num_partitions=self.world_size, even_divisible=True + )[self.rank] # TO DO: Add some function in ensembler for infer_files update? self.ensembler.infer_files = infer_files # self.kwargs has poped out args for set_image_save_transform diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index abca4616d9..0e5c734cc6 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -53,7 +53,7 @@ def import_bundle_algo_history( if best_metric is None: try: best_metric = algo.get_score() - except: + except BaseException: pass is_trained = best_metric is not None