From 8bb3d6f1aabe8acf9479b3b36643e01e9aa6795d Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Fri, 26 Mar 2021 20:54:42 -0400
Subject: [PATCH 01/34] Implement MaskedInferenceWSIDataset for pathology
 inference

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/datasets.py | 154 +++++++++++++++++++++++++++++--
 1 file changed, 145 insertions(+), 9 deletions(-)

diff --git a/monai/apps/pathology/datasets.py b/monai/apps/pathology/datasets.py
index a2f7b17ffe..2d70a171c1 100644
--- a/monai/apps/pathology/datasets.py
+++ b/monai/apps/pathology/datasets.py
@@ -9,15 +9,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
-from typing import Callable, List, Optional, Sequence, Tuple, Union
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 
 from monai.data import Dataset, SmartCacheDataset
 from monai.data.image_reader import WSIReader
 
-__all__ = ["PatchWSIDataset", "SmartCachePatchWSIDataset"]
+__all__ = ["PatchWSIDataset", "SmartCachePatchWSIDataset", "MaskedInferenceWSIDataset"]
 
 
 class PatchWSIDataset(Dataset):
@@ -26,13 +27,13 @@ class PatchWSIDataset(Dataset):
     It also reads labels for each patch and provides each patch with its associated class labels.
 
     Args:
-        data: the list of input samples including image, location, and label (see below for more details).
-        region_size: the region to be extracted from the whole slide image.
+        data: the list of input samples including image, location, and label (see the note below for more details).
+        region_size: the size of regions to be extracted from the whole slide image.
         grid_shape: the grid shape on which the patches should be extracted.
-        patch_size: the patches extracted from the region on the grid.
+        patch_size: the size of patches extracted from the region on the grid.
+        transform: transforms to be executed on input data.
         image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
             Defaults to CuCIM.
-        transform: transforms to be executed on input data.
 
     Note:
         The input data has the following form as an example:
@@ -52,8 +53,8 @@ def __init__(
         region_size: Union[int, Tuple[int, int]],
         grid_shape: Union[int, Tuple[int, int]],
         patch_size: int,
-        image_reader_name: str = "cuCIM",
         transform: Optional[Callable] = None,
+        image_reader_name: str = "cuCIM",
     ):
         super().__init__(data, transform)
 
@@ -71,7 +72,6 @@ def __init__(
         self.sub_region_size = (self.region_size[0] / self.grid_shape[0], self.region_size[1] / self.grid_shape[1])
 
         self.image_path_list = list({x["image"] for x in self.data})
-
         self.image_reader_name = image_reader_name
         self.image_reader = WSIReader(image_reader_name)
         self.wsi_object_dict = None
@@ -148,7 +148,13 @@ def __init__(
         num_replace_workers: Optional[int] = None,
         progress: bool = True,
     ):
-        patch_wsi_dataset = PatchWSIDataset(data, region_size, grid_shape, patch_size, image_reader_name)
+        patch_wsi_dataset = PatchWSIDataset(
+            data=data,
+            region_size=region_size,
+            grid_shape=grid_shape,
+            patch_size=patch_size,
+            image_reader_name=image_reader_name,
+        )
         super().__init__(
             data=patch_wsi_dataset,  # type: ignore
             transform=transform,
@@ -159,3 +165,133 @@ def __init__(
             num_replace_workers=num_replace_workers,
             progress=progress,
         )
+
+
+class MaskedInferenceWSIDataset(Dataset):
+    """
+    This dataset load the provided tissue masks at an arbitrary resolution level,
+    and extract patches based on that mask from the associated whole slide image.
+
+    Args:
+        data: a list of sample including path to the mask and path to the whole slide image
+            `[{"image": "path/to/image1.tiff", "label": "path/to/mask.npy}, ...]"`.
+        patch_size: the size of patches to be extracted from the whole slide image for inference.
+        transform: transforms to be executed on extracted patches.
+        image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
+            Defaults to CuCIM.
+
+    """
+
+    def __init__(
+        self,
+        data: List,
+        patch_size: Union[int, Tuple[int, int]],
+        transform: Optional[Callable] = None,
+        image_reader_name: str = "cuCIM",
+    ) -> None:
+        super().__init__(data, transform)
+
+        if isinstance(patch_size, int):
+            self.patch_size = np.array((patch_size, patch_size))
+        else:
+            self.patch_size = np.array(patch_size)
+        self.image_reader_name = image_reader_name
+        self.image_reader = WSIReader(image_reader_name)
+
+        # process data and create a list of dictionaries containing all required data and metadata
+        self.data_list = self._create_data_list(data)
+
+        # calculate cummulative number of patches for all whole slide images
+        self.cum_num_patches = np.cumsum([0] + [len(d["image_locations"]) for d in self.data_list])
+        self.total_num_patches = self.cum_num_patches[-1]
+
+    def _create_data_list(self, data: List[Dict]) -> List[Dict]:
+        data_list = []
+        print("Number of whole slide images: ", len(data))
+        for sample in data:
+            processed_data = self._preprocess_sample(sample)
+            data_list.append(processed_data)
+        return data_list
+
+    def _preprocess_sample(self, sample: Dict) -> Dict:
+        """
+        Preprocess input data to load WSIReader object and the foreground mask,
+        and define the locations where patches need to be extracted.
+        """
+        image = self.image_reader.read(sample["image"])
+        mask = np.load(sample["label"]).T
+        try:
+            level, ratio = self._calculate_mask_level(image, mask)
+        except ValueError as err:
+            err.args = (sample["label"],) + err.args
+            raise
+        print(f"Mask ({sample['label']}) at level {int(level)}, with ratio {int(ratio)}")
+
+        # get all indices for tissue region from the foreground mask
+        # note: output same size as the foreground mask and not original wsi image size
+        mask_locations = np.vstack(mask.nonzero()).T
+
+        # convert mask locations to image locations to extract patches
+        image_locations = (mask_locations + 0.5) * ratio - self.patch_size // 2
+
+        return {
+            "name": os.path.splitext(os.path.basename(sample["image"]))[0],
+            "image": image,
+            "mask_shape": mask.shape,
+            "mask_locations": mask_locations.astype(int).tolist(),
+            "image_locations": image_locations.astype(int).tolist(),
+            "level": level,
+        }
+
+    def _calculate_mask_level(self, image, mask) -> Tuple[int, int]:
+        """Calculate level of the mask and its ratio with respect to the whole slide image"""
+        dim_y_img, dim_x_img, _ = image.shape
+        dim_y_msk, dim_x_msk = mask.shape
+
+        ratio_x = dim_x_img / dim_x_msk
+        ratio_y = dim_y_img / dim_y_msk
+        level_x = np.log2(ratio_x)
+
+        if ratio_x != ratio_y:
+            raise ValueError(
+                "Image/Mask dimension does not match!"
+                " dim_x_img / dim_x_msk : {} / {},"
+                " dim_y_img / dim_y_msk : {} / {}".format(dim_x_img, dim_x_msk, dim_y_img, dim_y_msk)
+            )
+        elif not level_x.is_integer():
+            raise ValueError(
+                "Mask not at regular level (ratio not power of 2)," " image / mask ratio: {},".format(ratio_x)
+            )
+        return level_x, ratio_x
+
+    def _load_a_sample(self, index: int) -> Dict:
+        """
+        Load sample given the index
+
+        This method first, find the right patch to be extracted from the right whole slide image, and
+        then load this patch and provide it with its image name and the corrsponding mask location.
+        """
+        sample_num = np.argmax(self.cum_num_patches > index) - 1
+        sample = self.data_list[sample_num]
+        patch_num = index - self.cum_num_patches[sample_num]
+        image_location = sample["image_locations"][patch_num]
+        mask_location = sample["mask_locations"][patch_num]
+        image, _ = self.image_reader.get_data(
+            img=sample["image"],
+            location=image_location,
+            size=self.patch_size.tolist(),
+        )
+        processed_sample = {"image": image, "name": sample["name"], "mask_location": mask_location}
+        return processed_sample
+
+    def __len__(self):
+        return self.total_num_patches
+
+    def __getitem__(self, index):
+        try:
+            sample = self._load_a_sample(index)
+        except IndexError:
+            raise
+        if self.transform:
+            sample = self.transform(sample)
+        return sample

From 52e28da0a6e5f201f226e59787272e88d2c1666b Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Fri, 26 Mar 2021 20:55:54 -0400
Subject: [PATCH 02/34] Update pathology init

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/apps/pathology/__init__.py b/monai/apps/pathology/__init__.py
index 3af25365ba..e5028c152b 100644
--- a/monai/apps/pathology/__init__.py
+++ b/monai/apps/pathology/__init__.py
@@ -9,5 +9,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .datasets import PatchWSIDataset, SmartCacheDataset
+from .datasets import PatchWSIDataset, SmartCacheDataset, MaskedInferenceWSIDataset
 from .utils import ProbNMS

From d0fb5d3bc4291c5d45a2602c0688cd492c6cae7e Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Fri, 26 Mar 2021 20:56:33 -0400
Subject: [PATCH 03/34] Update docs

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 docs/source/apps.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/source/apps.rst b/docs/source/apps.rst
index 0c92d4c443..d81607c6b4 100644
--- a/docs/source/apps.rst
+++ b/docs/source/apps.rst
@@ -71,6 +71,8 @@ Applications
     :members:
 .. autoclass:: SmartCachePatchWSIDataset
     :members:
+.. autoclass:: MaskedInferenceWSIDataset
+    :members:
 
 .. automodule:: monai.apps.pathology.utils
 .. autoclass:: PathologyProbNMS

From 53d3edd2b8c0378fc430e07968b20fdedbe5f2ff Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Fri, 26 Mar 2021 21:56:24 -0400
Subject: [PATCH 04/34] Remove last elemnt of cum_num_patches

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/datasets.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/monai/apps/pathology/datasets.py b/monai/apps/pathology/datasets.py
index 2d70a171c1..08943b3f06 100644
--- a/monai/apps/pathology/datasets.py
+++ b/monai/apps/pathology/datasets.py
@@ -204,6 +204,7 @@ def __init__(
         # calculate cummulative number of patches for all whole slide images
         self.cum_num_patches = np.cumsum([0] + [len(d["image_locations"]) for d in self.data_list])
         self.total_num_patches = self.cum_num_patches[-1]
+        self.cum_num_patches = self.cum_num_patches[:-1]
 
     def _create_data_list(self, data: List[Dict]) -> List[Dict]:
         data_list = []
@@ -276,6 +277,7 @@ def _load_a_sample(self, index: int) -> Dict:
         patch_num = index - self.cum_num_patches[sample_num]
         image_location = sample["image_locations"][patch_num]
         mask_location = sample["mask_locations"][patch_num]
+
         image, _ = self.image_reader.get_data(
             img=sample["image"],
             location=image_location,

From dcba583d799a0d17f078a8b7308929448f158dca Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Fri, 26 Mar 2021 21:57:10 -0400
Subject: [PATCH 05/34] Add unittest with multiple cases for
 MaskedInferenceWSIDataset

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_masked_inference_wsi_dataset.py | 243 +++++++++++++++++++++
 1 file changed, 243 insertions(+)
 create mode 100644 tests/test_masked_inference_wsi_dataset.py

diff --git a/tests/test_masked_inference_wsi_dataset.py b/tests/test_masked_inference_wsi_dataset.py
new file mode 100644
index 0000000000..d69f54e0e9
--- /dev/null
+++ b/tests/test_masked_inference_wsi_dataset.py
@@ -0,0 +1,243 @@
+import os
+import unittest
+from unittest import skipUnless
+from urllib import request
+
+import numpy as np
+from numpy.testing import assert_array_equal
+from parameterized import parameterized
+
+from monai.apps.pathology.datasets import MaskedInferenceWSIDataset
+from monai.utils import optional_import
+from tests.utils import skip_if_quick
+
+_, has_cim = optional_import("cucim")
+_, has_osl = optional_import("openslide")
+
+FILE_URL = "http://openslide.cs.cmu.edu/download/openslide-testdata/Generic-TIFF/CMU-1.tiff"
+
+HEIGHT = 32914
+WIDTH = 46000
+
+mask = np.zeros((WIDTH // 2, HEIGHT // 2))
+mask[100, 100] = 1
+np.save("./tests/testing_data/mask1.npy", mask)
+mask[100, 100:102] = 1
+np.save("./tests/testing_data/mask2.npy", mask)
+mask[100:102, 100:102] = 1
+np.save("./tests/testing_data/mask4.npy", mask)
+
+TEST_CASE_0 = [
+    FILE_URL,
+    {
+        "data": [
+            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
+        ],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+    ],
+]
+
+TEST_CASE_1 = [
+    FILE_URL,
+    {
+        "data": [{"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask2.npy"}],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [101, 100],
+        },
+    ],
+]
+
+TEST_CASE_2 = [
+    FILE_URL,
+    {
+        "data": [{"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask4.npy"}],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 101],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [101, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [101, 101],
+        },
+    ],
+]
+
+TEST_CASE_3 = [
+    FILE_URL,
+    {
+        "data": [
+            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
+        ],
+        "patch_size": 2,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array(
+                [
+                    [[243, 243], [243, 243]],
+                    [[243, 243], [243, 243]],
+                    [[243, 243], [243, 243]],
+                ],
+                dtype=np.uint8,
+            ),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+    ],
+]
+
+TEST_CASE_4 = [
+    FILE_URL,
+    {
+        "data": [
+            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
+            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask2.npy"},
+        ],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [101, 100],
+        },
+    ],
+]
+
+
+TEST_CASE_OPENSLIDE_0 = [
+    FILE_URL,
+    {
+        "data": [
+            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
+        ],
+        "patch_size": 1,
+        "image_reader_name": "OpenSlide",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+    ],
+]
+
+TEST_CASE_OPENSLIDE_1 = [
+    FILE_URL,
+    {
+        "data": [{"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask2.npy"}],
+        "patch_size": 1,
+        "image_reader_name": "OpenSlide",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": "CMU-1",
+            "mask_location": [101, 100],
+        },
+    ],
+]
+
+
+class TestMaskedInferenceWSIDataset(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+            TEST_CASE_4,
+        ]
+    )
+    @skipUnless(has_cim, "Requires CuCIM")
+    @skip_if_quick
+    def test_read_patches_cucim(self, file_url, input_parameters, expected):
+        self.camelyon_data_download(file_url)
+        dataset = MaskedInferenceWSIDataset(**input_parameters)
+        samples = list(dataset)
+        self.compare_samples_expected(samples, expected)
+
+    @parameterized.expand(
+        [
+            TEST_CASE_OPENSLIDE_0,
+            TEST_CASE_OPENSLIDE_1,
+        ]
+    )
+    @skipUnless(has_osl, "Requires OpenSlide")
+    @skip_if_quick
+    def test_read_patches_openslide(self, file_url, input_parameters, expected):
+        self.camelyon_data_download(file_url)
+        dataset = MaskedInferenceWSIDataset(**input_parameters)
+        samples = list(dataset)
+        self.compare_samples_expected(samples, expected)
+
+    def camelyon_data_download(self, file_url):
+        filename = os.path.basename(file_url)
+        if not os.path.exists(filename):
+            print(f"Test image [{filename}] does not exist. Downloading...")
+            request.urlretrieve(file_url, filename)
+        return filename
+
+    def compare_samples_expected(self, samples, expected):
+        for i in range(len(samples)):
+            self.assertTupleEqual(samples[i]["image"].shape, expected[i]["image"].shape)
+            self.assertIsNone(assert_array_equal(samples[i]["image"], expected[i]["image"]))
+            self.assertEqual(samples[i]["name"], expected[i]["name"])
+            self.assertListEqual(samples[i]["mask_location"], expected[i]["mask_location"])
+
+
+if __name__ == "__main__":
+    unittest.main()

From 93d51ea8f7a87186ba923b639d23fa53c1663774 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Fri, 26 Mar 2021 22:06:25 -0400
Subject: [PATCH 06/34] sort imports in init

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/apps/pathology/__init__.py b/monai/apps/pathology/__init__.py
index e5028c152b..591edf1dad 100644
--- a/monai/apps/pathology/__init__.py
+++ b/monai/apps/pathology/__init__.py
@@ -9,5 +9,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .datasets import PatchWSIDataset, SmartCacheDataset, MaskedInferenceWSIDataset
+from .datasets import MaskedInferenceWSIDataset, PatchWSIDataset, SmartCacheDataset
 from .utils import ProbNMS

From 8d7e353551eaf97334a705b1ee29ff3cdeea4fba Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Fri, 26 Mar 2021 23:45:10 -0400
Subject: [PATCH 07/34] Remove list dataset

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_masked_inference_wsi_dataset.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/test_masked_inference_wsi_dataset.py b/tests/test_masked_inference_wsi_dataset.py
index d69f54e0e9..40b43a23a5 100644
--- a/tests/test_masked_inference_wsi_dataset.py
+++ b/tests/test_masked_inference_wsi_dataset.py
@@ -207,8 +207,7 @@ class TestMaskedInferenceWSIDataset(unittest.TestCase):
     def test_read_patches_cucim(self, file_url, input_parameters, expected):
         self.camelyon_data_download(file_url)
         dataset = MaskedInferenceWSIDataset(**input_parameters)
-        samples = list(dataset)
-        self.compare_samples_expected(samples, expected)
+        self.compare_samples_expected(dataset, expected)
 
     @parameterized.expand(
         [
@@ -221,8 +220,7 @@ def test_read_patches_cucim(self, file_url, input_parameters, expected):
     def test_read_patches_openslide(self, file_url, input_parameters, expected):
         self.camelyon_data_download(file_url)
         dataset = MaskedInferenceWSIDataset(**input_parameters)
-        samples = list(dataset)
-        self.compare_samples_expected(samples, expected)
+        self.compare_samples_expected(dataset, expected)
 
     def camelyon_data_download(self, file_url):
         filename = os.path.basename(file_url)
@@ -231,12 +229,12 @@ def camelyon_data_download(self, file_url):
             request.urlretrieve(file_url, filename)
         return filename
 
-    def compare_samples_expected(self, samples, expected):
-        for i in range(len(samples)):
-            self.assertTupleEqual(samples[i]["image"].shape, expected[i]["image"].shape)
-            self.assertIsNone(assert_array_equal(samples[i]["image"], expected[i]["image"]))
-            self.assertEqual(samples[i]["name"], expected[i]["name"])
-            self.assertListEqual(samples[i]["mask_location"], expected[i]["mask_location"])
+    def compare_samples_expected(self, dataset, expected):
+        for i in range(len(dataset)):
+            self.assertTupleEqual(dataset[i]["image"].shape, expected[i]["image"].shape)
+            self.assertIsNone(assert_array_equal(dataset[i]["image"], expected[i]["image"]))
+            self.assertEqual(dataset[i]["name"], expected[i]["name"])
+            self.assertListEqual(dataset[i]["mask_location"], expected[i]["mask_location"])
 
 
 if __name__ == "__main__":

From d0abacd75f6efe1988ba9b5330175f8e00195d1f Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sat, 27 Mar 2021 00:43:57 -0400
Subject: [PATCH 08/34] Remove try/except and add type hint

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/datasets.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/monai/apps/pathology/datasets.py b/monai/apps/pathology/datasets.py
index 7773da8a50..7e00c351dc 100644
--- a/monai/apps/pathology/datasets.py
+++ b/monai/apps/pathology/datasets.py
@@ -290,11 +290,8 @@ def _load_a_sample(self, index: int) -> Dict:
     def __len__(self):
         return self.total_num_patches
 
-    def __getitem__(self, index):
-        try:
-            sample = self._load_a_sample(index)
-        except IndexError:
-            raise
+    def __getitem__(self, index: int) -> Dict:
+        sample = self._load_a_sample(index)
         if self.transform:
             sample = self.transform(sample)
         return sample

From d450da24ab1aebfad0e0b2a83d84f8ba531af806 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sat, 27 Mar 2021 09:55:51 -0400
Subject: [PATCH 09/34] Convert the sample output to a list

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/datasets.py           | 4 ++--
 tests/test_masked_inference_wsi_dataset.py | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/monai/apps/pathology/datasets.py b/monai/apps/pathology/datasets.py
index 7e00c351dc..3e4032b6f9 100644
--- a/monai/apps/pathology/datasets.py
+++ b/monai/apps/pathology/datasets.py
@@ -290,8 +290,8 @@ def _load_a_sample(self, index: int) -> Dict:
     def __len__(self):
         return self.total_num_patches
 
-    def __getitem__(self, index: int) -> Dict:
-        sample = self._load_a_sample(index)
+    def __getitem__(self, index: int) -> List[Dict]:
+        sample = [self._load_a_sample(index)]
         if self.transform:
             sample = self.transform(sample)
         return sample
diff --git a/tests/test_masked_inference_wsi_dataset.py b/tests/test_masked_inference_wsi_dataset.py
index 40b43a23a5..5043310258 100644
--- a/tests/test_masked_inference_wsi_dataset.py
+++ b/tests/test_masked_inference_wsi_dataset.py
@@ -231,10 +231,10 @@ def camelyon_data_download(self, file_url):
 
     def compare_samples_expected(self, dataset, expected):
         for i in range(len(dataset)):
-            self.assertTupleEqual(dataset[i]["image"].shape, expected[i]["image"].shape)
-            self.assertIsNone(assert_array_equal(dataset[i]["image"], expected[i]["image"]))
-            self.assertEqual(dataset[i]["name"], expected[i]["name"])
-            self.assertListEqual(dataset[i]["mask_location"], expected[i]["mask_location"])
+            self.assertTupleEqual(dataset[i][0]["image"].shape, expected[i]["image"].shape)
+            self.assertIsNone(assert_array_equal(dataset[i][0]["image"], expected[i]["image"]))
+            self.assertEqual(dataset[i][0]["name"], expected[i]["name"])
+            self.assertListEqual(dataset[i][0]["mask_location"], expected[i]["mask_location"])
 
 
 if __name__ == "__main__":

From 1d02f0473b8a129a8a2657e7cff43f08231fcd60 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sat, 27 Mar 2021 10:29:18 -0400
Subject: [PATCH 10/34] Remove some type hints

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/datasets.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/monai/apps/pathology/datasets.py b/monai/apps/pathology/datasets.py
index 3e4032b6f9..9b3b023a08 100644
--- a/monai/apps/pathology/datasets.py
+++ b/monai/apps/pathology/datasets.py
@@ -266,7 +266,7 @@ def _calculate_mask_level(self, image, mask) -> Tuple[int, int]:
             )
         return level_x, ratio_x
 
-    def _load_a_sample(self, index: int) -> Dict:
+    def _load_a_sample(self, index):
         """
         Load sample given the index
 
@@ -290,7 +290,7 @@ def _load_a_sample(self, index: int) -> Dict:
     def __len__(self):
         return self.total_num_patches
 
-    def __getitem__(self, index: int) -> List[Dict]:
+    def __getitem__(self, index):
         sample = [self._load_a_sample(index)]
         if self.transform:
             sample = self.transform(sample)

From 3105c82c1588719183392111ef88c361858803e1 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sat, 27 Mar 2021 21:48:04 -0400
Subject: [PATCH 11/34] Implement FROC calcualtion for pathology

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/calculate_froc.py | 214 +++++++++++++++++++++++++
 1 file changed, 214 insertions(+)
 create mode 100644 monai/apps/pathology/calculate_froc.py

diff --git a/monai/apps/pathology/calculate_froc.py b/monai/apps/pathology/calculate_froc.py
new file mode 100644
index 0000000000..8a635ebc6d
--- /dev/null
+++ b/monai/apps/pathology/calculate_froc.py
@@ -0,0 +1,214 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from typing import Optional, Tuple, List, Dict
+
+import numpy as np
+
+from monai.data.image_reader import WSIReader
+from monai.metrics import compute_fp_tp_probs, compute_froc_curve_data, compute_froc_score
+from monai.utils import optional_import
+from monai.apps.pathology.utils import PathologyProbNMS
+
+measure, _ = optional_import("skimage.measure")
+ndimage, _ = optional_import("scipy.ndimage")
+
+
+class PathologyEvalFROC:
+    """
+    Evaluate with Free Response Operating Characteristic (FROC) score.
+
+    Args:
+        data: either the list of dictionaries containg probability maps (inference result) and 
+            tumor mask (ground truth), as below, or the path to a json file containing such list.
+            ```
+            [
+                {
+                    "prob_map": "path/to/prob1.npy",
+                    "tumor_mask": "path/to/image1.tiff",
+                    "level": 6,
+                    "pixel_spacing": 0.243
+                },
+                ...
+            ]
+            ```.
+        grow_distance: Euclidean distance (in micrometer) by which to grow the label the ground truth's tumors.
+            Defaults to 75, which is the equivalent size of 5 tumor cells.
+        itc_diameter: the maximum diameter of a region (in micrometer) to be considered as an isolated tumor cell.
+            Defaults to 200.
+        eval_thresholds: the false positive rates for calculating the average sensitivity.
+            Defaults to (0.25, 0.5, 1, 2, 4, 8) which is the same as the CAMELYON 16 Challenge.
+        image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
+            Defaults to CuCIM.
+    """
+
+    def __init__(
+        self,
+        data: Union(List[Dict], str),
+        grow_distance: int = 75,
+        itc_diameter: int = 200,
+        eval_thresholds: Tuple = (0.25, 0.5, 1, 2, 4, 8),
+        image_reader_name: str = "cuCIM",
+    ) -> None:
+
+        if isinstance(data, str):
+            self.data = self._load_data(data)
+        else:
+            self.data = data
+        self.grow_distance = grow_distance
+        self.itc_diameter = itc_diameter
+        self.eval_thresholds = eval_thresholds
+        self.image_reader = WSIReader(image_reader_name)
+        self.nms = PathologyProbNMS(
+            sigma=0.0,
+            prob_threshold=0.5,
+            box_size=48,
+        )
+
+    def _load_data(self, file_path):
+        with open(file_path, "r") as f:
+            data = json.load(f)
+        return data
+
+    def prepare_inference_result(self, sample):
+        """
+        Prepare the probability map for detection evaluation.
+
+        """
+        # load the probability map (the result of model inference)
+        prob_map = np.load(sample["prob_map"])
+
+        # apply non-maximal suppression
+        nms_outputs = self.nms(probs_map=prob_map, resolution_level=sample["level"])
+
+        # separate nms outputs
+        probs, x_coord, y_coord = zip(*nms_outputs)
+
+        return np.array(probs), np.array(x_coord), np.array(y_coord)
+
+    def prepare_ground_truth(self, sample):
+        """
+        Prapare the ground truth for evalution based on the binary tumor mask
+
+        """
+        # load binary tumor masks
+        img_obj = self.image_reader.read(sample["tumor_mask"])
+        tumor_mask = img_obj.get_data(level=sample["level"])[:, :, 0]
+
+        # calcualte pixel spacing at the mask level
+        mask_pixel_spacing = sample["pixel_spacing"] * pow(2, sample["level"])
+
+        # compute multi-instance mask from a binary mask
+        fill_hole_threshold = self.grow_distance / (mask_pixel_spacing * 2)
+        tumor_mask = self.compute_multi_instance_mask(mask=tumor_mask, threshold=fill_hole_threshold)
+
+        # identify isolated tumor cells
+        itc_threshold = (self.itc_diameter + self.grow_distance) / mask_pixel_spacing
+        itc_labels = self.compute_isolated_tumor_cells(tumor_mask=tumor_mask, threshold=itc_threshold)
+
+        return ground_truth, itc_labels
+
+    def compute_fp_tp(self):
+        """
+        Compute false positive and true positive probabilities for tumor detection,
+        by comparing the model outputs with the prepared ground truths for all samples
+
+        """
+        total_fp_probs, total_tp_probs = [], []
+        total_num_targets = 0
+        num_images = len(self.data)
+
+        for sample in self.data:
+            probs, x_coord, y_coord = self.prepare_inference_result(sample)
+            ground_truth, itc_labels = self.prepare_ground_truth(sample)
+
+            # compute FP and TP probabilities for a pair of an image and an ground truth mask
+            fp_probs, tp_probs, num_targets = compute_fp_tp_probs(
+                probs=probs,
+                y_coord=y_coord,
+                x_coord=x_coord,
+                evaluation_mask=ground_truth,
+                labels_to_exclude=itc_labels,
+                resolution_level=sample["level"],
+            )
+            total_fp_probs.extend(fp_probs)
+            total_tp_probs.extend(tp_probs)
+            total_num_targets += num_targets
+
+        return (
+            np.array(total_fp_probs),
+            np.array(total_tp_probs),
+            total_num_targets,
+            num_images,
+        )
+
+    def evaluate(self):
+        """
+        Evalaute the detection performance of a model based on the model probability map output,
+        the ground truth tumor mask, and their associated metadata (e.g., pixel_spacing, level)
+        """
+        # compute false positive (FP) and true positive (TP) probabilities for all images
+        fp_probs, tp_probs, num_targets, num_images = self.compute_fp_tp()
+
+        # compute FROC curve given the evaluation of all images
+        fps_per_image, total_sensitivity = compute_froc_curve_data(
+            fp_probs=fp_probs,
+            tp_probs=tp_probs,
+            num_targets=num_targets,
+            num_images=num_images,
+        )
+
+        # compute FROC score give specific evaluation threshold
+        froc_score = compute_froc_score(
+            fps_per_image=fps_per_image,
+            total_sensitivity=total_sensitivity,
+            eval_thresholds=self.eval_thresholds,
+        )
+
+        return froc_score
+
+    def compute_multi_instance_mask(self, mask: np.ndarray, threshold: float):
+        """
+        This method computes the segmentation mask according to the binary tumor mask.
+
+        Args:
+            mask: the binary mask array
+            threshold: the threashold to fill holes
+        """
+        assert 0.0 <= ms.max() <= 1.0
+        neg = 255 - mask * 255
+        distance = ndimage.morphology.distance_transform_edt(neg)
+        binary = distance < threshold
+
+        filled_image = ndimage.morphology.binary_fill_holes(binary)
+        multi_instance_mask = measure.label(filled_image, connectivity=2)
+
+        return multi_instance_mask
+
+    def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float) -> List[int]:
+        """
+        This method computes identifies Isolated Tumor Cells (ITC) and return their labels.
+
+        Args:
+            tumor_mask: the tumor mask.
+            threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
+                A region with the longest diameter less than this threshold is considered as an ITC.
+        """
+        max_label = np.amax(tumor_mask)
+        properties = measure.regionprops(tumor_mask, coordinates="rc")
+        itc_list = []
+        for i in range(max_label):
+            if properties[i].major_axis_length < threshold:
+                itc_list.append(i + 1)
+
+        return itc_list
\ No newline at end of file

From 56a002d17283d3f33943a2e920cfe458b8df9bea Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sat, 27 Mar 2021 21:48:25 -0400
Subject: [PATCH 12/34] Update ProbNMS doctring

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/utils/prob_nms.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/monai/utils/prob_nms.py b/monai/utils/prob_nms.py
index bdffdfe005..41ca80a988 100644
--- a/monai/utils/prob_nms.py
+++ b/monai/utils/prob_nms.py
@@ -22,8 +22,9 @@ class ProbNMS:
         prob_threshold: the probability threshold, the function will stop searching if
             the highest probability is no larger than the threshold. The value should be
             no less than 0.0. Defaults to 0.5.
-        box_size: determines the sizes of the removing area of the selected coordinates for
-            each dimensions. Defaults to 48.
+        box_size: the box size (in pixel) to be removed around the the pixel with the maximum probability.
+            It can an integer that defines the size of a square or cube, 
+            or a list containing different values for each dimensions. Defaults to 48.
 
     Return:
         a list of selected lists, where inner lists contain probability and coordinates.

From 3d776e48fa1f7733953abd5aae7978d028c3153a Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sat, 27 Mar 2021 22:01:26 -0400
Subject: [PATCH 13/34] Update docs and change namings

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 docs/source/apps.rst                                      | 4 ++++
 monai/apps/pathology/__init__.py                          | 1 +
 monai/apps/pathology/{calculate_froc.py => evaluators.py} | 2 +-
 3 files changed, 6 insertions(+), 1 deletion(-)
 rename monai/apps/pathology/{calculate_froc.py => evaluators.py} (99%)

diff --git a/docs/source/apps.rst b/docs/source/apps.rst
index d81607c6b4..02f5703d41 100644
--- a/docs/source/apps.rst
+++ b/docs/source/apps.rst
@@ -77,3 +77,7 @@ Applications
 .. automodule:: monai.apps.pathology.utils
 .. autoclass:: PathologyProbNMS
     :members:
+
+.. automodule:: monai.apps.pathology.evaluators
+.. autoclass:: EvaluateTumorFROC
+    :members:
diff --git a/monai/apps/pathology/__init__.py b/monai/apps/pathology/__init__.py
index 591edf1dad..a93edd0e89 100644
--- a/monai/apps/pathology/__init__.py
+++ b/monai/apps/pathology/__init__.py
@@ -10,4 +10,5 @@
 # limitations under the License.
 
 from .datasets import MaskedInferenceWSIDataset, PatchWSIDataset, SmartCacheDataset
+from .evaluators import EvaluateTumorFROC
 from .utils import ProbNMS
diff --git a/monai/apps/pathology/calculate_froc.py b/monai/apps/pathology/evaluators.py
similarity index 99%
rename from monai/apps/pathology/calculate_froc.py
rename to monai/apps/pathology/evaluators.py
index 8a635ebc6d..212739d4f1 100644
--- a/monai/apps/pathology/calculate_froc.py
+++ b/monai/apps/pathology/evaluators.py
@@ -24,7 +24,7 @@
 ndimage, _ = optional_import("scipy.ndimage")
 
 
-class PathologyEvalFROC:
+class EvaluateTumorFROC:
     """
     Evaluate with Free Response Operating Characteristic (FROC) score.
 

From c3cd6af06f4dae733e135ab3bdec043f378c4b7a Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sat, 27 Mar 2021 22:59:44 -0400
Subject: [PATCH 14/34] Fix a bug and minor changes

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 36 +++++++++++++++++-------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index 212739d4f1..9fa13e0bcc 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -11,14 +11,14 @@
 
 import json
 import os
-from typing import Optional, Tuple, List, Dict
+from typing import Union, Dict, List, Optional, Tuple
 
 import numpy as np
 
+from monai.apps.pathology.utils import PathologyProbNMS
 from monai.data.image_reader import WSIReader
 from monai.metrics import compute_fp_tp_probs, compute_froc_curve_data, compute_froc_score
 from monai.utils import optional_import
-from monai.apps.pathology.utils import PathologyProbNMS
 
 measure, _ = optional_import("skimage.measure")
 ndimage, _ = optional_import("scipy.ndimage")
@@ -29,13 +29,13 @@ class EvaluateTumorFROC:
     Evaluate with Free Response Operating Characteristic (FROC) score.
 
     Args:
-        data: either the list of dictionaries containg probability maps (inference result) and 
+        data: either the list of dictionaries containg probability maps (inference result) and
             tumor mask (ground truth), as below, or the path to a json file containing such list.
             ```
             [
                 {
-                    "prob_map": "path/to/prob1.npy",
-                    "tumor_mask": "path/to/image1.tiff",
+                    "prob_map": "path/to/prob_map_1.npy",
+                    "tumor_mask": "path/to/ground_truth_1.tiff",
                     "level": 6,
                     "pixel_spacing": 0.243
                 },
@@ -54,7 +54,7 @@ class EvaluateTumorFROC:
 
     def __init__(
         self,
-        data: Union(List[Dict], str),
+        data: Union[List[Dict], str],
         grow_distance: int = 75,
         itc_diameter: int = 200,
         eval_thresholds: Tuple = (0.25, 0.5, 1, 2, 4, 8),
@@ -92,7 +92,10 @@ def prepare_inference_result(self, sample):
         nms_outputs = self.nms(probs_map=prob_map, resolution_level=sample["level"])
 
         # separate nms outputs
-        probs, x_coord, y_coord = zip(*nms_outputs)
+        if nms_outputs:
+            probs, x_coord, y_coord = zip(*nms_outputs)
+        else:
+            probs, x_coord, y_coord = [], [], []
 
         return np.array(probs), np.array(x_coord), np.array(y_coord)
 
@@ -103,20 +106,21 @@ def prepare_ground_truth(self, sample):
         """
         # load binary tumor masks
         img_obj = self.image_reader.read(sample["tumor_mask"])
-        tumor_mask = img_obj.get_data(level=sample["level"])[:, :, 0]
-
+        
+        tumor_mask = self.image_reader.get_data(img_obj, level=sample["level"])[0][0]
+        print('> ', tumor_mask.shape, '|', tumor_mask.min(), '|', tumor_mask.max())
         # calcualte pixel spacing at the mask level
         mask_pixel_spacing = sample["pixel_spacing"] * pow(2, sample["level"])
 
         # compute multi-instance mask from a binary mask
-        fill_hole_threshold = self.grow_distance / (mask_pixel_spacing * 2)
-        tumor_mask = self.compute_multi_instance_mask(mask=tumor_mask, threshold=fill_hole_threshold)
+        grow_pixel_threshold = self.grow_distance / (mask_pixel_spacing * 2)
+        tumor_mask = self.compute_multi_instance_mask(mask=tumor_mask, threshold=grow_pixel_threshold)
 
         # identify isolated tumor cells
         itc_threshold = (self.itc_diameter + self.grow_distance) / mask_pixel_spacing
         itc_labels = self.compute_isolated_tumor_cells(tumor_mask=tumor_mask, threshold=itc_threshold)
 
-        return ground_truth, itc_labels
+        return tumor_mask, itc_labels
 
     def compute_fp_tp(self):
         """
@@ -129,9 +133,8 @@ def compute_fp_tp(self):
         num_images = len(self.data)
 
         for sample in self.data:
-            probs, x_coord, y_coord = self.prepare_inference_result(sample)
+            probs, y_coord, x_coord = self.prepare_inference_result(sample)
             ground_truth, itc_labels = self.prepare_ground_truth(sample)
-
             # compute FP and TP probabilities for a pair of an image and an ground truth mask
             fp_probs, tp_probs, num_targets = compute_fp_tp_probs(
                 probs=probs,
@@ -185,7 +188,8 @@ def compute_multi_instance_mask(self, mask: np.ndarray, threshold: float):
             mask: the binary mask array
             threshold: the threashold to fill holes
         """
-        assert 0.0 <= ms.max() <= 1.0
+        # make sure it is between 0 and 1
+        assert 0 <= mask.max() <= 1, "The input mask should be a binary mask!"
         neg = 255 - mask * 255
         distance = ndimage.morphology.distance_transform_edt(neg)
         binary = distance < threshold
@@ -211,4 +215,4 @@ def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float)
             if properties[i].major_axis_length < threshold:
                 itc_list.append(i + 1)
 
-        return itc_list
\ No newline at end of file
+        return itc_list

From bddcb4ff804553a9ee2ff3b7ec354642b6f7e6d9 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 00:34:00 -0400
Subject: [PATCH 15/34] Minor changes

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index 9fa13e0bcc..a9e05f601a 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -10,8 +10,7 @@
 # limitations under the License.
 
 import json
-import os
-from typing import Union, Dict, List, Optional, Tuple
+from typing import Dict, List, Tuple, Union
 
 import numpy as np
 
@@ -106,9 +105,8 @@ def prepare_ground_truth(self, sample):
         """
         # load binary tumor masks
         img_obj = self.image_reader.read(sample["tumor_mask"])
-        
         tumor_mask = self.image_reader.get_data(img_obj, level=sample["level"])[0][0]
-        print('> ', tumor_mask.shape, '|', tumor_mask.min(), '|', tumor_mask.max())
+
         # calcualte pixel spacing at the mask level
         mask_pixel_spacing = sample["pixel_spacing"] * pow(2, sample["level"])
 

From d6eca07160662b397a85e5443f993061b47c28c4 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 15:21:22 -0400
Subject: [PATCH 16/34] Fix docstring formatting

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/utils/prob_nms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/utils/prob_nms.py b/monai/utils/prob_nms.py
index 41ca80a988..5dfdaca8e0 100644
--- a/monai/utils/prob_nms.py
+++ b/monai/utils/prob_nms.py
@@ -23,7 +23,7 @@ class ProbNMS:
             the highest probability is no larger than the threshold. The value should be
             no less than 0.0. Defaults to 0.5.
         box_size: the box size (in pixel) to be removed around the the pixel with the maximum probability.
-            It can an integer that defines the size of a square or cube, 
+            It can an integer that defines the size of a square or cube,
             or a list containing different values for each dimensions. Defaults to 48.
 
     Return:

From 44dd729ff30963e68fac52089fe2023865f8dd95 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 19:50:41 -0400
Subject: [PATCH 17/34] Add a type hint

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index a9e05f601a..fd815e82ee 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -206,7 +206,7 @@ def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float)
             threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
                 A region with the longest diameter less than this threshold is considered as an ITC.
         """
-        max_label = np.amax(tumor_mask)
+        max_label: int = np.amax(tumor_mask)
         properties = measure.regionprops(tumor_mask, coordinates="rc")
         itc_list = []
         for i in range(max_label):

From 90d86dc0ed21900bc4a9abc92fdadbadf24a5a98 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 19:55:31 -0400
Subject: [PATCH 18/34] Implement unittests for EvaluateTumorFROC

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_evaluate_tumor_froc.py | 316 ++++++++++++++++++++++++++++++
 1 file changed, 316 insertions(+)
 create mode 100644 tests/test_evaluate_tumor_froc.py

diff --git a/tests/test_evaluate_tumor_froc.py b/tests/test_evaluate_tumor_froc.py
new file mode 100644
index 0000000000..e0768193a9
--- /dev/null
+++ b/tests/test_evaluate_tumor_froc.py
@@ -0,0 +1,316 @@
+import os
+import unittest
+from unittest import skipUnless
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.apps.pathology.evaluators import EvaluateTumorFROC
+from monai.utils import optional_import
+
+_, has_cucim = optional_import("cucim")
+_, has_skimage = optional_import("skimage.measure")
+_, has_sp = optional_import("scipy.ndimage")
+
+PILImage, has_pil = optional_import("PIL.Image")
+
+
+def save_as_tif(filename, array):
+    array = array[::-1, ...]  # Upside-down
+    img = PILImage.fromarray(array)
+    if not filename.endswith(".tif"):
+        filename += ".tif"
+    img.save(os.path.join("tests", "testing_data", filename))
+
+
+def around(val, interval=3):
+    return slice(val - interval, val + interval)
+
+
+# mask and prediction image size
+HEIGHT = 101
+WIDTH = 800
+
+# -------------------------------------
+# Ground Truth - Binary Masks
+# -------------------------------------
+
+# ground truth with no tumor
+ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
+save_as_tif("ground_truth_0", ground_truth)
+
+# ground truth with one tumor
+ground_truth[around(HEIGHT // 2), around(1 * WIDTH // 7)] = 1
+save_as_tif("ground_truth_1", ground_truth)
+
+# ground truth with two tumors
+ground_truth[around(HEIGHT // 2), around(2 * WIDTH // 7)] = 1
+save_as_tif("ground_truth_2", ground_truth)
+
+# ground truth with three tumors
+ground_truth[around(HEIGHT // 2), around(3 * WIDTH // 7)] = 1
+save_as_tif("ground_truth_3", ground_truth)
+
+# ground truth with four tumors
+ground_truth[around(HEIGHT // 2), around(4 * WIDTH // 7)] = 1
+save_as_tif("ground_truth_4", ground_truth)
+
+# -------------------------------------
+# predictions - Probability Maps
+# -------------------------------------
+
+# prediction with no tumor
+prob_map = np.zeros((HEIGHT, WIDTH))
+np.save("./tests/testing_data/prob_map_0_0.npy", prob_map)
+
+# prediction with one incorrect tumor
+prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
+np.save("./tests/testing_data/prob_map_0_1.npy", prob_map)
+
+# prediction with correct first tumors and an incorrect tumor
+prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
+np.save("./tests/testing_data/prob_map_1_1.npy", prob_map)
+
+# prediction with correct firt two tumors and an incorrect tumor
+prob_map[HEIGHT // 2, 2 * WIDTH // 7] = 0.8
+np.save("./tests/testing_data/prob_map_2_1.npy", prob_map)
+
+# prediction with two incorrect tumors
+prob_map = np.zeros((HEIGHT, WIDTH))
+prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
+prob_map[HEIGHT // 2, 6 * WIDTH // 7] = 0.4
+np.save("./tests/testing_data/prob_map_0_2.npy", prob_map)
+
+# prediction with correct first tumors and two incorrect tumors
+prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
+np.save("./tests/testing_data/prob_map_1_2.npy", prob_map)
+
+
+TEST_CASE_0 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_0_0.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_0.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    np.nan,
+]
+
+
+TEST_CASE_1 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_0_0.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    0.0,
+]
+
+TEST_CASE_2 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0,
+]
+
+TEST_CASE_3 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_2_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0,
+]
+
+
+TEST_CASE_4 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_2_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0,
+]
+
+TEST_CASE_5 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    0.5,
+]
+
+
+TEST_CASE_5 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    2.0 / 3.0,
+]
+
+TEST_CASE_6 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_3.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    0.4,
+]
+
+TEST_CASE_7 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_0_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_3.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0 / 3.0,
+]
+
+TEST_CASE_8 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/prob_map_0_2.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_4.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_3.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    2.0 / 9.0,
+]
+
+
+class TestEvaluateTumorFROC(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+            TEST_CASE_4,
+            TEST_CASE_5,
+            TEST_CASE_6,
+            TEST_CASE_7,
+            TEST_CASE_8,
+        ]
+    )
+    @skipUnless(has_cucim, "Requires cucim")
+    @skipUnless(has_skimage, "Requires skimage")
+    @skipUnless(has_sp, "Requires scipy")
+    def test_read_patches_cucim(self, input_parameters, expected):
+        froc = EvaluateTumorFROC(**input_parameters)
+        froc_score = froc.evaluate()
+        if np.isnan(expected):
+            self.assertTrue(np.isnan(froc_score))
+        else:
+            self.assertAlmostEqual(froc_score, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 11a8b6dc32552f25bd864f7e60def7ea8cd9d976 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 20:13:19 -0400
Subject: [PATCH 19/34] Ignore type for np.amax

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index fd815e82ee..2442f8a820 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -206,7 +206,7 @@ def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float)
             threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
                 A region with the longest diameter less than this threshold is considered as an ITC.
         """
-        max_label: int = np.amax(tumor_mask)
+        max_label = np.amax(tumor_mask) # type: ignore
         properties = measure.regionprops(tumor_mask, coordinates="rc")
         itc_list = []
         for i in range(max_label):

From c9e5a3f2811ff71e681ce92586b0d8f97524710a Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 20:13:59 -0400
Subject: [PATCH 20/34] Remove space

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index 2442f8a820..ef9d87524e 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -206,7 +206,7 @@ def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float)
             threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
                 A region with the longest diameter less than this threshold is considered as an ITC.
         """
-        max_label = np.amax(tumor_mask) # type: ignore
+        max_label = np.amax(tumor_mask)  # type: ignore
         properties = measure.regionprops(tumor_mask, coordinates="rc")
         itc_list = []
         for i in range(max_label):

From 486399a0152de958aaf958d1ca942a7caa512b6c Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 21:58:04 -0400
Subject: [PATCH 21/34] Ignore type for range instead of np.amax

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index ef9d87524e..f5e59b4ba8 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -209,7 +209,7 @@ def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float)
         max_label = np.amax(tumor_mask)  # type: ignore
         properties = measure.regionprops(tumor_mask, coordinates="rc")
         itc_list = []
-        for i in range(max_label):
+        for i in range(max_label):  # type: ignore
             if properties[i].major_axis_length < threshold:
                 itc_list.append(i + 1)
 

From 52fcc26bec393e93de22856c5188a5eaf3670c70 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 23:46:33 -0400
Subject: [PATCH 22/34] Skip test if PIL is not available

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_evaluate_tumor_froc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_evaluate_tumor_froc.py b/tests/test_evaluate_tumor_froc.py
index e0768193a9..3dae6d5663 100644
--- a/tests/test_evaluate_tumor_froc.py
+++ b/tests/test_evaluate_tumor_froc.py
@@ -11,7 +11,6 @@
 _, has_cucim = optional_import("cucim")
 _, has_skimage = optional_import("skimage.measure")
 _, has_sp = optional_import("scipy.ndimage")
-
 PILImage, has_pil = optional_import("PIL.Image")
 
 
@@ -303,6 +302,7 @@ class TestEvaluateTumorFROC(unittest.TestCase):
     @skipUnless(has_cucim, "Requires cucim")
     @skipUnless(has_skimage, "Requires skimage")
     @skipUnless(has_sp, "Requires scipy")
+    @skipUnless(has_pil, "Requires PIL")
     def test_read_patches_cucim(self, input_parameters, expected):
         froc = EvaluateTumorFROC(**input_parameters)
         froc_score = froc.evaluate()

From a5a75000c2929a4e61b2fc28f2bfd65d0f41cdc9 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Sun, 28 Mar 2021 23:47:03 -0400
Subject: [PATCH 23/34] Update docstring

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index f5e59b4ba8..ed7a2e923b 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -30,17 +30,12 @@ class EvaluateTumorFROC:
     Args:
         data: either the list of dictionaries containg probability maps (inference result) and
             tumor mask (ground truth), as below, or the path to a json file containing such list.
-            ```
-            [
-                {
-                    "prob_map": "path/to/prob_map_1.npy",
-                    "tumor_mask": "path/to/ground_truth_1.tiff",
-                    "level": 6,
-                    "pixel_spacing": 0.243
-                },
-                ...
-            ]
-            ```.
+            `{
+            "prob_map": "path/to/prob_map_1.npy",
+            "tumor_mask": "path/to/ground_truth_1.tiff",
+            "level": 6,
+            "pixel_spacing": 0.243
+            }`
         grow_distance: Euclidean distance (in micrometer) by which to grow the label the ground truth's tumors.
             Defaults to 75, which is the equivalent size of 5 tumor cells.
         itc_diameter: the maximum diameter of a region (in micrometer) to be considered as an isolated tumor cell.
@@ -49,6 +44,7 @@ class EvaluateTumorFROC:
             Defaults to (0.25, 0.5, 1, 2, 4, 8) which is the same as the CAMELYON 16 Challenge.
         image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
             Defaults to CuCIM.
+
     """
 
     def __init__(
@@ -206,7 +202,7 @@ def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float)
             threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
                 A region with the longest diameter less than this threshold is considered as an ITC.
         """
-        max_label = np.amax(tumor_mask)  # type: ignore
+        max_label = np.amax(tumor_mask)
         properties = measure.regionprops(tumor_mask, coordinates="rc")
         itc_list = []
         for i in range(max_label):  # type: ignore

From c98f8037c6b8c514f7fc1bf2e1add7ca1fc36991 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Mon, 29 Mar 2021 00:26:31 -0400
Subject: [PATCH 24/34] Skip ground truth generating if PIL is not available

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_evaluate_tumor_froc.py | 107 +++++++++++++++---------------
 1 file changed, 54 insertions(+), 53 deletions(-)

diff --git a/tests/test_evaluate_tumor_froc.py b/tests/test_evaluate_tumor_froc.py
index 3dae6d5663..5ff70c1feb 100644
--- a/tests/test_evaluate_tumor_froc.py
+++ b/tests/test_evaluate_tumor_froc.py
@@ -30,59 +30,60 @@ def around(val, interval=3):
 HEIGHT = 101
 WIDTH = 800
 
-# -------------------------------------
-# Ground Truth - Binary Masks
-# -------------------------------------
-
-# ground truth with no tumor
-ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
-save_as_tif("ground_truth_0", ground_truth)
-
-# ground truth with one tumor
-ground_truth[around(HEIGHT // 2), around(1 * WIDTH // 7)] = 1
-save_as_tif("ground_truth_1", ground_truth)
-
-# ground truth with two tumors
-ground_truth[around(HEIGHT // 2), around(2 * WIDTH // 7)] = 1
-save_as_tif("ground_truth_2", ground_truth)
-
-# ground truth with three tumors
-ground_truth[around(HEIGHT // 2), around(3 * WIDTH // 7)] = 1
-save_as_tif("ground_truth_3", ground_truth)
-
-# ground truth with four tumors
-ground_truth[around(HEIGHT // 2), around(4 * WIDTH // 7)] = 1
-save_as_tif("ground_truth_4", ground_truth)
-
-# -------------------------------------
-# predictions - Probability Maps
-# -------------------------------------
-
-# prediction with no tumor
-prob_map = np.zeros((HEIGHT, WIDTH))
-np.save("./tests/testing_data/prob_map_0_0.npy", prob_map)
-
-# prediction with one incorrect tumor
-prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
-np.save("./tests/testing_data/prob_map_0_1.npy", prob_map)
-
-# prediction with correct first tumors and an incorrect tumor
-prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
-np.save("./tests/testing_data/prob_map_1_1.npy", prob_map)
-
-# prediction with correct firt two tumors and an incorrect tumor
-prob_map[HEIGHT // 2, 2 * WIDTH // 7] = 0.8
-np.save("./tests/testing_data/prob_map_2_1.npy", prob_map)
-
-# prediction with two incorrect tumors
-prob_map = np.zeros((HEIGHT, WIDTH))
-prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
-prob_map[HEIGHT // 2, 6 * WIDTH // 7] = 0.4
-np.save("./tests/testing_data/prob_map_0_2.npy", prob_map)
-
-# prediction with correct first tumors and two incorrect tumors
-prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
-np.save("./tests/testing_data/prob_map_1_2.npy", prob_map)
+if has_pil:
+    # -------------------------------------
+    # Ground Truth - Binary Masks
+    # -------------------------------------
+
+    # ground truth with no tumor
+    ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
+    save_as_tif("ground_truth_0", ground_truth)
+
+    # ground truth with one tumor
+    ground_truth[around(HEIGHT // 2), around(1 * WIDTH // 7)] = 1
+    save_as_tif("ground_truth_1", ground_truth)
+
+    # ground truth with two tumors
+    ground_truth[around(HEIGHT // 2), around(2 * WIDTH // 7)] = 1
+    save_as_tif("ground_truth_2", ground_truth)
+
+    # ground truth with three tumors
+    ground_truth[around(HEIGHT // 2), around(3 * WIDTH // 7)] = 1
+    save_as_tif("ground_truth_3", ground_truth)
+
+    # ground truth with four tumors
+    ground_truth[around(HEIGHT // 2), around(4 * WIDTH // 7)] = 1
+    save_as_tif("ground_truth_4", ground_truth)
+
+    # -------------------------------------
+    # predictions - Probability Maps
+    # -------------------------------------
+
+    # prediction with no tumor
+    prob_map = np.zeros((HEIGHT, WIDTH))
+    np.save("./tests/testing_data/prob_map_0_0.npy", prob_map)
+
+    # prediction with one incorrect tumor
+    prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
+    np.save("./tests/testing_data/prob_map_0_1.npy", prob_map)
+
+    # prediction with correct first tumors and an incorrect tumor
+    prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
+    np.save("./tests/testing_data/prob_map_1_1.npy", prob_map)
+
+    # prediction with correct firt two tumors and an incorrect tumor
+    prob_map[HEIGHT // 2, 2 * WIDTH // 7] = 0.8
+    np.save("./tests/testing_data/prob_map_2_1.npy", prob_map)
+
+    # prediction with two incorrect tumors
+    prob_map = np.zeros((HEIGHT, WIDTH))
+    prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
+    prob_map[HEIGHT // 2, 6 * WIDTH // 7] = 0.4
+    np.save("./tests/testing_data/prob_map_0_2.npy", prob_map)
+
+    # prediction with correct first tumors and two incorrect tumors
+    prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
+    np.save("./tests/testing_data/prob_map_1_2.npy", prob_map)
 
 
 TEST_CASE_0 = [

From dbc0e24a2b15427590d95034b18c4e6290a299f5 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Mon, 29 Mar 2021 17:36:08 -0400
Subject: [PATCH 25/34] Update unittest

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_evaluate_tumor_froc.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/test_evaluate_tumor_froc.py b/tests/test_evaluate_tumor_froc.py
index 5ff70c1feb..c1a98b37c8 100644
--- a/tests/test_evaluate_tumor_froc.py
+++ b/tests/test_evaluate_tumor_froc.py
@@ -30,11 +30,11 @@ def around(val, interval=3):
 HEIGHT = 101
 WIDTH = 800
 
-if has_pil:
+def prepare_test_data():
     # -------------------------------------
     # Ground Truth - Binary Masks
     # -------------------------------------
-
+    print('*' * 69)
     # ground truth with no tumor
     ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
     save_as_tif("ground_truth_0", ground_truth)
@@ -287,6 +287,13 @@ def around(val, interval=3):
 
 
 class TestEvaluateTumorFROC(unittest.TestCase):
+    @skipUnless(has_cucim, "Requires cucim")
+    @skipUnless(has_skimage, "Requires skimage")
+    @skipUnless(has_sp, "Requires scipy")
+    @skipUnless(has_pil, "Requires PIL")
+    def setUp(self):
+        prepare_test_data()
+
     @parameterized.expand(
         [
             TEST_CASE_0,
@@ -300,10 +307,6 @@ class TestEvaluateTumorFROC(unittest.TestCase):
             TEST_CASE_8,
         ]
     )
-    @skipUnless(has_cucim, "Requires cucim")
-    @skipUnless(has_skimage, "Requires skimage")
-    @skipUnless(has_sp, "Requires scipy")
-    @skipUnless(has_pil, "Requires PIL")
     def test_read_patches_cucim(self, input_parameters, expected):
         froc = EvaluateTumorFROC(**input_parameters)
         froc_score = froc.evaluate()

From 5d8de3e872e605571a477599210353b221275e02 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Mon, 29 Mar 2021 19:27:05 -0400
Subject: [PATCH 26/34] Remove print

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_evaluate_tumor_froc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_evaluate_tumor_froc.py b/tests/test_evaluate_tumor_froc.py
index c1a98b37c8..0080a0f7a9 100644
--- a/tests/test_evaluate_tumor_froc.py
+++ b/tests/test_evaluate_tumor_froc.py
@@ -30,11 +30,11 @@ def around(val, interval=3):
 HEIGHT = 101
 WIDTH = 800
 
+
 def prepare_test_data():
     # -------------------------------------
     # Ground Truth - Binary Masks
     # -------------------------------------
-    print('*' * 69)
     # ground truth with no tumor
     ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
     save_as_tif("ground_truth_0", ground_truth)

From fd7c161797711754a323812e8db0901ef043ddc6 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Tue, 30 Mar 2021 15:57:31 -0400
Subject: [PATCH 27/34] Rename TumorFROC and add few type hints

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/evaluators.py | 10 +++++-----
 monai/utils/prob_nms.py            |  2 +-
 tests/test_evaluate_tumor_froc.py  |  4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/evaluators.py
index ed7a2e923b..d3a14c6b43 100644
--- a/monai/apps/pathology/evaluators.py
+++ b/monai/apps/pathology/evaluators.py
@@ -23,7 +23,7 @@
 ndimage, _ = optional_import("scipy.ndimage")
 
 
-class EvaluateTumorFROC:
+class TumorFROC:
     """
     Evaluate with Free Response Operating Characteristic (FROC) score.
 
@@ -70,9 +70,9 @@ def __init__(
             box_size=48,
         )
 
-    def _load_data(self, file_path):
+    def _load_data(self, file_path: str) -> Dict:
         with open(file_path, "r") as f:
-            data = json.load(f)
+            data: Dict = json.load(f)
         return data
 
     def prepare_inference_result(self, sample):
@@ -96,14 +96,14 @@ def prepare_inference_result(self, sample):
 
     def prepare_ground_truth(self, sample):
         """
-        Prapare the ground truth for evalution based on the binary tumor mask
+        Prepare the ground truth for evaluation based on the binary tumor mask
 
         """
         # load binary tumor masks
         img_obj = self.image_reader.read(sample["tumor_mask"])
         tumor_mask = self.image_reader.get_data(img_obj, level=sample["level"])[0][0]
 
-        # calcualte pixel spacing at the mask level
+        # calculate pixel spacing at the mask level
         mask_pixel_spacing = sample["pixel_spacing"] * pow(2, sample["level"])
 
         # compute multi-instance mask from a binary mask
diff --git a/monai/utils/prob_nms.py b/monai/utils/prob_nms.py
index 5dfdaca8e0..02456d6e27 100644
--- a/monai/utils/prob_nms.py
+++ b/monai/utils/prob_nms.py
@@ -23,7 +23,7 @@ class ProbNMS:
             the highest probability is no larger than the threshold. The value should be
             no less than 0.0. Defaults to 0.5.
         box_size: the box size (in pixel) to be removed around the the pixel with the maximum probability.
-            It can an integer that defines the size of a square or cube,
+            It can be an integer that defines the size of a square or cube,
             or a list containing different values for each dimensions. Defaults to 48.
 
     Return:
diff --git a/tests/test_evaluate_tumor_froc.py b/tests/test_evaluate_tumor_froc.py
index 5ff70c1feb..e295a25cf5 100644
--- a/tests/test_evaluate_tumor_froc.py
+++ b/tests/test_evaluate_tumor_froc.py
@@ -5,7 +5,7 @@
 import numpy as np
 from parameterized import parameterized
 
-from monai.apps.pathology.evaluators import EvaluateTumorFROC
+from monai.apps.pathology.evaluators import TumorFROC
 from monai.utils import optional_import
 
 _, has_cucim = optional_import("cucim")
@@ -305,7 +305,7 @@ class TestEvaluateTumorFROC(unittest.TestCase):
     @skipUnless(has_sp, "Requires scipy")
     @skipUnless(has_pil, "Requires PIL")
     def test_read_patches_cucim(self, input_parameters, expected):
-        froc = EvaluateTumorFROC(**input_parameters)
+        froc = TumorFROC(**input_parameters)
         froc_score = froc.evaluate()
         if np.isnan(expected):
             self.assertTrue(np.isnan(froc_score))

From e83dcdaace617abc8307164d525186a8f5ae63e1 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Tue, 30 Mar 2021 15:59:15 -0400
Subject: [PATCH 28/34] Rename evaluators to metrics

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/{evaluators.py => metrics.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename monai/apps/pathology/{evaluators.py => metrics.py} (100%)

diff --git a/monai/apps/pathology/evaluators.py b/monai/apps/pathology/metrics.py
similarity index 100%
rename from monai/apps/pathology/evaluators.py
rename to monai/apps/pathology/metrics.py

From 57d86efd242295c7c32ebfd8123dde68395b241d Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Tue, 30 Mar 2021 16:11:14 -0400
Subject: [PATCH 29/34] Remove non-relevant files

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/__init__.py              |   3 +-
 monai/apps/pathology/datasets.py              | 153 +----------
 tests/test_masked_inference_wsi_dataset.py    | 241 ------------------
 ...luate_tumor_froc.py => test_tumor_froc.py} |   0
 4 files changed, 10 insertions(+), 387 deletions(-)
 delete mode 100644 tests/test_masked_inference_wsi_dataset.py
 rename tests/{test_evaluate_tumor_froc.py => test_tumor_froc.py} (100%)

diff --git a/monai/apps/pathology/__init__.py b/monai/apps/pathology/__init__.py
index a93edd0e89..3af25365ba 100644
--- a/monai/apps/pathology/__init__.py
+++ b/monai/apps/pathology/__init__.py
@@ -9,6 +9,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .datasets import MaskedInferenceWSIDataset, PatchWSIDataset, SmartCacheDataset
-from .evaluators import EvaluateTumorFROC
+from .datasets import PatchWSIDataset, SmartCacheDataset
 from .utils import ProbNMS
diff --git a/monai/apps/pathology/datasets.py b/monai/apps/pathology/datasets.py
index 9b3b023a08..59f7e3aceb 100644
--- a/monai/apps/pathology/datasets.py
+++ b/monai/apps/pathology/datasets.py
@@ -9,16 +9,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import sys
-from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Callable, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 
 from monai.data import Dataset, SmartCacheDataset
 from monai.data.image_reader import WSIReader
 
-__all__ = ["PatchWSIDataset", "SmartCachePatchWSIDataset", "MaskedInferenceWSIDataset"]
+__all__ = ["PatchWSIDataset", "SmartCachePatchWSIDataset"]
 
 
 class PatchWSIDataset(Dataset):
@@ -27,13 +26,13 @@ class PatchWSIDataset(Dataset):
     It also reads labels for each patch and provides each patch with its associated class labels.
 
     Args:
-        data: the list of input samples including image, location, and label (see the note below for more details).
-        region_size: the size of regions to be extracted from the whole slide image.
+        data: the list of input samples including image, location, and label (see below for more details).
+        region_size: the region to be extracted from the whole slide image.
         grid_shape: the grid shape on which the patches should be extracted.
-        patch_size: the size of patches extracted from the region on the grid.
-        transform: transforms to be executed on input data.
+        patch_size: the patches extracted from the region on the grid.
         image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
             Defaults to CuCIM.
+        transform: transforms to be executed on input data.
 
     Note:
         The input data has the following form as an example:
@@ -53,8 +52,8 @@ def __init__(
         region_size: Union[int, Tuple[int, int]],
         grid_shape: Union[int, Tuple[int, int]],
         patch_size: int,
-        transform: Optional[Callable] = None,
         image_reader_name: str = "cuCIM",
+        transform: Optional[Callable] = None,
     ):
         super().__init__(data, transform)
 
@@ -72,6 +71,7 @@ def __init__(
         self.sub_region_size = (self.region_size[0] / self.grid_shape[0], self.region_size[1] / self.grid_shape[1])
 
         self.image_path_list = list({x["image"] for x in self.data})
+
         self.image_reader_name = image_reader_name
         self.image_reader = WSIReader(image_reader_name)
         self.wsi_object_dict = None
@@ -148,13 +148,7 @@ def __init__(
         num_replace_workers: Optional[int] = None,
         progress: bool = True,
     ):
-        patch_wsi_dataset = PatchWSIDataset(
-            data=data,
-            region_size=region_size,
-            grid_shape=grid_shape,
-            patch_size=patch_size,
-            image_reader_name=image_reader_name,
-        )
+        patch_wsi_dataset = PatchWSIDataset(data, region_size, grid_shape, patch_size, image_reader_name)
         super().__init__(
             data=patch_wsi_dataset,  # type: ignore
             transform=transform,
@@ -166,132 +160,3 @@ def __init__(
             progress=progress,
             shuffle=False,
         )
-
-
-class MaskedInferenceWSIDataset(Dataset):
-    """
-    This dataset load the provided tissue masks at an arbitrary resolution level,
-    and extract patches based on that mask from the associated whole slide image.
-
-    Args:
-        data: a list of sample including path to the mask and path to the whole slide image
-            `[{"image": "path/to/image1.tiff", "label": "path/to/mask.npy}, ...]"`.
-        patch_size: the size of patches to be extracted from the whole slide image for inference.
-        transform: transforms to be executed on extracted patches.
-        image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
-            Defaults to CuCIM.
-
-    """
-
-    def __init__(
-        self,
-        data: List,
-        patch_size: Union[int, Tuple[int, int]],
-        transform: Optional[Callable] = None,
-        image_reader_name: str = "cuCIM",
-    ) -> None:
-        super().__init__(data, transform)
-
-        if isinstance(patch_size, int):
-            self.patch_size = np.array((patch_size, patch_size))
-        else:
-            self.patch_size = np.array(patch_size)
-        self.image_reader_name = image_reader_name
-        self.image_reader = WSIReader(image_reader_name)
-
-        # process data and create a list of dictionaries containing all required data and metadata
-        self.data_list = self._create_data_list(data)
-
-        # calculate cummulative number of patches for all whole slide images
-        self.cum_num_patches = np.cumsum([0] + [len(d["image_locations"]) for d in self.data_list])
-        self.total_num_patches = self.cum_num_patches[-1]
-        self.cum_num_patches = self.cum_num_patches[:-1]
-
-    def _create_data_list(self, data: List[Dict]) -> List[Dict]:
-        data_list = []
-        print("Number of whole slide images: ", len(data))
-        for sample in data:
-            processed_data = self._preprocess_sample(sample)
-            data_list.append(processed_data)
-        return data_list
-
-    def _preprocess_sample(self, sample: Dict) -> Dict:
-        """
-        Preprocess input data to load WSIReader object and the foreground mask,
-        and define the locations where patches need to be extracted.
-        """
-        image = self.image_reader.read(sample["image"])
-        mask = np.load(sample["label"]).T
-        try:
-            level, ratio = self._calculate_mask_level(image, mask)
-        except ValueError as err:
-            err.args = (sample["label"],) + err.args
-            raise
-        print(f"Mask ({sample['label']}) at level {int(level)}, with ratio {int(ratio)}")
-
-        # get all indices for tissue region from the foreground mask
-        # note: output same size as the foreground mask and not original wsi image size
-        mask_locations = np.vstack(mask.nonzero()).T
-
-        # convert mask locations to image locations to extract patches
-        image_locations = (mask_locations + 0.5) * ratio - self.patch_size // 2
-
-        return {
-            "name": os.path.splitext(os.path.basename(sample["image"]))[0],
-            "image": image,
-            "mask_shape": mask.shape,
-            "mask_locations": mask_locations.astype(int).tolist(),
-            "image_locations": image_locations.astype(int).tolist(),
-            "level": level,
-        }
-
-    def _calculate_mask_level(self, image, mask) -> Tuple[int, int]:
-        """Calculate level of the mask and its ratio with respect to the whole slide image"""
-        dim_y_img, dim_x_img, _ = image.shape
-        dim_y_msk, dim_x_msk = mask.shape
-
-        ratio_x = dim_x_img / dim_x_msk
-        ratio_y = dim_y_img / dim_y_msk
-        level_x = np.log2(ratio_x)
-
-        if ratio_x != ratio_y:
-            raise ValueError(
-                "Image/Mask dimension does not match!"
-                " dim_x_img / dim_x_msk : {} / {},"
-                " dim_y_img / dim_y_msk : {} / {}".format(dim_x_img, dim_x_msk, dim_y_img, dim_y_msk)
-            )
-        elif not level_x.is_integer():
-            raise ValueError(
-                "Mask not at regular level (ratio not power of 2)," " image / mask ratio: {},".format(ratio_x)
-            )
-        return level_x, ratio_x
-
-    def _load_a_sample(self, index):
-        """
-        Load sample given the index
-
-        This method first, find the right patch to be extracted from the right whole slide image, and
-        then load this patch and provide it with its image name and the corrsponding mask location.
-        """
-        sample_num = np.argmax(self.cum_num_patches > index) - 1
-        sample = self.data_list[sample_num]
-        patch_num = index - self.cum_num_patches[sample_num]
-        image_location = sample["image_locations"][patch_num]
-        mask_location = sample["mask_locations"][patch_num]
-
-        image, _ = self.image_reader.get_data(
-            img=sample["image"],
-            location=image_location,
-            size=self.patch_size.tolist(),
-        )
-        processed_sample = {"image": image, "name": sample["name"], "mask_location": mask_location}
-        return processed_sample
-
-    def __len__(self):
-        return self.total_num_patches
-
-    def __getitem__(self, index):
-        sample = [self._load_a_sample(index)]
-        if self.transform:
-            sample = self.transform(sample)
-        return sample
diff --git a/tests/test_masked_inference_wsi_dataset.py b/tests/test_masked_inference_wsi_dataset.py
deleted file mode 100644
index 5043310258..0000000000
--- a/tests/test_masked_inference_wsi_dataset.py
+++ /dev/null
@@ -1,241 +0,0 @@
-import os
-import unittest
-from unittest import skipUnless
-from urllib import request
-
-import numpy as np
-from numpy.testing import assert_array_equal
-from parameterized import parameterized
-
-from monai.apps.pathology.datasets import MaskedInferenceWSIDataset
-from monai.utils import optional_import
-from tests.utils import skip_if_quick
-
-_, has_cim = optional_import("cucim")
-_, has_osl = optional_import("openslide")
-
-FILE_URL = "http://openslide.cs.cmu.edu/download/openslide-testdata/Generic-TIFF/CMU-1.tiff"
-
-HEIGHT = 32914
-WIDTH = 46000
-
-mask = np.zeros((WIDTH // 2, HEIGHT // 2))
-mask[100, 100] = 1
-np.save("./tests/testing_data/mask1.npy", mask)
-mask[100, 100:102] = 1
-np.save("./tests/testing_data/mask2.npy", mask)
-mask[100:102, 100:102] = 1
-np.save("./tests/testing_data/mask4.npy", mask)
-
-TEST_CASE_0 = [
-    FILE_URL,
-    {
-        "data": [
-            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
-        ],
-        "patch_size": 1,
-        "image_reader_name": "cuCIM",
-    },
-    [
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-    ],
-]
-
-TEST_CASE_1 = [
-    FILE_URL,
-    {
-        "data": [{"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask2.npy"}],
-        "patch_size": 1,
-        "image_reader_name": "cuCIM",
-    },
-    [
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [101, 100],
-        },
-    ],
-]
-
-TEST_CASE_2 = [
-    FILE_URL,
-    {
-        "data": [{"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask4.npy"}],
-        "patch_size": 1,
-        "image_reader_name": "cuCIM",
-    },
-    [
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 101],
-        },
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [101, 100],
-        },
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [101, 101],
-        },
-    ],
-]
-
-TEST_CASE_3 = [
-    FILE_URL,
-    {
-        "data": [
-            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
-        ],
-        "patch_size": 2,
-        "image_reader_name": "cuCIM",
-    },
-    [
-        {
-            "image": np.array(
-                [
-                    [[243, 243], [243, 243]],
-                    [[243, 243], [243, 243]],
-                    [[243, 243], [243, 243]],
-                ],
-                dtype=np.uint8,
-            ),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-    ],
-]
-
-TEST_CASE_4 = [
-    FILE_URL,
-    {
-        "data": [
-            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
-            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask2.npy"},
-        ],
-        "patch_size": 1,
-        "image_reader_name": "cuCIM",
-    },
-    [
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [101, 100],
-        },
-    ],
-]
-
-
-TEST_CASE_OPENSLIDE_0 = [
-    FILE_URL,
-    {
-        "data": [
-            {"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask1.npy"},
-        ],
-        "patch_size": 1,
-        "image_reader_name": "OpenSlide",
-    },
-    [
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-    ],
-]
-
-TEST_CASE_OPENSLIDE_1 = [
-    FILE_URL,
-    {
-        "data": [{"image": "./CMU-1.tiff", "label": "./tests/testing_data/mask2.npy"}],
-        "patch_size": 1,
-        "image_reader_name": "OpenSlide",
-    },
-    [
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [100, 100],
-        },
-        {
-            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
-            "name": "CMU-1",
-            "mask_location": [101, 100],
-        },
-    ],
-]
-
-
-class TestMaskedInferenceWSIDataset(unittest.TestCase):
-    @parameterized.expand(
-        [
-            TEST_CASE_0,
-            TEST_CASE_1,
-            TEST_CASE_2,
-            TEST_CASE_3,
-            TEST_CASE_4,
-        ]
-    )
-    @skipUnless(has_cim, "Requires CuCIM")
-    @skip_if_quick
-    def test_read_patches_cucim(self, file_url, input_parameters, expected):
-        self.camelyon_data_download(file_url)
-        dataset = MaskedInferenceWSIDataset(**input_parameters)
-        self.compare_samples_expected(dataset, expected)
-
-    @parameterized.expand(
-        [
-            TEST_CASE_OPENSLIDE_0,
-            TEST_CASE_OPENSLIDE_1,
-        ]
-    )
-    @skipUnless(has_osl, "Requires OpenSlide")
-    @skip_if_quick
-    def test_read_patches_openslide(self, file_url, input_parameters, expected):
-        self.camelyon_data_download(file_url)
-        dataset = MaskedInferenceWSIDataset(**input_parameters)
-        self.compare_samples_expected(dataset, expected)
-
-    def camelyon_data_download(self, file_url):
-        filename = os.path.basename(file_url)
-        if not os.path.exists(filename):
-            print(f"Test image [{filename}] does not exist. Downloading...")
-            request.urlretrieve(file_url, filename)
-        return filename
-
-    def compare_samples_expected(self, dataset, expected):
-        for i in range(len(dataset)):
-            self.assertTupleEqual(dataset[i][0]["image"].shape, expected[i]["image"].shape)
-            self.assertIsNone(assert_array_equal(dataset[i][0]["image"], expected[i]["image"]))
-            self.assertEqual(dataset[i][0]["name"], expected[i]["name"])
-            self.assertListEqual(dataset[i][0]["mask_location"], expected[i]["mask_location"])
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/tests/test_evaluate_tumor_froc.py b/tests/test_tumor_froc.py
similarity index 100%
rename from tests/test_evaluate_tumor_froc.py
rename to tests/test_tumor_froc.py

From 2e4b5fd5e3218ff4605797324aab56fd0d3e2129 Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Tue, 30 Mar 2021 16:22:05 -0400
Subject: [PATCH 30/34] Rename to LesionFROC and minor changes

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 docs/source/apps.rst                              | 6 ++----
 monai/apps/pathology/__init__.py                  | 1 +
 monai/apps/pathology/metrics.py                   | 8 ++++----
 tests/{test_tumor_froc.py => test_lesion_froc.py} | 4 ++--
 4 files changed, 9 insertions(+), 10 deletions(-)
 rename tests/{test_tumor_froc.py => test_lesion_froc.py} (98%)

diff --git a/docs/source/apps.rst b/docs/source/apps.rst
index 02f5703d41..95cc3a050c 100644
--- a/docs/source/apps.rst
+++ b/docs/source/apps.rst
@@ -71,13 +71,11 @@ Applications
     :members:
 .. autoclass:: SmartCachePatchWSIDataset
     :members:
-.. autoclass:: MaskedInferenceWSIDataset
-    :members:
 
 .. automodule:: monai.apps.pathology.utils
 .. autoclass:: PathologyProbNMS
     :members:
 
-.. automodule:: monai.apps.pathology.evaluators
-.. autoclass:: EvaluateTumorFROC
+.. automodule:: monai.apps.pathology.metrics
+.. autoclass:: LesionFROC
     :members:
diff --git a/monai/apps/pathology/__init__.py b/monai/apps/pathology/__init__.py
index 3af25365ba..0b638869e0 100644
--- a/monai/apps/pathology/__init__.py
+++ b/monai/apps/pathology/__init__.py
@@ -10,4 +10,5 @@
 # limitations under the License.
 
 from .datasets import PatchWSIDataset, SmartCacheDataset
+from .metrics import LesionFROC
 from .utils import ProbNMS
diff --git a/monai/apps/pathology/metrics.py b/monai/apps/pathology/metrics.py
index d3a14c6b43..67dd5014fc 100644
--- a/monai/apps/pathology/metrics.py
+++ b/monai/apps/pathology/metrics.py
@@ -23,7 +23,7 @@
 ndimage, _ = optional_import("scipy.ndimage")
 
 
-class TumorFROC:
+class LesionFROC:
     """
     Evaluate with Free Response Operating Characteristic (FROC) score.
 
@@ -70,12 +70,12 @@ def __init__(
             box_size=48,
         )
 
-    def _load_data(self, file_path: str) -> Dict:
+    def _load_data(self, file_path: str) -> List[Dict]:
         with open(file_path, "r") as f:
-            data: Dict = json.load(f)
+            data: List[Dict] = json.load(f)
         return data
 
-    def prepare_inference_result(self, sample):
+    def prepare_inference_result(self, sample: Dict):
         """
         Prepare the probability map for detection evaluation.
 
diff --git a/tests/test_tumor_froc.py b/tests/test_lesion_froc.py
similarity index 98%
rename from tests/test_tumor_froc.py
rename to tests/test_lesion_froc.py
index e295a25cf5..47198d15b0 100644
--- a/tests/test_tumor_froc.py
+++ b/tests/test_lesion_froc.py
@@ -5,7 +5,7 @@
 import numpy as np
 from parameterized import parameterized
 
-from monai.apps.pathology.evaluators import TumorFROC
+from monai.apps.pathology.metrics import LesionFROC
 from monai.utils import optional_import
 
 _, has_cucim = optional_import("cucim")
@@ -305,7 +305,7 @@ class TestEvaluateTumorFROC(unittest.TestCase):
     @skipUnless(has_sp, "Requires scipy")
     @skipUnless(has_pil, "Requires PIL")
     def test_read_patches_cucim(self, input_parameters, expected):
-        froc = TumorFROC(**input_parameters)
+        froc = LesionFROC(**input_parameters)
         froc_score = froc.evaluate()
         if np.isnan(expected):
             self.assertTrue(np.isnan(froc_score))

From f130796325e99179ee0659584b3865c4841388fa Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Tue, 30 Mar 2021 16:27:31 -0400
Subject: [PATCH 31/34] Update test

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 tests/test_lesion_froc.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/test_lesion_froc.py b/tests/test_lesion_froc.py
index 47198d15b0..c43ff98063 100644
--- a/tests/test_lesion_froc.py
+++ b/tests/test_lesion_froc.py
@@ -30,11 +30,11 @@ def around(val, interval=3):
 HEIGHT = 101
 WIDTH = 800
 
-if has_pil:
+
+def prepare_test_data():
     # -------------------------------------
     # Ground Truth - Binary Masks
     # -------------------------------------
-
     # ground truth with no tumor
     ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
     save_as_tif("ground_truth_0", ground_truth)
@@ -287,6 +287,13 @@ def around(val, interval=3):
 
 
 class TestEvaluateTumorFROC(unittest.TestCase):
+    @skipUnless(has_cucim, "Requires cucim")
+    @skipUnless(has_skimage, "Requires skimage")
+    @skipUnless(has_sp, "Requires scipy")
+    @skipUnless(has_pil, "Requires PIL")
+    def setUp(self):
+        prepare_test_data()
+
     @parameterized.expand(
         [
             TEST_CASE_0,
@@ -300,10 +307,6 @@ class TestEvaluateTumorFROC(unittest.TestCase):
             TEST_CASE_8,
         ]
     )
-    @skipUnless(has_cucim, "Requires cucim")
-    @skipUnless(has_skimage, "Requires skimage")
-    @skipUnless(has_sp, "Requires scipy")
-    @skipUnless(has_pil, "Requires PIL")
     def test_read_patches_cucim(self, input_parameters, expected):
         froc = LesionFROC(**input_parameters)
         froc_score = froc.evaluate()

From 9bfbc752b5c7126c24e0228d2681f4bdf6e9f61f Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Wed, 31 Mar 2021 10:55:08 -0400
Subject: [PATCH 32/34] Address PR comments

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 .gitignore                       |  3 ++
 docs/source/apps.rst             |  1 +
 monai/apps/pathology/__init__.py |  2 +-
 monai/apps/pathology/metrics.py  | 60 ++++++----------------
 monai/apps/pathology/utils.py    | 45 ++++++++++++++++-
 tests/test_lesion_froc.py        | 86 ++++++++++++++++----------------
 6 files changed, 105 insertions(+), 92 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4889d2d917..7444d7f2f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,6 +48,9 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 
+# temporary unittest artifacts
+tests/testing_data/temp_*
+
 # Translations
 *.mo
 *.pot
diff --git a/docs/source/apps.rst b/docs/source/apps.rst
index af7f9c5256..08addfec2e 100644
--- a/docs/source/apps.rst
+++ b/docs/source/apps.rst
@@ -75,6 +75,7 @@ Applications
     :members:
 
 .. automodule:: monai.apps.pathology.utils
+    :members:
 .. autoclass:: PathologyProbNMS
     :members:
 
diff --git a/monai/apps/pathology/__init__.py b/monai/apps/pathology/__init__.py
index 8c04031810..b4fdfa8dbd 100644
--- a/monai/apps/pathology/__init__.py
+++ b/monai/apps/pathology/__init__.py
@@ -11,4 +11,4 @@
 
 from .datasets import MaskedInferenceWSIDataset, PatchWSIDataset, SmartCacheDataset
 from .metrics import LesionFROC
-from .utils import ProbNMS
+from .utils import PathologyProbNMS, compute_isolated_tumor_cells, compute_multi_instance_mask
diff --git a/monai/apps/pathology/metrics.py b/monai/apps/pathology/metrics.py
index 67dd5014fc..8cacf61112 100644
--- a/monai/apps/pathology/metrics.py
+++ b/monai/apps/pathology/metrics.py
@@ -14,13 +14,9 @@
 
 import numpy as np
 
-from monai.apps.pathology.utils import PathologyProbNMS
+from monai.apps.pathology.utils import PathologyProbNMS, compute_isolated_tumor_cells, compute_multi_instance_mask
 from monai.data.image_reader import WSIReader
 from monai.metrics import compute_fp_tp_probs, compute_froc_curve_data, compute_froc_score
-from monai.utils import optional_import
-
-measure, _ = optional_import("skimage.measure")
-ndimage, _ = optional_import("scipy.ndimage")
 
 
 class LesionFROC:
@@ -28,7 +24,7 @@ class LesionFROC:
     Evaluate with Free Response Operating Characteristic (FROC) score.
 
     Args:
-        data: either the list of dictionaries containg probability maps (inference result) and
+        data: either the list of dictionaries containing probability maps (inference result) and
             tumor mask (ground truth), as below, or the path to a json file containing such list.
             `{
             "prob_map": "path/to/prob_map_1.npy",
@@ -42,9 +38,15 @@ class LesionFROC:
             Defaults to 200.
         eval_thresholds: the false positive rates for calculating the average sensitivity.
             Defaults to (0.25, 0.5, 1, 2, 4, 8) which is the same as the CAMELYON 16 Challenge.
+        nms_sigma: the standard deviation for gaussian filter of non-maximal suppression. Defaults to 0.0.
+        nms_prob_threshold: the probability threshold of non-maximal suppression. Defaults to 0.5.
+        nms_box_size: the box size (in pixel) to be removed around the the pixel for non-maximal suppression.
         image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
             Defaults to CuCIM.
 
+    Note:
+        For more info on `nms_*` parameters look at monai.utils.prob_nms.ProbNMS`.
+
     """
 
     def __init__(
@@ -53,6 +55,9 @@ def __init__(
         grow_distance: int = 75,
         itc_diameter: int = 200,
         eval_thresholds: Tuple = (0.25, 0.5, 1, 2, 4, 8),
+        nms_sigma: float = 0.0,
+        nms_prob_threshold: float = 0.5,
+        nms_box_size: int = 48,
         image_reader_name: str = "cuCIM",
     ) -> None:
 
@@ -108,11 +113,11 @@ def prepare_ground_truth(self, sample):
 
         # compute multi-instance mask from a binary mask
         grow_pixel_threshold = self.grow_distance / (mask_pixel_spacing * 2)
-        tumor_mask = self.compute_multi_instance_mask(mask=tumor_mask, threshold=grow_pixel_threshold)
+        tumor_mask = compute_multi_instance_mask(mask=tumor_mask, threshold=grow_pixel_threshold)
 
         # identify isolated tumor cells
         itc_threshold = (self.itc_diameter + self.grow_distance) / mask_pixel_spacing
-        itc_labels = self.compute_isolated_tumor_cells(tumor_mask=tumor_mask, threshold=itc_threshold)
+        itc_labels = compute_isolated_tumor_cells(tumor_mask=tumor_mask, threshold=itc_threshold)
 
         return tumor_mask, itc_labels
 
@@ -151,7 +156,7 @@ def compute_fp_tp(self):
 
     def evaluate(self):
         """
-        Evalaute the detection performance of a model based on the model probability map output,
+        Evaluate the detection performance of a model based on the model probability map output,
         the ground truth tumor mask, and their associated metadata (e.g., pixel_spacing, level)
         """
         # compute false positive (FP) and true positive (TP) probabilities for all images
@@ -173,40 +178,3 @@ def evaluate(self):
         )
 
         return froc_score
-
-    def compute_multi_instance_mask(self, mask: np.ndarray, threshold: float):
-        """
-        This method computes the segmentation mask according to the binary tumor mask.
-
-        Args:
-            mask: the binary mask array
-            threshold: the threashold to fill holes
-        """
-        # make sure it is between 0 and 1
-        assert 0 <= mask.max() <= 1, "The input mask should be a binary mask!"
-        neg = 255 - mask * 255
-        distance = ndimage.morphology.distance_transform_edt(neg)
-        binary = distance < threshold
-
-        filled_image = ndimage.morphology.binary_fill_holes(binary)
-        multi_instance_mask = measure.label(filled_image, connectivity=2)
-
-        return multi_instance_mask
-
-    def compute_isolated_tumor_cells(self, tumor_mask: np.ndarray, threshold: float) -> List[int]:
-        """
-        This method computes identifies Isolated Tumor Cells (ITC) and return their labels.
-
-        Args:
-            tumor_mask: the tumor mask.
-            threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
-                A region with the longest diameter less than this threshold is considered as an ITC.
-        """
-        max_label = np.amax(tumor_mask)
-        properties = measure.regionprops(tumor_mask, coordinates="rc")
-        itc_list = []
-        for i in range(max_label):  # type: ignore
-            if properties[i].major_axis_length < threshold:
-                itc_list.append(i + 1)
-
-        return itc_list
diff --git a/monai/apps/pathology/utils.py b/monai/apps/pathology/utils.py
index b0803526fd..ae77bfafd1 100644
--- a/monai/apps/pathology/utils.py
+++ b/monai/apps/pathology/utils.py
@@ -9,12 +9,53 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Union
+from typing import List, Union
 
 import numpy as np
 import torch
 
-from monai.utils import ProbNMS
+from monai.utils import ProbNMS, optional_import
+
+measure, _ = optional_import("skimage.measure")
+ndimage, _ = optional_import("scipy.ndimage")
+
+
+def compute_multi_instance_mask(mask: np.ndarray, threshold: float):
+    """
+    This method computes the segmentation mask according to the binary tumor mask.
+
+    Args:
+        mask: the binary mask array
+        threshold: the threshold to fill holes
+    """
+
+    neg = 255 - mask * 255
+    distance = ndimage.morphology.distance_transform_edt(neg)
+    binary = distance < threshold
+
+    filled_image = ndimage.morphology.binary_fill_holes(binary)
+    multi_instance_mask = measure.label(filled_image, connectivity=2)
+
+    return multi_instance_mask
+
+
+def compute_isolated_tumor_cells(tumor_mask: np.ndarray, threshold: float) -> List[int]:
+    """
+    This method computes identifies Isolated Tumor Cells (ITC) and return their labels.
+
+    Args:
+        tumor_mask: the tumor mask.
+        threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
+            A region with the longest diameter less than this threshold is considered as an ITC.
+    """
+    max_label = np.amax(tumor_mask)
+    properties = measure.regionprops(tumor_mask, coordinates="rc")
+    itc_list = []
+    for i in range(max_label):  # type: ignore
+        if properties[i].major_axis_length < threshold:
+            itc_list.append(i + 1)
+
+    return itc_list
 
 
 class PathologyProbNMS(ProbNMS):
diff --git a/tests/test_lesion_froc.py b/tests/test_lesion_froc.py
index c43ff98063..6702997c64 100644
--- a/tests/test_lesion_froc.py
+++ b/tests/test_lesion_froc.py
@@ -37,23 +37,23 @@ def prepare_test_data():
     # -------------------------------------
     # ground truth with no tumor
     ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
-    save_as_tif("ground_truth_0", ground_truth)
+    save_as_tif("temp_ground_truth_0", ground_truth)
 
     # ground truth with one tumor
     ground_truth[around(HEIGHT // 2), around(1 * WIDTH // 7)] = 1
-    save_as_tif("ground_truth_1", ground_truth)
+    save_as_tif("temp_ground_truth_1", ground_truth)
 
     # ground truth with two tumors
     ground_truth[around(HEIGHT // 2), around(2 * WIDTH // 7)] = 1
-    save_as_tif("ground_truth_2", ground_truth)
+    save_as_tif("temp_ground_truth_2", ground_truth)
 
     # ground truth with three tumors
     ground_truth[around(HEIGHT // 2), around(3 * WIDTH // 7)] = 1
-    save_as_tif("ground_truth_3", ground_truth)
+    save_as_tif("temp_ground_truth_3", ground_truth)
 
     # ground truth with four tumors
     ground_truth[around(HEIGHT // 2), around(4 * WIDTH // 7)] = 1
-    save_as_tif("ground_truth_4", ground_truth)
+    save_as_tif("temp_ground_truth_4", ground_truth)
 
     # -------------------------------------
     # predictions - Probability Maps
@@ -61,37 +61,37 @@ def prepare_test_data():
 
     # prediction with no tumor
     prob_map = np.zeros((HEIGHT, WIDTH))
-    np.save("./tests/testing_data/prob_map_0_0.npy", prob_map)
+    np.save("./tests/testing_data/temp_prob_map_0_0.npy", prob_map)
 
     # prediction with one incorrect tumor
     prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
-    np.save("./tests/testing_data/prob_map_0_1.npy", prob_map)
+    np.save("./tests/testing_data/temp_prob_map_0_1.npy", prob_map)
 
     # prediction with correct first tumors and an incorrect tumor
     prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
-    np.save("./tests/testing_data/prob_map_1_1.npy", prob_map)
+    np.save("./tests/testing_data/temp_prob_map_1_1.npy", prob_map)
 
     # prediction with correct firt two tumors and an incorrect tumor
     prob_map[HEIGHT // 2, 2 * WIDTH // 7] = 0.8
-    np.save("./tests/testing_data/prob_map_2_1.npy", prob_map)
+    np.save("./tests/testing_data/temp_prob_map_2_1.npy", prob_map)
 
     # prediction with two incorrect tumors
     prob_map = np.zeros((HEIGHT, WIDTH))
     prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
     prob_map[HEIGHT // 2, 6 * WIDTH // 7] = 0.4
-    np.save("./tests/testing_data/prob_map_0_2.npy", prob_map)
+    np.save("./tests/testing_data/temp_prob_map_0_2.npy", prob_map)
 
     # prediction with correct first tumors and two incorrect tumors
     prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
-    np.save("./tests/testing_data/prob_map_1_2.npy", prob_map)
+    np.save("./tests/testing_data/temp_prob_map_1_2.npy", prob_map)
 
 
 TEST_CASE_0 = [
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_0_0.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_0.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_0_0.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_0.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             }
@@ -107,8 +107,8 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_0_0.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_0_0.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             }
@@ -123,8 +123,8 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             }
@@ -139,8 +139,8 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_2_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_2_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             }
@@ -156,8 +156,8 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_2_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_2_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             }
@@ -172,8 +172,8 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             }
@@ -189,14 +189,14 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
             {
-                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
@@ -211,14 +211,14 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_3.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_3.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
             {
-                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
@@ -233,20 +233,20 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_0_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_1.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_0_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
             {
-                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_3.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_3.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
             {
-                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
@@ -261,20 +261,20 @@ def prepare_test_data():
     {
         "data": [
             {
-                "prob_map": "./tests/testing_data/prob_map_0_2.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_4.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_0_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_4.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
             {
-                "prob_map": "./tests/testing_data/prob_map_1_1.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_3.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_3.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },
             {
-                "prob_map": "./tests/testing_data/prob_map_1_2.npy",
-                "tumor_mask": "./tests/testing_data/ground_truth_2.tif",
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
                 "level": 0,
                 "pixel_spacing": 1,
             },

From 2611362c15a268d6c4f367a5d5a8d35b1fc1bfdf Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Wed, 31 Mar 2021 10:56:49 -0400
Subject: [PATCH 33/34] Update nms

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 monai/apps/pathology/metrics.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/monai/apps/pathology/metrics.py b/monai/apps/pathology/metrics.py
index 8cacf61112..63b9d073a7 100644
--- a/monai/apps/pathology/metrics.py
+++ b/monai/apps/pathology/metrics.py
@@ -70,9 +70,9 @@ def __init__(
         self.eval_thresholds = eval_thresholds
         self.image_reader = WSIReader(image_reader_name)
         self.nms = PathologyProbNMS(
-            sigma=0.0,
-            prob_threshold=0.5,
-            box_size=48,
+            sigma=nms_sigma,
+            prob_threshold=nms_prob_threshold,
+            box_size=nms_box_size,
         )
 
     def _load_data(self, file_path: str) -> List[Dict]:

From 7e7fe68ae07677f46a9c65c93eb7ef02cc420e7e Mon Sep 17 00:00:00 2001
From: Behrooz <3968947+behxyz@users.noreply.github.com>
Date: Wed, 31 Mar 2021 11:26:46 -0400
Subject: [PATCH 34/34] Update docs

Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com>
---
 docs/source/apps.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/source/apps.rst b/docs/source/apps.rst
index 08addfec2e..c3ce1e3726 100644
--- a/docs/source/apps.rst
+++ b/docs/source/apps.rst
@@ -75,7 +75,8 @@ Applications
     :members:
 
 .. automodule:: monai.apps.pathology.utils
-    :members:
+.. autofunction:: compute_multi_instance_mask
+.. autofunction:: compute_isolated_tumor_cells
 .. autoclass:: PathologyProbNMS
     :members: