From 80f9793dc283961712a3d27783bfd4e57b99fbdf Mon Sep 17 00:00:00 2001 From: "Wei_Chuan, Chiang" Date: Thu, 30 May 2024 18:03:52 +0800 Subject: [PATCH 1/9] Fixes #5905 Signed-off-by: Wei_Chuan, Chiang --- monai/transforms/utility/array.py | 6 ++++ monai/transforms/utility/dictionary.py | 10 +++++-- tests/test_data_stats.py | 38 ++++++++++++++++++++++++-- tests/test_data_statsd.py | 37 +++++++++++++++++++++++-- 4 files changed, 83 insertions(+), 8 deletions(-) diff --git a/monai/transforms/utility/array.py b/monai/transforms/utility/array.py index 5dfbcb0e91..245dd9db90 100644 --- a/monai/transforms/utility/array.py +++ b/monai/transforms/utility/array.py @@ -654,6 +654,7 @@ def __init__( data_shape: bool = True, value_range: bool = True, data_value: bool = False, + meta_tensor: bool = False, additional_info: Callable | None = None, name: str = "DataStats", ) -> None: @@ -665,6 +666,7 @@ def __init__( value_range: whether to show the value range of input data. data_value: whether to show the raw value of input data. a typical example is to print some properties of Nifti image: affine, pixdim, etc. + meta_tensor: whether to show the data of MetaTensor. additional_info: user can define callable function to extract additional info from input data. name: identifier of `logging.logger` to use, defaulting to "DataStats". @@ -679,6 +681,7 @@ def __init__( self.data_shape = data_shape self.value_range = value_range self.data_value = data_value + self.meta_tensor = meta_tensor if additional_info is not None and not callable(additional_info): raise TypeError(f"additional_info must be None or callable but is {type(additional_info).__name__}.") self.additional_info = additional_info @@ -705,6 +708,7 @@ def __call__( data_shape: bool | None = None, value_range: bool | None = None, data_value: bool | None = None, + meta_tensor: bool | None = None, additional_info: Callable | None = None, ) -> NdarrayOrTensor: """ @@ -725,6 +729,8 @@ def __call__( lines.append(f"Value range: (not a PyTorch or Numpy array, type: {type(img)})") if self.data_value if data_value is None else data_value: lines.append(f"Value: {img}") + if self.meta_tensor if meta_tensor is None else meta_tensor: + lines.append(f"MetaTensor: {MetaTensor(img).meta.__repr__()}") additional_info = self.additional_info if additional_info is None else additional_info if additional_info is not None: lines.append(f"Additional info: {additional_info(img)}") diff --git a/monai/transforms/utility/dictionary.py b/monai/transforms/utility/dictionary.py index 7e3a7b0454..aab2b6bc95 100644 --- a/monai/transforms/utility/dictionary.py +++ b/monai/transforms/utility/dictionary.py @@ -789,6 +789,7 @@ def __init__( data_shape: Sequence[bool] | bool = True, value_range: Sequence[bool] | bool = True, data_value: Sequence[bool] | bool = False, + meta_tensor: Sequence[bool] | bool = False, additional_info: Sequence[Callable] | Callable | None = None, name: str = "DataStats", allow_missing_keys: bool = False, @@ -808,6 +809,8 @@ def __init__( data_value: whether to show the raw value of input data. it also can be a sequence of bool, each element corresponds to a key in ``keys``. a typical example is to print some properties of Nifti image: affine, pixdim, etc. + meta_tensor: whether to show the data of MetaTensor. + it also can be a sequence of bool, each element corresponds to a key in ``keys``. additional_info: user can define callable function to extract additional info from input data. it also can be a sequence of string, each element corresponds to a key in ``keys``. @@ -821,15 +824,16 @@ def __init__( self.data_shape = ensure_tuple_rep(data_shape, len(self.keys)) self.value_range = ensure_tuple_rep(value_range, len(self.keys)) self.data_value = ensure_tuple_rep(data_value, len(self.keys)) + self.meta_tensor = ensure_tuple_rep(meta_tensor, len(self.keys)) self.additional_info = ensure_tuple_rep(additional_info, len(self.keys)) self.printer = DataStats(name=name) def __call__(self, data: Mapping[Hashable, NdarrayOrTensor]) -> dict[Hashable, NdarrayOrTensor]: d = dict(data) - for key, prefix, data_type, data_shape, value_range, data_value, additional_info in self.key_iterator( - d, self.prefix, self.data_type, self.data_shape, self.value_range, self.data_value, self.additional_info + for key, prefix, data_type, data_shape, value_range, data_value, meta_tensor, additional_info in self.key_iterator( + d, self.prefix, self.data_type, self.data_shape, self.value_range, self.data_value, self.meta_tensor, self.additional_info ): - d[key] = self.printer(d[key], prefix, data_type, data_shape, value_range, data_value, additional_info) + d[key] = self.printer(d[key], prefix, data_type, data_shape, value_range, data_value, meta_tensor, additional_info) return d diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py index 05453b0694..b15935e231 100644 --- a/tests/test_data_stats.py +++ b/tests/test_data_stats.py @@ -24,6 +24,7 @@ from parameterized import parameterized from monai.transforms import DataStats +from monai.data import MetaTensor TEST_CASE_1 = [ { @@ -130,20 +131,50 @@ ] TEST_CASE_8 = [ + { + "prefix": "test data", + "data_type": True, + "data_shape": True, + "value_range": True, + "data_value": True, + "additional_info": np.mean, + "name": "DataStats", + }, np.array([[0, 1], [1, 2]]), "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n", ] +TEST_CASE_9 = [ + np.array([[0, 1], [1, 2]]), + "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" + "Value: [[0 1]\n [1 2]]\n" + "MetaTensor: {affine: tensor([[1., 0., 0., 0.],\n" + " [0., 1., 0., 0.],\n" + " [0., 0., 1., 0.],\n" + " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" + "Additional info: 1.0\n", +] + +TEST_CASE_10 = [ + MetaTensor(torch.tensor([[0, 1], [1, 2]]), affine=torch.as_tensor([[2,0,0,0],[0,2,0,0],[0,0,2,0],[0,0,0,1]], dtype=torch.float64), meta={"some": "info"}), + "test data statistics:\nType: torch.int64\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n" + "Value: tensor([[0, 1],\n [1, 2]])\n" + "MetaTensor: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" + " [0., 2., 0., 0.],\n" + " [0., 0., 2., 0.],\n" + " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" + "Additional info: 1.0\n", +] class TestDataStats(unittest.TestCase): - @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7]) + @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]) def test_value(self, input_param, input_data, expected_print): transform = DataStats(**input_param) _ = transform(input_data) - @parameterized.expand([TEST_CASE_8]) + @parameterized.expand([TEST_CASE_9, TEST_CASE_10]) def test_file(self, input_data, expected_print): with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "test_data_stats.log") @@ -158,6 +189,7 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, + "meta_tensor": True, "additional_info": np.mean, "name": name, } @@ -168,7 +200,7 @@ def test_file(self, input_data, expected_print): logger.removeHandler(h) with open(filename) as f: content = f.read() - if sys.platform != "win32": + if sys.platform != "win32": self.assertEqual(content, expected_print) def test_multiple_data_stats(self): diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py index ef88300c10..69225def62 100644 --- a/tests/test_data_statsd.py +++ b/tests/test_data_statsd.py @@ -22,6 +22,7 @@ from parameterized import parameterized from monai.transforms import DataStatsd +from monai.data import MetaTensor TEST_CASE_1 = [ { @@ -150,22 +151,53 @@ ] TEST_CASE_9 = [ + { + "keys": "img", + "prefix": "test data", + "data_shape": True, + "value_range": True, + "data_value": True, + "meta_tensor": True, + "additional_info": np.mean, + "name": "DataStats", + }, {"img": np.array([[0, 1], [1, 2]])}, "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n", ] +TEST_CASE_10 = [ + {"img": np.array([[0, 1], [1, 2]])}, + "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" + "Value: [[0 1]\n [1 2]]\n" + "MetaTensor: {affine: tensor([[1., 0., 0., 0.],\n" + " [0., 1., 0., 0.],\n" + " [0., 0., 1., 0.],\n" + " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" + "Additional info: 1.0\n", +] + +TEST_CASE_11 = [ + {"img": (MetaTensor(torch.tensor([[0, 1], [1, 2]]), affine=torch.as_tensor([[2,0,0,0],[0,2,0,0],[0,0,2,0],[0,0,0,1]], dtype=torch.float64), meta={"some": "info"}))}, + "test data statistics:\nType: torch.int64\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n" + "Value: tensor([[0, 1],\n [1, 2]])\n" + "MetaTensor: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" + " [0., 2., 0., 0.],\n" + " [0., 0., 2., 0.],\n" + " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" + "Additional info: 1.0\n", +] class TestDataStatsd(unittest.TestCase): @parameterized.expand( - [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8, TEST_CASE_9] ) def test_value(self, input_param, input_data, expected_print): transform = DataStatsd(**input_param) _ = transform(input_data) - @parameterized.expand([TEST_CASE_9]) + @parameterized.expand([TEST_CASE_10, TEST_CASE_11]) def test_file(self, input_data, expected_print): with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "test_stats.log") @@ -180,6 +212,7 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, + "meta_tensor": True, "additional_info": np.mean, "name": name, } From a6c5b97177a51b7cdd85e3429379ca71b16075ec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 10:16:48 +0000 Subject: [PATCH 2/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_data_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py index b15935e231..8086b2eb92 100644 --- a/tests/test_data_stats.py +++ b/tests/test_data_stats.py @@ -200,7 +200,7 @@ def test_file(self, input_data, expected_print): logger.removeHandler(h) with open(filename) as f: content = f.read() - if sys.platform != "win32": + if sys.platform != "win32": self.assertEqual(content, expected_print) def test_multiple_data_stats(self): From 08f1e5767221c8181076c5df82457446c98a805e Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 30 May 2024 21:58:51 +0800 Subject: [PATCH 3/9] Ensure deterministic in MixUp, CutMix, CutOut (#7813) Fixes #7697 ### Description A few sentences describing the changes proposed in this pull request. ### Types of changes - [x] Non-breaking change (fix or new feature that would not break existing functionality). - [ ] Breaking change (fix or new feature that would cause existing functionality to change). - [ ] New tests added to cover the changes. - [ ] Integration tests passed locally by running `./runtests.sh -f -u --net --coverage`. - [ ] Quick tests passed locally by running `./runtests.sh --quick --unittests --disttests`. - [ ] In-line docstrings updated. - [ ] Documentation updated, tested `make html` command in the `docs/` folder. --------- Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> Co-authored-by: Eric Kerfoot <17726042+ericspod@users.noreply.github.com> --- monai/transforms/regularization/array.py | 52 ++++++++---- monai/transforms/regularization/dictionary.py | 80 ++++++++++++------ tests/test_regularization.py | 83 ++++++++++++------- 3 files changed, 146 insertions(+), 69 deletions(-) diff --git a/monai/transforms/regularization/array.py b/monai/transforms/regularization/array.py index 0b495c8623..a7436bda84 100644 --- a/monai/transforms/regularization/array.py +++ b/monai/transforms/regularization/array.py @@ -16,6 +16,9 @@ import torch +from monai.data.meta_obj import get_track_meta +from monai.utils.type_conversion import convert_to_dst_type, convert_to_tensor + from ..transform import RandomizableTransform __all__ = ["MixUp", "CutMix", "CutOut", "Mixer"] @@ -53,9 +56,11 @@ def randomize(self, data=None) -> None: as needed. You need to call this method everytime you apply the transform to a new batch. """ + super().randomize(None) self._params = ( torch.from_numpy(self.R.beta(self.alpha, self.alpha, self.batch_size)).type(torch.float32), self.R.permutation(self.batch_size), + [torch.from_numpy(self.R.randint(0, d, size=(1,))) for d in data.shape[2:]] if data is not None else [], ) @@ -69,7 +74,7 @@ class MixUp(Mixer): """ def apply(self, data: torch.Tensor): - weight, perm = self._params + weight, perm, _ = self._params nsamples, *dims = data.shape if len(weight) != nsamples: raise ValueError(f"Expected batch of size: {len(weight)}, but got {nsamples}") @@ -80,11 +85,18 @@ def apply(self, data: torch.Tensor): mixweight = weight[(Ellipsis,) + (None,) * len(dims)] return mixweight * data + (1 - mixweight) * data[perm, ...] - def __call__(self, data: torch.Tensor, labels: torch.Tensor | None = None): - self.randomize() + def __call__(self, data: torch.Tensor, labels: torch.Tensor | None = None, randomize=True): + data_t = convert_to_tensor(data, track_meta=get_track_meta()) + if labels is not None: + labels_t = convert_to_tensor(labels, track_meta=get_track_meta()) + if randomize: + self.randomize() if labels is None: - return self.apply(data) - return self.apply(data), self.apply(labels) + return convert_to_dst_type(self.apply(data_t), dst=data)[0] + return ( + convert_to_dst_type(self.apply(data_t), dst=data)[0], + convert_to_dst_type(self.apply(labels_t), dst=labels)[0], + ) class CutMix(Mixer): @@ -113,14 +125,13 @@ class CutMix(Mixer): """ def apply(self, data: torch.Tensor): - weights, perm = self._params + weights, perm, coords = self._params nsamples, _, *dims = data.shape if len(weights) != nsamples: raise ValueError(f"Expected batch of size: {len(weights)}, but got {nsamples}") mask = torch.ones_like(data) for s, weight in enumerate(weights): - coords = [torch.randint(0, d, size=(1,)) for d in dims] lengths = [d * sqrt(1 - weight) for d in dims] idx = [slice(None)] + [slice(c, min(ceil(c + ln), d)) for c, ln, d in zip(coords, lengths, dims)] mask[s][idx] = 0 @@ -128,7 +139,7 @@ def apply(self, data: torch.Tensor): return mask * data + (1 - mask) * data[perm, ...] def apply_on_labels(self, labels: torch.Tensor): - weights, perm = self._params + weights, perm, _ = self._params nsamples, *dims = labels.shape if len(weights) != nsamples: raise ValueError(f"Expected batch of size: {len(weights)}, but got {nsamples}") @@ -136,10 +147,16 @@ def apply_on_labels(self, labels: torch.Tensor): mixweight = weights[(Ellipsis,) + (None,) * len(dims)] return mixweight * labels + (1 - mixweight) * labels[perm, ...] - def __call__(self, data: torch.Tensor, labels: torch.Tensor | None = None): - self.randomize() - augmented = self.apply(data) - return (augmented, self.apply_on_labels(labels)) if labels is not None else augmented + def __call__(self, data: torch.Tensor, labels: torch.Tensor | None = None, randomize=True): + data_t = convert_to_tensor(data, track_meta=get_track_meta()) + if labels is not None: + labels_t = convert_to_tensor(labels, track_meta=get_track_meta()) + if randomize: + self.randomize(data) + augmented = convert_to_dst_type(self.apply(data_t), dst=data)[0] + if labels is not None: + augmented_label = convert_to_dst_type(self.apply(labels_t), dst=labels)[0] + return (augmented, augmented_label) if labels is not None else augmented class CutOut(Mixer): @@ -155,20 +172,21 @@ class CutOut(Mixer): """ def apply(self, data: torch.Tensor): - weights, _ = self._params + weights, _, coords = self._params nsamples, _, *dims = data.shape if len(weights) != nsamples: raise ValueError(f"Expected batch of size: {len(weights)}, but got {nsamples}") mask = torch.ones_like(data) for s, weight in enumerate(weights): - coords = [torch.randint(0, d, size=(1,)) for d in dims] lengths = [d * sqrt(1 - weight) for d in dims] idx = [slice(None)] + [slice(c, min(ceil(c + ln), d)) for c, ln, d in zip(coords, lengths, dims)] mask[s][idx] = 0 return mask * data - def __call__(self, data: torch.Tensor): - self.randomize() - return self.apply(data) + def __call__(self, data: torch.Tensor, randomize=True): + data_t = convert_to_tensor(data, track_meta=get_track_meta()) + if randomize: + self.randomize(data) + return convert_to_dst_type(self.apply(data_t), dst=data)[0] diff --git a/monai/transforms/regularization/dictionary.py b/monai/transforms/regularization/dictionary.py index 373913da99..d8815e47b9 100644 --- a/monai/transforms/regularization/dictionary.py +++ b/monai/transforms/regularization/dictionary.py @@ -11,16 +11,23 @@ from __future__ import annotations +from collections.abc import Hashable + +import numpy as np + from monai.config import KeysCollection +from monai.config.type_definitions import NdarrayOrTensor +from monai.data.meta_obj import get_track_meta +from monai.utils import convert_to_tensor from monai.utils.misc import ensure_tuple -from ..transform import MapTransform +from ..transform import MapTransform, RandomizableTransform from .array import CutMix, CutOut, MixUp __all__ = ["MixUpd", "MixUpD", "MixUpDict", "CutMixd", "CutMixD", "CutMixDict", "CutOutd", "CutOutD", "CutOutDict"] -class MixUpd(MapTransform): +class MixUpd(MapTransform, RandomizableTransform): """ Dictionary-based version :py:class:`monai.transforms.MixUp`. @@ -31,18 +38,24 @@ class MixUpd(MapTransform): def __init__( self, keys: KeysCollection, batch_size: int, alpha: float = 1.0, allow_missing_keys: bool = False ) -> None: - super().__init__(keys, allow_missing_keys) + MapTransform.__init__(self, keys, allow_missing_keys) self.mixup = MixUp(batch_size, alpha) + def set_random_state(self, seed: int | None = None, state: np.random.RandomState | None = None) -> MixUpd: + super().set_random_state(seed, state) + self.mixup.set_random_state(seed, state) + return self + def __call__(self, data): - self.mixup.randomize() - result = dict(data) - for k in self.keys: - result[k] = self.mixup.apply(data[k]) - return result + d = dict(data) + # all the keys share the same random state + self.mixup.randomize(None) + for k in self.key_iterator(d): + d[k] = self.mixup(data[k], randomize=False) + return d -class CutMixd(MapTransform): +class CutMixd(MapTransform, RandomizableTransform): """ Dictionary-based version :py:class:`monai.transforms.CutMix`. @@ -63,17 +76,27 @@ def __init__( self.mixer = CutMix(batch_size, alpha) self.label_keys = ensure_tuple(label_keys) if label_keys is not None else [] - def __call__(self, data): - self.mixer.randomize() - result = dict(data) - for k in self.keys: - result[k] = self.mixer.apply(data[k]) - for k in self.label_keys: - result[k] = self.mixer.apply_on_labels(data[k]) - return result - + def set_random_state(self, seed: int | None = None, state: np.random.RandomState | None = None) -> CutMixd: + super().set_random_state(seed, state) + self.mixer.set_random_state(seed, state) + return self -class CutOutd(MapTransform): + def __call__(self, data): + d = dict(data) + first_key: Hashable = self.first_key(d) + if first_key == (): + out: dict[Hashable, NdarrayOrTensor] = convert_to_tensor(d, track_meta=get_track_meta()) + return out + self.mixer.randomize(d[first_key]) + for key, label_key in self.key_iterator(d, self.label_keys): + ret = self.mixer(data[key], data.get(label_key, None), randomize=False) + d[key] = ret[0] + if label_key in d: + d[label_key] = ret[1] + return d + + +class CutOutd(MapTransform, RandomizableTransform): """ Dictionary-based version :py:class:`monai.transforms.CutOut`. @@ -84,12 +107,21 @@ def __init__(self, keys: KeysCollection, batch_size: int, allow_missing_keys: bo super().__init__(keys, allow_missing_keys) self.cutout = CutOut(batch_size) + def set_random_state(self, seed: int | None = None, state: np.random.RandomState | None = None) -> CutOutd: + super().set_random_state(seed, state) + self.cutout.set_random_state(seed, state) + return self + def __call__(self, data): - result = dict(data) - self.cutout.randomize() - for k in self.keys: - result[k] = self.cutout(data[k]) - return result + d = dict(data) + first_key: Hashable = self.first_key(d) + if first_key == (): + out: dict[Hashable, NdarrayOrTensor] = convert_to_tensor(d, track_meta=get_track_meta()) + return out + self.cutout.randomize(d[first_key]) + for k in self.key_iterator(d): + d[k] = self.cutout(data[k], randomize=False) + return d MixUpD = MixUpDict = MixUpd diff --git a/tests/test_regularization.py b/tests/test_regularization.py index 32df2f7b41..12d64637d5 100644 --- a/tests/test_regularization.py +++ b/tests/test_regularization.py @@ -13,29 +13,31 @@ import unittest +import numpy as np import torch -from monai.transforms import CutMix, CutMixd, CutOut, MixUp, MixUpd -from monai.utils import set_determinism +from monai.transforms import CutMix, CutMixd, CutOut, CutOutd, MixUp, MixUpd +from tests.utils import assert_allclose -@unittest.skip("Mixup is non-deterministic. Skip it temporarily") class TestMixup(unittest.TestCase): - def setUp(self) -> None: - set_determinism(seed=0) - - def tearDown(self) -> None: - set_determinism(None) - def test_mixup(self): for dims in [2, 3]: shape = (6, 3) + (32,) * dims sample = torch.rand(*shape, dtype=torch.float32) mixup = MixUp(6, 1.0) + mixup.set_random_state(seed=0) output = mixup(sample) + np.random.seed(0) + # simulate the randomize() of transform + np.random.random() + weight = torch.from_numpy(np.random.beta(1.0, 1.0, 6)).type(torch.float32) + perm = np.random.permutation(6) self.assertEqual(output.shape, sample.shape) - self.assertTrue(any(not torch.allclose(sample, mixup(sample)) for _ in range(10))) + mixweight = weight[(Ellipsis,) + (None,) * (dims + 1)] + expected = mixweight * sample + (1 - mixweight) * sample[perm, ...] + assert_allclose(output, expected, type_test=False, atol=1e-7) with self.assertRaises(ValueError): MixUp(6, -0.5) @@ -53,27 +55,32 @@ def test_mixupd(self): t = torch.rand(*shape, dtype=torch.float32) sample = {"a": t, "b": t} mixup = MixUpd(["a", "b"], 6) + mixup.set_random_state(seed=0) output = mixup(sample) - self.assertTrue(torch.allclose(output["a"], output["b"])) + np.random.seed(0) + # simulate the randomize() of transform + np.random.random() + weight = torch.from_numpy(np.random.beta(1.0, 1.0, 6)).type(torch.float32) + perm = np.random.permutation(6) + self.assertEqual(output["a"].shape, sample["a"].shape) + mixweight = weight[(Ellipsis,) + (None,) * (dims + 1)] + expected = mixweight * sample["a"] + (1 - mixweight) * sample["a"][perm, ...] + assert_allclose(output["a"], expected, type_test=False, atol=1e-7) + assert_allclose(output["a"], output["b"], type_test=False, atol=1e-7) + # self.assertTrue(torch.allclose(output["a"], output["b"])) with self.assertRaises(ValueError): MixUpd(["k1", "k2"], 6, -0.5) -@unittest.skip("CutMix is non-deterministic. Skip it temporarily") class TestCutMix(unittest.TestCase): - def setUp(self) -> None: - set_determinism(seed=0) - - def tearDown(self) -> None: - set_determinism(None) - def test_cutmix(self): for dims in [2, 3]: shape = (6, 3) + (32,) * dims sample = torch.rand(*shape, dtype=torch.float32) cutmix = CutMix(6, 1.0) + cutmix.set_random_state(seed=0) output = cutmix(sample) self.assertEqual(output.shape, sample.shape) self.assertTrue(any(not torch.allclose(sample, cutmix(sample)) for _ in range(10))) @@ -85,30 +92,50 @@ def test_cutmixd(self): label = torch.randint(0, 1, shape) sample = {"a": t, "b": t, "lbl1": label, "lbl2": label} cutmix = CutMixd(["a", "b"], 6, label_keys=("lbl1", "lbl2")) + cutmix.set_random_state(seed=123) output = cutmix(sample) - # croppings are different on each application - self.assertTrue(not torch.allclose(output["a"], output["b"])) # but mixing of labels is not affected by it self.assertTrue(torch.allclose(output["lbl1"], output["lbl2"])) -@unittest.skip("CutOut is non-deterministic. Skip it temporarily") class TestCutOut(unittest.TestCase): - def setUp(self) -> None: - set_determinism(seed=0) - - def tearDown(self) -> None: - set_determinism(None) - def test_cutout(self): for dims in [2, 3]: shape = (6, 3) + (32,) * dims sample = torch.rand(*shape, dtype=torch.float32) cutout = CutOut(6, 1.0) + cutout.set_random_state(seed=123) output = cutout(sample) + np.random.seed(123) + # simulate the randomize() of transform + np.random.random() + weight = torch.from_numpy(np.random.beta(1.0, 1.0, 6)).type(torch.float32) + perm = np.random.permutation(6) + coords = [torch.from_numpy(np.random.randint(0, d, size=(1,))) for d in sample.shape[2:]] + assert_allclose(weight, cutout._params[0]) + assert_allclose(perm, cutout._params[1]) + self.assertSequenceEqual(coords, cutout._params[2]) self.assertEqual(output.shape, sample.shape) - self.assertTrue(any(not torch.allclose(sample, cutout(sample)) for _ in range(10))) + + def test_cutoutd(self): + for dims in [2, 3]: + shape = (6, 3) + (32,) * dims + t = torch.rand(*shape, dtype=torch.float32) + sample = {"a": t, "b": t} + cutout = CutOutd(["a", "b"], 6, 1.0) + cutout.set_random_state(seed=123) + output = cutout(sample) + np.random.seed(123) + # simulate the randomize() of transform + np.random.random() + weight = torch.from_numpy(np.random.beta(1.0, 1.0, 6)).type(torch.float32) + perm = np.random.permutation(6) + coords = [torch.from_numpy(np.random.randint(0, d, size=(1,))) for d in t.shape[2:]] + assert_allclose(weight, cutout.cutout._params[0]) + assert_allclose(perm, cutout.cutout._params[1]) + self.assertSequenceEqual(coords, cutout.cutout._params[2]) + self.assertEqual(output["a"].shape, sample["a"].shape) if __name__ == "__main__": From 6b1c58ccdb55fa91c2856b495636c5c475ed5e01 Mon Sep 17 00:00:00 2001 From: Suraj Pai Date: Thu, 30 May 2024 23:26:46 -0400 Subject: [PATCH 4/9] Refactor Dataset to use Compose for transforms (#7784) Fixes #7646 ### Description A few sentences describing the changes proposed in this pull request. ### Types of changes - [x] Non-breaking change (fix or new feature that would not break existing functionality). - [ ] Breaking change (fix or new feature that would cause existing functionality to change). - [x] New tests added to cover the changes. - [ ] Integration tests passed locally by running `./runtests.sh -f -u --net --coverage`. - [x] Quick tests passed locally by running `./runtests.sh --quick --unittests --disttests`. - [x] In-line docstrings updated. - [ ] Documentation updated, tested `make html` command in the `docs/` folder. --------- Signed-off-by: Suraj Pai Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> Co-authored-by: Ben Murray --- monai/data/dataset.py | 52 +++++++++-------------------- tests/test_arraydataset.py | 2 +- tests/test_dataset.py | 68 +++++++++++++++++++++++++++++++++++++- tests/test_profiling.py | 4 ++- 4 files changed, 86 insertions(+), 40 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index 79e066303e..871b523289 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -36,15 +36,7 @@ from monai.data.meta_tensor import MetaTensor from monai.data.utils import SUPPORTED_PICKLE_MOD, convert_tables_to_dicts, pickle_hashing -from monai.transforms import ( - Compose, - Randomizable, - RandomizableTrait, - Transform, - apply_transform, - convert_to_contiguous, - reset_ops_id, -) +from monai.transforms import Compose, Randomizable, RandomizableTrait, Transform, convert_to_contiguous, reset_ops_id from monai.utils import MAX_SEED, convert_to_tensor, get_seed, look_up_option, min_version, optional_import from monai.utils.misc import first @@ -77,15 +69,19 @@ class Dataset(_TorchDataset): }, }, }] """ - def __init__(self, data: Sequence, transform: Callable | None = None) -> None: + def __init__(self, data: Sequence, transform: Sequence[Callable] | Callable | None = None) -> None: """ Args: data: input data to load and transform to generate dataset for model. - transform: a callable data transform on input data. - + transform: a callable, sequence of callables or None. If transform is not + a `Compose` instance, it will be wrapped in a `Compose` instance. Sequences + of callables are applied in order and if `None` is passed, the data is returned as is. """ self.data = data - self.transform: Any = transform + try: + self.transform = Compose(transform) if not isinstance(transform, Compose) else transform + except Exception as e: + raise ValueError("`transform` must be a callable or a list of callables that is Composable") from e def __len__(self) -> int: return len(self.data) @@ -95,7 +91,7 @@ def _transform(self, index: int): Fetch single data item from `self.data`. """ data_i = self.data[index] - return apply_transform(self.transform, data_i) if self.transform is not None else data_i + return self.transform(data_i) def __getitem__(self, index: int | slice | Sequence[int]): """ @@ -264,8 +260,6 @@ def __init__( using the cached content and with re-created transform instances. """ - if not isinstance(transform, Compose): - transform = Compose(transform) super().__init__(data=data, transform=transform) self.cache_dir = Path(cache_dir) if cache_dir is not None else None self.hash_func = hash_func @@ -323,9 +317,6 @@ def _pre_transform(self, item_transformed): random transform object """ - if not isinstance(self.transform, Compose): - raise ValueError("transform must be an instance of monai.transforms.Compose.") - first_random = self.transform.get_index_of_first( lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) ) @@ -346,9 +337,6 @@ def _post_transform(self, item_transformed): the transformed element through the random transforms """ - if not isinstance(self.transform, Compose): - raise ValueError("transform must be an instance of monai.transforms.Compose.") - first_random = self.transform.get_index_of_first( lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) ) @@ -501,9 +489,6 @@ def _pre_transform(self, item_transformed): Returns: the transformed element up to the N transform object """ - if not isinstance(self.transform, Compose): - raise ValueError("transform must be an instance of monai.transforms.Compose.") - item_transformed = self.transform(item_transformed, end=self.cache_n_trans, threading=True) reset_ops_id(item_transformed) @@ -519,9 +504,6 @@ def _post_transform(self, item_transformed): Returns: the final transformed result """ - if not isinstance(self.transform, Compose): - raise ValueError("transform must be an instance of monai.transforms.Compose.") - return self.transform(item_transformed, start=self.cache_n_trans) @@ -809,8 +791,6 @@ def __init__( Not following these recommendations may lead to runtime errors or duplicated cache across processes. """ - if not isinstance(transform, Compose): - transform = Compose(transform) super().__init__(data=data, transform=transform) self.set_num = cache_num # tracking the user-provided `cache_num` option self.set_rate = cache_rate # tracking the user-provided `cache_rate` option @@ -1282,8 +1262,10 @@ def to_list(x): data = [] for dataset in self.data: data.extend(to_list(dataset[index])) + if self.transform is not None: - data = apply_transform(self.transform, data, map_items=False) # transform the list data + self.transform.map_items = False # Compose object map_items to false so transform is applied to list + data = self.transform(data) # use tuple instead of list as the default collate_fn callback of MONAI DataLoader flattens nested lists return tuple(data) @@ -1432,15 +1414,11 @@ def __len__(self): def _transform(self, index: int): data = {k: v[index] for k, v in self.arrays.items()} - - if not self.transform: - return data - - result = apply_transform(self.transform, data) + result = self.transform(data) if self.transform is not None else data if isinstance(result, dict) or (isinstance(result, list) and isinstance(result[0], dict)): return result - raise AssertionError("With a dict supplied to apply_transform, should return a dict or a list of dicts.") + raise AssertionError("With a dict supplied to Compose, should return a dict or a list of dicts.") class CSVDataset(Dataset): diff --git a/tests/test_arraydataset.py b/tests/test_arraydataset.py index efc014a267..b61b3c139c 100644 --- a/tests/test_arraydataset.py +++ b/tests/test_arraydataset.py @@ -41,7 +41,7 @@ class TestCompose(Compose): - def __call__(self, input_, lazy): + def __call__(self, input_, lazy=False): img = self.transforms[0](input_) metadata = img.meta img = self.transforms[1](img) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 1398009c63..0d37ae2efd 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -23,7 +23,7 @@ from parameterized import parameterized from monai.data import Dataset -from monai.transforms import Compose, LoadImaged, SimulateDelayd +from monai.transforms import Compose, Lambda, LoadImage, LoadImaged, SimulateDelay, SimulateDelayd from tests.test_compose import TEST_COMPOSE_LAZY_ON_CALL_LOGGING_TEST_CASES, data_from_keys TEST_CASE_1 = [(128, 128, 128)] @@ -99,6 +99,72 @@ def test_dataset_lazy_on_call(self): data[0, 0:2, 0:2] = 1 +class TestTupleDataset(unittest.TestCase): + + @parameterized.expand([TEST_CASE_1]) + def test_shape(self, expected_shape): + test_image = nib.Nifti1Image(np.random.randint(0, 2, size=[128, 128, 128]).astype(float), np.eye(4)) + with tempfile.TemporaryDirectory() as tempdir: + nib.save(test_image, os.path.join(tempdir, "test_image1.nii.gz")) + nib.save(test_image, os.path.join(tempdir, "test_label1.nii.gz")) + nib.save(test_image, os.path.join(tempdir, "test_image2.nii.gz")) + nib.save(test_image, os.path.join(tempdir, "test_label2.nii.gz")) + test_data = [ + (os.path.join(tempdir, "test_image1.nii.gz"), os.path.join(tempdir, "test_label1.nii.gz")), + (os.path.join(tempdir, "test_image2.nii.gz"), os.path.join(tempdir, "test_label2.nii.gz")), + ] + + test_transform = Compose([LoadImage(), SimulateDelay(delay_time=1e-5)]) + + # Here test_transform is applied element by element for the tuple. + dataset = Dataset(data=test_data, transform=test_transform) + data1 = dataset[0] + data2 = dataset[1] + + # Output is a list/tuple + self.assertTrue(isinstance(data1, (list, tuple))) + self.assertTrue(isinstance(data2, (list, tuple))) + + # Number of elements are 2 + self.assertEqual(len(data1), 2) + self.assertEqual(len(data2), 2) + + # Output shapes are as expected + self.assertTupleEqual(data1[0].shape, expected_shape) + self.assertTupleEqual(data1[1].shape, expected_shape) + self.assertTupleEqual(data2[0].shape, expected_shape) + self.assertTupleEqual(data2[1].shape, expected_shape) + + # Here test_transform is applied to the tuple as a whole. + test_transform = Compose( + [ + # LoadImage creates a channel-stacked image when applied to a tuple + LoadImage(), + # Get the channel-stacked image and the label + Lambda(func=lambda x: (x[0].permute(2, 1, 0), x[1])), + ], + map_items=False, + ) + + dataset = Dataset(data=test_data, transform=test_transform) + data1 = dataset[0] + data2 = dataset[1] + + # Output is a list/tuple + self.assertTrue(isinstance(data1, (list, tuple))) + self.assertTrue(isinstance(data2, (list, tuple))) + + # Number of elements are 2 + self.assertEqual(len(data1), 2) + self.assertEqual(len(data2), 2) + + # Output shapes are as expected + self.assertTupleEqual(data1[0].shape, expected_shape) + self.assertTupleEqual(data1[1].shape, expected_shape) + self.assertTupleEqual(data2[0].shape, expected_shape) + self.assertTupleEqual(data2[1].shape, expected_shape) + + class TestDatsesetWithLazy(unittest.TestCase): LOGGER_NAME = "a_logger_name" diff --git a/tests/test_profiling.py b/tests/test_profiling.py index 6bee7ba262..649d980ebf 100644 --- a/tests/test_profiling.py +++ b/tests/test_profiling.py @@ -35,6 +35,7 @@ def setUp(self): self.scale = mt.ScaleIntensity() self.scale_call_name = "ScaleIntensity.__call__" + self.compose_call_name = "Compose.__call__" self.test_comp = mt.Compose([mt.ScaleIntensity(), mt.RandAxisFlip(0.5)]) self.test_image = torch.rand(1, 16, 16, 16) self.pid = os.getpid() @@ -82,7 +83,7 @@ def test_profile_multithread(self): self.assertSequenceEqual(batch.shape, (4, 1, 16, 16, 16)) results = wp.get_results() - self.assertSequenceEqual(list(results), [self.scale_call_name]) + self.assertSequenceEqual(list(results), [self.scale_call_name, self.compose_call_name]) prs = results[self.scale_call_name] @@ -98,6 +99,7 @@ def test_profile_context(self): self.scale(self.test_image) results = wp.get_results() + self.assertSequenceEqual(set(results), {"ScaleIntensity.__call__", "context"}) prs = results["context"] From 9eff6d38d217f0b13221a4681e85a6eb4b1075b8 Mon Sep 17 00:00:00 2001 From: "Wei_Chuan, Chiang" Date: Fri, 31 May 2024 17:08:44 +0800 Subject: [PATCH 5/9] Fix format issue and change argument name to 'meta_info' Signed-off-by: Wei_Chuan, Chiang --- monai/transforms/utility/array.py | 12 +++++----- monai/transforms/utility/dictionary.py | 30 ++++++++++++++++++----- tests/test_data_stats.py | 19 ++++++++++----- tests/test_data_statsd.py | 33 ++++++++++++++++++++------ 4 files changed, 69 insertions(+), 25 deletions(-) diff --git a/monai/transforms/utility/array.py b/monai/transforms/utility/array.py index 245dd9db90..a93ab68c05 100644 --- a/monai/transforms/utility/array.py +++ b/monai/transforms/utility/array.py @@ -654,7 +654,7 @@ def __init__( data_shape: bool = True, value_range: bool = True, data_value: bool = False, - meta_tensor: bool = False, + meta_info: bool = False, additional_info: Callable | None = None, name: str = "DataStats", ) -> None: @@ -666,7 +666,7 @@ def __init__( value_range: whether to show the value range of input data. data_value: whether to show the raw value of input data. a typical example is to print some properties of Nifti image: affine, pixdim, etc. - meta_tensor: whether to show the data of MetaTensor. + meta_info: whether to show the data of MetaTensor. additional_info: user can define callable function to extract additional info from input data. name: identifier of `logging.logger` to use, defaulting to "DataStats". @@ -681,7 +681,7 @@ def __init__( self.data_shape = data_shape self.value_range = value_range self.data_value = data_value - self.meta_tensor = meta_tensor + self.meta_info = meta_info if additional_info is not None and not callable(additional_info): raise TypeError(f"additional_info must be None or callable but is {type(additional_info).__name__}.") self.additional_info = additional_info @@ -708,7 +708,7 @@ def __call__( data_shape: bool | None = None, value_range: bool | None = None, data_value: bool | None = None, - meta_tensor: bool | None = None, + meta_info: bool | None = None, additional_info: Callable | None = None, ) -> NdarrayOrTensor: """ @@ -729,8 +729,8 @@ def __call__( lines.append(f"Value range: (not a PyTorch or Numpy array, type: {type(img)})") if self.data_value if data_value is None else data_value: lines.append(f"Value: {img}") - if self.meta_tensor if meta_tensor is None else meta_tensor: - lines.append(f"MetaTensor: {MetaTensor(img).meta.__repr__()}") + if self.meta_info if meta_info is None else meta_info: + lines.append(f"Meta_Info: {MetaTensor(img).meta.__repr__()}") additional_info = self.additional_info if additional_info is None else additional_info if additional_info is not None: lines.append(f"Additional info: {additional_info(img)}") diff --git a/monai/transforms/utility/dictionary.py b/monai/transforms/utility/dictionary.py index aab2b6bc95..7ba1d479f6 100644 --- a/monai/transforms/utility/dictionary.py +++ b/monai/transforms/utility/dictionary.py @@ -789,7 +789,7 @@ def __init__( data_shape: Sequence[bool] | bool = True, value_range: Sequence[bool] | bool = True, data_value: Sequence[bool] | bool = False, - meta_tensor: Sequence[bool] | bool = False, + meta_info: Sequence[bool] | bool = False, additional_info: Sequence[Callable] | Callable | None = None, name: str = "DataStats", allow_missing_keys: bool = False, @@ -809,7 +809,7 @@ def __init__( data_value: whether to show the raw value of input data. it also can be a sequence of bool, each element corresponds to a key in ``keys``. a typical example is to print some properties of Nifti image: affine, pixdim, etc. - meta_tensor: whether to show the data of MetaTensor. + meta_info: whether to show the data of MetaTensor. it also can be a sequence of bool, each element corresponds to a key in ``keys``. additional_info: user can define callable function to extract additional info from input data. it also can be a sequence of string, each element @@ -824,16 +824,34 @@ def __init__( self.data_shape = ensure_tuple_rep(data_shape, len(self.keys)) self.value_range = ensure_tuple_rep(value_range, len(self.keys)) self.data_value = ensure_tuple_rep(data_value, len(self.keys)) - self.meta_tensor = ensure_tuple_rep(meta_tensor, len(self.keys)) + self.meta_info = ensure_tuple_rep(meta_info, len(self.keys)) self.additional_info = ensure_tuple_rep(additional_info, len(self.keys)) self.printer = DataStats(name=name) def __call__(self, data: Mapping[Hashable, NdarrayOrTensor]) -> dict[Hashable, NdarrayOrTensor]: d = dict(data) - for key, prefix, data_type, data_shape, value_range, data_value, meta_tensor, additional_info in self.key_iterator( - d, self.prefix, self.data_type, self.data_shape, self.value_range, self.data_value, self.meta_tensor, self.additional_info + for ( + key, + prefix, + data_type, + data_shape, + value_range, + data_value, + meta_info, + additional_info, + ) in self.key_iterator( + d, + self.prefix, + self.data_type, + self.data_shape, + self.value_range, + self.data_value, + self.meta_info, + self.additional_info, ): - d[key] = self.printer(d[key], prefix, data_type, data_shape, value_range, data_value, meta_tensor, additional_info) + d[key] = self.printer( + d[key], prefix, data_type, data_shape, value_range, data_value, meta_info, additional_info + ) return d diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py index 8086b2eb92..c0c040ee6e 100644 --- a/tests/test_data_stats.py +++ b/tests/test_data_stats.py @@ -23,8 +23,8 @@ import torch from parameterized import parameterized +from monai.data.meta_tensor import MetaTensor from monai.transforms import DataStats -from monai.data import MetaTensor TEST_CASE_1 = [ { @@ -149,7 +149,7 @@ np.array([[0, 1], [1, 2]]), "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\n" - "MetaTensor: {affine: tensor([[1., 0., 0., 0.],\n" + "Meta_Info: {affine: tensor([[1., 0., 0., 0.],\n" " [0., 1., 0., 0.],\n" " [0., 0., 1., 0.],\n" " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" @@ -157,19 +157,26 @@ ] TEST_CASE_10 = [ - MetaTensor(torch.tensor([[0, 1], [1, 2]]), affine=torch.as_tensor([[2,0,0,0],[0,2,0,0],[0,0,2,0],[0,0,0,1]], dtype=torch.float64), meta={"some": "info"}), + MetaTensor( + torch.tensor([[0, 1], [1, 2]]), + affine=torch.as_tensor([[2, 0, 0, 0], [0, 2, 0, 0], [0, 0, 2, 0], [0, 0, 0, 1]], dtype=torch.float64), + meta={"some": "info"}, + ), "test data statistics:\nType: torch.int64\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n" "Value: tensor([[0, 1],\n [1, 2]])\n" - "MetaTensor: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" + "Meta_Info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" " [0., 2., 0., 0.],\n" " [0., 0., 2., 0.],\n" " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" "Additional info: 1.0\n", ] + class TestDataStats(unittest.TestCase): - @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]) + @parameterized.expand( + [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + ) def test_value(self, input_param, input_data, expected_print): transform = DataStats(**input_param) _ = transform(input_data) @@ -189,7 +196,7 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, - "meta_tensor": True, + "meta_info": True, "additional_info": np.mean, "name": name, } diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py index 69225def62..9c77a0aa4a 100644 --- a/tests/test_data_statsd.py +++ b/tests/test_data_statsd.py @@ -21,8 +21,8 @@ import torch from parameterized import parameterized +from monai.data.meta_tensor import MetaTensor from monai.transforms import DataStatsd -from monai.data import MetaTensor TEST_CASE_1 = [ { @@ -157,7 +157,7 @@ "data_shape": True, "value_range": True, "data_value": True, - "meta_tensor": True, + "meta_info": True, "additional_info": np.mean, "name": "DataStats", }, @@ -170,7 +170,7 @@ {"img": np.array([[0, 1], [1, 2]])}, "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\n" - "MetaTensor: {affine: tensor([[1., 0., 0., 0.],\n" + "Meta_Info: {affine: tensor([[1., 0., 0., 0.],\n" " [0., 1., 0., 0.],\n" " [0., 0., 1., 0.],\n" " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" @@ -178,20 +178,39 @@ ] TEST_CASE_11 = [ - {"img": (MetaTensor(torch.tensor([[0, 1], [1, 2]]), affine=torch.as_tensor([[2,0,0,0],[0,2,0,0],[0,0,2,0],[0,0,0,1]], dtype=torch.float64), meta={"some": "info"}))}, + { + "img": ( + MetaTensor( + torch.tensor([[0, 1], [1, 2]]), + affine=torch.as_tensor([[2, 0, 0, 0], [0, 2, 0, 0], [0, 0, 2, 0], [0, 0, 0, 1]], dtype=torch.float64), + meta={"some": "info"}, + ) + ) + }, "test data statistics:\nType: torch.int64\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n" "Value: tensor([[0, 1],\n [1, 2]])\n" - "MetaTensor: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" + "Meta_Info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" " [0., 2., 0., 0.],\n" " [0., 0., 2., 0.],\n" " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" "Additional info: 1.0\n", ] + class TestDataStatsd(unittest.TestCase): @parameterized.expand( - [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8, TEST_CASE_9] + [ + TEST_CASE_1, + TEST_CASE_2, + TEST_CASE_3, + TEST_CASE_4, + TEST_CASE_5, + TEST_CASE_6, + TEST_CASE_7, + TEST_CASE_8, + TEST_CASE_9, + ] ) def test_value(self, input_param, input_data, expected_print): transform = DataStatsd(**input_param) @@ -212,7 +231,7 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, - "meta_tensor": True, + "meta_info": True, "additional_info": np.mean, "name": name, } From 68b521b1a1a23ae25b5f38881a3da6595d1a67fb Mon Sep 17 00:00:00 2001 From: "Wei_Chuan, Chiang" Date: Mon, 3 Jun 2024 15:33:32 +0800 Subject: [PATCH 6/9] Fix format issue Signed-off-by: Wei_Chuan, Chiang --- tests/test_data_stats.py | 3 ++- tests/test_data_statsd.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py index c0c040ee6e..760afdfa6c 100644 --- a/tests/test_data_stats.py +++ b/tests/test_data_stats.py @@ -162,7 +162,8 @@ affine=torch.as_tensor([[2, 0, 0, 0], [0, 2, 0, 0], [0, 0, 2, 0], [0, 0, 0, 1]], dtype=torch.float64), meta={"some": "info"}, ), - "test data statistics:\nType: torch.int64\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n" + "test data statistics:\nType: torch.int64\n" + "Shape: torch.Size([2, 2])\nValue range: (0, 2)\n" "Value: tensor([[0, 1],\n [1, 2]])\n" "Meta_Info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" " [0., 2., 0., 0.],\n" diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py index 9c77a0aa4a..2f0d43a430 100644 --- a/tests/test_data_statsd.py +++ b/tests/test_data_statsd.py @@ -187,7 +187,8 @@ ) ) }, - "test data statistics:\nType: torch.int64\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n" + "test data statistics:\nType: torch.int64\n" + "Shape: torch.Size([2, 2])\nValue range: (0, 2)\n" "Value: tensor([[0, 1],\n [1, 2]])\n" "Meta_Info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" " [0., 2., 0., 0.],\n" From 3f0a7f3e540acb1659a8ec3f167fd789476858a1 Mon Sep 17 00:00:00 2001 From: "Wei_Chuan, Chiang" Date: Wed, 28 Aug 2024 15:44:13 +0800 Subject: [PATCH 7/9] fix-issue-5905 Signed-off-by: Wei_Chuan, Chiang --- monai/transforms/utility/array.py | 3 +- tests/test_data_stats.py | 45 ++---------------------- tests/test_data_statsd.py | 57 ++----------------------------- 3 files changed, 7 insertions(+), 98 deletions(-) diff --git a/monai/transforms/utility/array.py b/monai/transforms/utility/array.py index a93ab68c05..bff8ef1244 100644 --- a/monai/transforms/utility/array.py +++ b/monai/transforms/utility/array.py @@ -730,7 +730,8 @@ def __call__( if self.data_value if data_value is None else data_value: lines.append(f"Value: {img}") if self.meta_info if meta_info is None else meta_info: - lines.append(f"Meta_Info: {MetaTensor(img).meta.__repr__()}") + metadata = getattr(img, "meta", "(input is not a MetaTensor)") + lines.append(f"Meta info: {repr(metadata)}") additional_info = self.additional_info if additional_info is None else additional_info if additional_info is not None: lines.append(f"Additional info: {additional_info(img)}") diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py index 760afdfa6c..a3408070ae 100644 --- a/tests/test_data_stats.py +++ b/tests/test_data_stats.py @@ -23,7 +23,6 @@ import torch from parameterized import parameterized -from monai.data.meta_tensor import MetaTensor from monai.transforms import DataStats TEST_CASE_1 = [ @@ -131,58 +130,20 @@ ] TEST_CASE_8 = [ - { - "prefix": "test data", - "data_type": True, - "data_shape": True, - "value_range": True, - "data_value": True, - "additional_info": np.mean, - "name": "DataStats", - }, np.array([[0, 1], [1, 2]]), "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n", ] -TEST_CASE_9 = [ - np.array([[0, 1], [1, 2]]), - "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" - "Value: [[0 1]\n [1 2]]\n" - "Meta_Info: {affine: tensor([[1., 0., 0., 0.],\n" - " [0., 1., 0., 0.],\n" - " [0., 0., 1., 0.],\n" - " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" - "Additional info: 1.0\n", -] - -TEST_CASE_10 = [ - MetaTensor( - torch.tensor([[0, 1], [1, 2]]), - affine=torch.as_tensor([[2, 0, 0, 0], [0, 2, 0, 0], [0, 0, 2, 0], [0, 0, 0, 1]], dtype=torch.float64), - meta={"some": "info"}, - ), - "test data statistics:\nType: torch.int64\n" - "Shape: torch.Size([2, 2])\nValue range: (0, 2)\n" - "Value: tensor([[0, 1],\n [1, 2]])\n" - "Meta_Info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" - " [0., 2., 0., 0.],\n" - " [0., 0., 2., 0.],\n" - " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" - "Additional info: 1.0\n", -] - class TestDataStats(unittest.TestCase): - @parameterized.expand( - [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] - ) + @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7]) def test_value(self, input_param, input_data, expected_print): transform = DataStats(**input_param) _ = transform(input_data) - @parameterized.expand([TEST_CASE_9, TEST_CASE_10]) + @parameterized.expand([TEST_CASE_8]) def test_file(self, input_data, expected_print): with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "test_data_stats.log") @@ -197,7 +158,6 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, - "meta_info": True, "additional_info": np.mean, "name": name, } @@ -210,6 +170,7 @@ def test_file(self, input_data, expected_print): content = f.read() if sys.platform != "win32": self.assertEqual(content, expected_print) + # self.assertEqual(content, expected_print) def test_multiple_data_stats(self): with patch("sys.stdout", new=StringIO()) as out: diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py index 2f0d43a430..ef88300c10 100644 --- a/tests/test_data_statsd.py +++ b/tests/test_data_statsd.py @@ -21,7 +21,6 @@ import torch from parameterized import parameterized -from monai.data.meta_tensor import MetaTensor from monai.transforms import DataStatsd TEST_CASE_1 = [ @@ -151,73 +150,22 @@ ] TEST_CASE_9 = [ - { - "keys": "img", - "prefix": "test data", - "data_shape": True, - "value_range": True, - "data_value": True, - "meta_info": True, - "additional_info": np.mean, - "name": "DataStats", - }, {"img": np.array([[0, 1], [1, 2]])}, "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n", ] -TEST_CASE_10 = [ - {"img": np.array([[0, 1], [1, 2]])}, - "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" - "Value: [[0 1]\n [1 2]]\n" - "Meta_Info: {affine: tensor([[1., 0., 0., 0.],\n" - " [0., 1., 0., 0.],\n" - " [0., 0., 1., 0.],\n" - " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" - "Additional info: 1.0\n", -] - -TEST_CASE_11 = [ - { - "img": ( - MetaTensor( - torch.tensor([[0, 1], [1, 2]]), - affine=torch.as_tensor([[2, 0, 0, 0], [0, 2, 0, 0], [0, 0, 2, 0], [0, 0, 0, 1]], dtype=torch.float64), - meta={"some": "info"}, - ) - ) - }, - "test data statistics:\nType: torch.int64\n" - "Shape: torch.Size([2, 2])\nValue range: (0, 2)\n" - "Value: tensor([[0, 1],\n [1, 2]])\n" - "Meta_Info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" - " [0., 2., 0., 0.],\n" - " [0., 0., 2., 0.],\n" - " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" - "Additional info: 1.0\n", -] - class TestDataStatsd(unittest.TestCase): @parameterized.expand( - [ - TEST_CASE_1, - TEST_CASE_2, - TEST_CASE_3, - TEST_CASE_4, - TEST_CASE_5, - TEST_CASE_6, - TEST_CASE_7, - TEST_CASE_8, - TEST_CASE_9, - ] + [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] ) def test_value(self, input_param, input_data, expected_print): transform = DataStatsd(**input_param) _ = transform(input_data) - @parameterized.expand([TEST_CASE_10, TEST_CASE_11]) + @parameterized.expand([TEST_CASE_9]) def test_file(self, input_data, expected_print): with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "test_stats.log") @@ -232,7 +180,6 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, - "meta_info": True, "additional_info": np.mean, "name": name, } From ccff9f24d1ec0d242dcdeebcb820ac2343ca27c4 Mon Sep 17 00:00:00 2001 From: "Wei_Chuan, Chiang" Date: Wed, 28 Aug 2024 16:12:43 +0800 Subject: [PATCH 8/9] fix issue 5905 Signed-off-by: Wei_Chuan, Chiang --- tests/test_data_stats.py | 42 +++++++++++++++++++++++++++--- tests/test_data_statsd.py | 54 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py index a3408070ae..e1d78d2861 100644 --- a/tests/test_data_stats.py +++ b/tests/test_data_stats.py @@ -23,6 +23,7 @@ import torch from parameterized import parameterized +from monai.data.meta_tensor import MetaTensor from monai.transforms import DataStats TEST_CASE_1 = [ @@ -130,20 +131,55 @@ ] TEST_CASE_8 = [ + { + "prefix": "test data", + "data_type": True, + "data_shape": True, + "value_range": True, + "data_value": True, + "additional_info": np.mean, + "name": "DataStats", + }, np.array([[0, 1], [1, 2]]), "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n", ] +TEST_CASE_9 = [ + np.array([[0, 1], [1, 2]]), + "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" + "Value: [[0 1]\n [1 2]]\n" + "Meta info: None\n" + "Additional info: 1.0\n", +] + +TEST_CASE_10 = [ + MetaTensor( + torch.tensor([[0, 1], [1, 2]]), + affine=torch.as_tensor([[2, 0, 0, 0], [0, 2, 0, 0], [0, 0, 2, 0], [0, 0, 0, 1]], dtype=torch.float64), + meta={"some": "info"}, + ), + "test data statistics:\nType: torch.int64\n" + "Shape: torch.Size([2, 2])\nValue range: (0, 2)\n" + "Value: tensor([[0, 1],\n [1, 2]])\n" + "Meta info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" + " [0., 2., 0., 0.],\n" + " [0., 0., 2., 0.],\n" + " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" + "Additional info: 1.0\n", +] + class TestDataStats(unittest.TestCase): - @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7]) + @parameterized.expand( + [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + ) def test_value(self, input_param, input_data, expected_print): transform = DataStats(**input_param) _ = transform(input_data) - @parameterized.expand([TEST_CASE_8]) + @parameterized.expand([TEST_CASE_9, TEST_CASE_10]) def test_file(self, input_data, expected_print): with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "test_data_stats.log") @@ -158,6 +194,7 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, + "meta_info": True, "additional_info": np.mean, "name": name, } @@ -170,7 +207,6 @@ def test_file(self, input_data, expected_print): content = f.read() if sys.platform != "win32": self.assertEqual(content, expected_print) - # self.assertEqual(content, expected_print) def test_multiple_data_stats(self): with patch("sys.stdout", new=StringIO()) as out: diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py index ef88300c10..3f66a97e6a 100644 --- a/tests/test_data_statsd.py +++ b/tests/test_data_statsd.py @@ -21,6 +21,7 @@ import torch from parameterized import parameterized +from monai.data.meta_tensor import MetaTensor from monai.transforms import DataStatsd TEST_CASE_1 = [ @@ -150,22 +151,70 @@ ] TEST_CASE_9 = [ + { + "keys": "img", + "prefix": "test data", + "data_shape": True, + "value_range": True, + "data_value": True, + "meta_info": False, + "additional_info": np.mean, + "name": "DataStats", + }, {"img": np.array([[0, 1], [1, 2]])}, "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n", ] +TEST_CASE_10 = [ + {"img": np.array([[0, 1], [1, 2]])}, + "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" + "Value: [[0 1]\n [1 2]]\n" + "Meta info: None\n" + "Additional info: 1.0\n", +] + +TEST_CASE_11 = [ + { + "img": ( + MetaTensor( + torch.tensor([[0, 1], [1, 2]]), + affine=torch.as_tensor([[2, 0, 0, 0], [0, 2, 0, 0], [0, 0, 2, 0], [0, 0, 0, 1]], dtype=torch.float64), + meta={"some": "info"}, + ) + ) + }, + "test data statistics:\nType: torch.int64\n" + "Shape: torch.Size([2, 2])\nValue range: (0, 2)\n" + "Value: tensor([[0, 1],\n [1, 2]])\n" + "Meta info: {'some': 'info', affine: tensor([[2., 0., 0., 0.],\n" + " [0., 2., 0., 0.],\n" + " [0., 0., 2., 0.],\n" + " [0., 0., 0., 1.]], dtype=torch.float64), space: RAS}\n" + "Additional info: 1.0\n", +] + class TestDataStatsd(unittest.TestCase): @parameterized.expand( - [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + [ + TEST_CASE_1, + TEST_CASE_2, + TEST_CASE_3, + TEST_CASE_4, + TEST_CASE_5, + TEST_CASE_6, + TEST_CASE_7, + TEST_CASE_8, + TEST_CASE_9, + ] ) def test_value(self, input_param, input_data, expected_print): transform = DataStatsd(**input_param) _ = transform(input_data) - @parameterized.expand([TEST_CASE_9]) + @parameterized.expand([TEST_CASE_10, TEST_CASE_11]) def test_file(self, input_data, expected_print): with tempfile.TemporaryDirectory() as tempdir: filename = os.path.join(tempdir, "test_stats.log") @@ -180,6 +229,7 @@ def test_file(self, input_data, expected_print): "data_shape": True, "value_range": True, "data_value": True, + "meta_info": True, "additional_info": np.mean, "name": name, } From 399637e410da9e53df4069579ffb2781bede304e Mon Sep 17 00:00:00 2001 From: "Wei_Chuan, Chiang" Date: Wed, 28 Aug 2024 16:45:41 +0800 Subject: [PATCH 9/9] fix issue 5905 format issue Signed-off-by: Wei_Chuan, Chiang --- tests/test_data_stats.py | 2 +- tests/test_data_statsd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py index e1d78d2861..f9b424f8e1 100644 --- a/tests/test_data_stats.py +++ b/tests/test_data_stats.py @@ -149,7 +149,7 @@ np.array([[0, 1], [1, 2]]), "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\n" - "Meta info: None\n" + "Meta info: '(input is not a MetaTensor)'\n" "Additional info: 1.0\n", ] diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py index 3f66a97e6a..a28a938c40 100644 --- a/tests/test_data_statsd.py +++ b/tests/test_data_statsd.py @@ -170,7 +170,7 @@ {"img": np.array([[0, 1], [1, 2]])}, "test data statistics:\nType: int64\nShape: (2, 2)\nValue range: (0, 2)\n" "Value: [[0 1]\n [1 2]]\n" - "Meta info: None\n" + "Meta info: '(input is not a MetaTensor)'\n" "Additional info: 1.0\n", ]