From 38578dd7172d533403a755795412bae9c8364955 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 15 Jun 2024 21:11:56 +0200 Subject: [PATCH 01/20] test z[selection] for orthogonal selection --- tests/v3/test_indexing.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 00ea947b49..cd8e38502b 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -15,6 +15,7 @@ from zarr.buffer import BufferPrototype, NDBuffer from zarr.common import ChunkCoords from zarr.indexing import ( + is_pure_fancy_indexing, make_slice_selection, normalize_integer_selection, oindex, @@ -520,6 +521,12 @@ def _test_get_orthogonal_selection(a, z, selection): actual = z.oindex[selection] assert_array_equal(expect, actual) + # if is not fancy indexing, z[selection] should be orthogonal selection + selection_ndim = len(selection) if isinstance(selection, tuple) else 1 + if not is_pure_fancy_indexing(selection, selection_ndim): + actual = z[selection] + assert_array_equal(expect, actual) + # noinspection PyStatementEffect def test_get_orthogonal_selection_1d_bool(store: StorePath): @@ -1718,3 +1725,6 @@ def test_accessed_chunks(shape, chunks, ops): ) == 1 # Check that no other chunks were accessed assert len(delta_counts) == 0 + +def test_is_pure_orthogonal_indexing(): + pass From 7b6470f64a0c5d3944b7e92ce3d8e9df9529a25d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 15 Jun 2024 21:14:58 +0200 Subject: [PATCH 02/20] include boolean indexing in is_pure_orthogonal_indexing --- src/zarr/indexing.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 98130fe0cd..4a9768eb3f 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -107,6 +107,10 @@ def is_integer(x: Any) -> TypeGuard[int]: """ return isinstance(x, numbers.Integral) +def is_bool(x: Any) -> TypeGuard[int]: + """True if x is a boolean (both pure Python or NumPy).""" + return type(x) in [bool, np.bool_] + def is_integer_list(x: Any) -> TypeGuard[list[int]]: """True if x is a list of integers. @@ -117,6 +121,15 @@ def is_integer_list(x: Any) -> TypeGuard[list[int]]: """ return isinstance(x, list) and len(x) > 0 and is_integer(x[0]) +def is_bool_list(x: Any) -> TypeGuard[list[int]]: + """True if x is a list of boolean. + + This function assumes ie *does not check* that all elements of the list + have the same type. Mixed type lists will result in other errors that will + bubble up anyway. + """ + return isinstance(x, list) and len(x) > 0 and is_bool(x[0]) + def is_integer_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.intp]]: t = not np.isscalar(x) and hasattr(x, "shape") and hasattr(x, "dtype") and x.dtype.kind in "ui" @@ -131,6 +144,8 @@ def is_bool_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.b t = t and hasattr(x, "shape") and len(x.shape) == ndim return t +def is_int_or_bool_iterable(x: Any) -> bool: + return is_integer_list(x) or is_integer_array(x) or is_bool_array(x) or is_bool_list(x) def is_scalar(value: Any, dtype: np.dtype[Any]) -> bool: if np.isscalar(value): @@ -180,20 +195,20 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or if not ndim: return False - # Case 1: Selection is a single iterable of integers - if is_integer_list(selection) or is_integer_array(selection, ndim=1): + if not isinstance(selection, tuple): + selection = (selection,) + + # Case 1: Selection contains of iterable of integers or boolean + if all(is_int_or_bool_iterable(s) for s in selection): return True - # Case two: selection contains either zero or one integer iterables. + # Case 2: selection contains either zero or one integer iterables. # All other selection elements are slices or integers return ( isinstance(selection, tuple) and len(selection) == ndim - and sum(is_integer_list(elem) or is_integer_array(elem) for elem in selection) <= 1 - and all( - is_integer_list(elem) or is_integer_array(elem) or isinstance(elem, int | slice) - for elem in selection - ) + and sum(is_int_or_bool_iterable(s) for s in selection) <= 1 + and all(is_int_or_bool_iterable(s) or isinstance(s, int | slice) for s in selection) ) From badf8185c8a29800253fcd6de342882908b05c17 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 15 Jun 2024 21:16:11 +0200 Subject: [PATCH 03/20] Revert "test z[selection] for orthogonal selection" This reverts commit 38578dd7172d533403a755795412bae9c8364955. --- tests/v3/test_indexing.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index cd8e38502b..00ea947b49 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -15,7 +15,6 @@ from zarr.buffer import BufferPrototype, NDBuffer from zarr.common import ChunkCoords from zarr.indexing import ( - is_pure_fancy_indexing, make_slice_selection, normalize_integer_selection, oindex, @@ -521,12 +520,6 @@ def _test_get_orthogonal_selection(a, z, selection): actual = z.oindex[selection] assert_array_equal(expect, actual) - # if is not fancy indexing, z[selection] should be orthogonal selection - selection_ndim = len(selection) if isinstance(selection, tuple) else 1 - if not is_pure_fancy_indexing(selection, selection_ndim): - actual = z[selection] - assert_array_equal(expect, actual) - # noinspection PyStatementEffect def test_get_orthogonal_selection_1d_bool(store: StorePath): @@ -1725,6 +1718,3 @@ def test_accessed_chunks(shape, chunks, ops): ) == 1 # Check that no other chunks were accessed assert len(delta_counts) == 0 - -def test_is_pure_orthogonal_indexing(): - pass From dd764e2182c7b09a7b6dd2f4ddf3f1907693b2a6 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 15 Jun 2024 22:59:54 +0200 Subject: [PATCH 04/20] add test_indexing_equals_numpy --- tests/v3/test_indexing.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 00ea947b49..13b2ab15c4 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1718,3 +1718,33 @@ def test_accessed_chunks(shape, chunks, ops): ) == 1 # Check that no other chunks were accessed assert len(delta_counts) == 0 + +@pytest.mark.parametrize( + "selection", + [ + # basic selection + ..., + 1, ..., + slice(None), + (1,3), + ([1, 2, 3],9), + np.arange(1000), + slice(5, 15), + (slice(2, 4), 4), + [1,3], + # mask selection + np.tile([True, False], (1000,5)), + np.full((1000, 10), False), + # coordinate selection + ([1, 2, 3, 4], [5, 6, 7, 8]), + ([100, 200, 300], [4, 5, 6]), + # orthogonal selection + (np.tile([True, False], 500), np.tile([True, False], 5)), + ]) +def test_indexing_equals_numpy(store, selection): + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + expected = a[selection] + actual = z[selection] + assert_array_equal(expected, actual, err_msg=f"selection: {selection}") + z.oindex \ No newline at end of file From 26b920cdbee44d183c1566eaf77eea261d1768a8 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 15 Jun 2024 23:01:06 +0200 Subject: [PATCH 05/20] extend _test_get_mask_selection for square bracket notation --- tests/v3/test_indexing.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 13b2ab15c4..9858809482 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1264,6 +1264,8 @@ def _test_get_mask_selection(a, z, selection): assert_array_equal(expect, actual) actual = z.vindex[selection] assert_array_equal(expect, actual) + actual = z[selection] + assert_array_equal(expect, actual) mask_selections_1d_bad = [ @@ -1336,6 +1338,10 @@ def _test_set_mask_selection(v, a, z, selection): z[:] = 0 z.vindex[selection] = v[selection] assert_array_equal(a, z[:]) + z[:] = 0 + z[selection] = v[selection] + assert_array_equal(a, z[:]) + def test_set_mask_selection_1d(store: StorePath): From 782a712619d77022fbaf2ea31e35031380503207 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 15 Jun 2024 23:17:36 +0200 Subject: [PATCH 06/20] fix is_pure_fancy_indexing for mask selection --- src/zarr/indexing.py | 9 +++++++-- tests/v3/test_indexing.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 4a9768eb3f..3c99bda613 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -158,7 +158,7 @@ def is_scalar(value: Any, dtype: np.dtype[Any]) -> bool: def is_pure_fancy_indexing(selection: Any, ndim: int) -> bool: - """Check whether a selection contains only scalars or integer array-likes. + """Check whether a selection contains only scalars or integer/bool array-likes. Parameters ---------- @@ -171,9 +171,14 @@ def is_pure_fancy_indexing(selection: Any, ndim: int) -> bool: True if the selection is a pure fancy indexing expression (ie not mixed with boolean or slices). """ + if is_bool_array(selection): + # is mask selection + return True + if ndim == 1: - if is_integer_list(selection) or is_integer_array(selection): + if is_integer_list(selection) or is_integer_array(selection) or is_bool_list(selection): return True + # if not, we go through the normal path below, because a 1-tuple # of integers is also allowed. no_slicing = ( diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 9858809482..d08756068c 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1730,7 +1730,7 @@ def test_accessed_chunks(shape, chunks, ops): [ # basic selection ..., - 1, ..., + (1, ...), slice(None), (1,3), ([1, 2, 3],9), From a94b9952e5347f064e89a58989ba48c6aed241d4 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 16 Jun 2024 00:08:13 +0200 Subject: [PATCH 07/20] add test_orthogonal_bool_indexing_like_numpy_ix --- tests/v3/test_indexing.py | 49 +++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index d08756068c..0dee62e8fc 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1729,28 +1729,41 @@ def test_accessed_chunks(shape, chunks, ops): "selection", [ # basic selection - ..., - (1, ...), - slice(None), - (1,3), - ([1, 2, 3],9), - np.arange(1000), - slice(5, 15), - (slice(2, 4), 4), + [...], + [1, ...], + [slice(None)], [1,3], + [(1,3)], + [[1, 2, 3],9], + [np.arange(1000)], + [slice(5, 15)], + [slice(2, 4), 4], + [[1,3]], # mask selection - np.tile([True, False], (1000,5)), - np.full((1000, 10), False), + [np.tile([True, False], (1000,5))], + [np.full((1000, 10), False)], # coordinate selection - ([1, 2, 3, 4], [5, 6, 7, 8]), - ([100, 200, 300], [4, 5, 6]), - # orthogonal selection - (np.tile([True, False], 500), np.tile([True, False], 5)), - ]) + [[1, 2, 3, 4], [5, 6, 7, 8]], + [[100, 200, 300], [4, 5, 6]], +]) def test_indexing_equals_numpy(store, selection): a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) - expected = a[selection] - actual = z[selection] + expected = a[*selection] + actual = z[*selection] assert_array_equal(expected, actual, err_msg=f"selection: {selection}") - z.oindex \ No newline at end of file + +@pytest.mark.parametrize( + "selection", + [ + [np.tile([True, False], 500), np.tile([True, False], 5)], + [np.full(1000, False), np.tile([True, False], 5)], + [np.full(1000, True), np.full(10, True)], + [np.full(1000, True), [True, False]*5], + ]) +def test_orthogonal_bool_indexing_like_numpy_ix(store, selection): + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + expected = a[np.ix_(*selection)] + actual = z[*selection] + assert_array_equal(expected, actual, err_msg=f"{selection=}") From 97a06f0df3316ea52129a24b986863f09ccdba06 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 16 Jun 2024 00:49:14 +0200 Subject: [PATCH 08/20] fix for mypy --- src/zarr/indexing.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 3c99bda613..e0b20fc780 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -201,7 +201,7 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or return False if not isinstance(selection, tuple): - selection = (selection,) + return False # Case 1: Selection contains of iterable of integers or boolean if all(is_int_or_bool_iterable(s) for s in selection): @@ -210,8 +210,7 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or # Case 2: selection contains either zero or one integer iterables. # All other selection elements are slices or integers return ( - isinstance(selection, tuple) - and len(selection) == ndim + len(selection) == ndim and sum(is_int_or_bool_iterable(s) for s in selection) <= 1 and all(is_int_or_bool_iterable(s) or isinstance(s, int | slice) for s in selection) ) From 85ca73f707e633b938ca26b6769deba1a75caba5 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 17 Jun 2024 11:01:10 +0200 Subject: [PATCH 09/20] ruff format --- src/zarr/indexing.py | 4 ++++ tests/v3/test_indexing.py | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index e0b20fc780..5e2f967b75 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -107,6 +107,7 @@ def is_integer(x: Any) -> TypeGuard[int]: """ return isinstance(x, numbers.Integral) + def is_bool(x: Any) -> TypeGuard[int]: """True if x is a boolean (both pure Python or NumPy).""" return type(x) in [bool, np.bool_] @@ -121,6 +122,7 @@ def is_integer_list(x: Any) -> TypeGuard[list[int]]: """ return isinstance(x, list) and len(x) > 0 and is_integer(x[0]) + def is_bool_list(x: Any) -> TypeGuard[list[int]]: """True if x is a list of boolean. @@ -144,9 +146,11 @@ def is_bool_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.b t = t and hasattr(x, "shape") and len(x.shape) == ndim return t + def is_int_or_bool_iterable(x: Any) -> bool: return is_integer_list(x) or is_integer_array(x) or is_bool_array(x) or is_bool_list(x) + def is_scalar(value: Any, dtype: np.dtype[Any]) -> bool: if np.isscalar(value): return True diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 0dee62e8fc..b0fa0febdb 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1343,7 +1343,6 @@ def _test_set_mask_selection(v, a, z, selection): assert_array_equal(a, z[:]) - def test_set_mask_selection_1d(store: StorePath): # setup v = np.arange(1050, dtype=int) @@ -1725,6 +1724,7 @@ def test_accessed_chunks(shape, chunks, ops): # Check that no other chunks were accessed assert len(delta_counts) == 0 + @pytest.mark.parametrize( "selection", [ @@ -1732,20 +1732,21 @@ def test_accessed_chunks(shape, chunks, ops): [...], [1, ...], [slice(None)], - [1,3], - [(1,3)], - [[1, 2, 3],9], + [1, 3], + [(1, 3)], + [[1, 2, 3], 9], [np.arange(1000)], [slice(5, 15)], [slice(2, 4), 4], - [[1,3]], + [[1, 3]], # mask selection - [np.tile([True, False], (1000,5))], + [np.tile([True, False], (1000, 5))], [np.full((1000, 10), False)], # coordinate selection [[1, 2, 3, 4], [5, 6, 7, 8]], [[100, 200, 300], [4, 5, 6]], -]) + ], +) def test_indexing_equals_numpy(store, selection): a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) @@ -1753,14 +1754,16 @@ def test_indexing_equals_numpy(store, selection): actual = z[*selection] assert_array_equal(expected, actual, err_msg=f"selection: {selection}") + @pytest.mark.parametrize( "selection", [ [np.tile([True, False], 500), np.tile([True, False], 5)], [np.full(1000, False), np.tile([True, False], 5)], [np.full(1000, True), np.full(10, True)], - [np.full(1000, True), [True, False]*5], - ]) + [np.full(1000, True), [True, False] * 5], + ], +) def test_orthogonal_bool_indexing_like_numpy_ix(store, selection): a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) From 46a2d4b9774461f66056f7a95a18480ac4d5fc0f Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 17 Jun 2024 12:04:12 +0200 Subject: [PATCH 10/20] fix is_pure_orthogonal_indexing --- src/zarr/indexing.py | 7 +++---- tests/v3/test_indexing.py | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 5e2f967b75..48c78ecba9 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -204,17 +204,16 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or if not ndim: return False - if not isinstance(selection, tuple): - return False + selection = (selection,) if not isinstance(selection, tuple) else selection # Case 1: Selection contains of iterable of integers or boolean - if all(is_int_or_bool_iterable(s) for s in selection): + if len(selection) == ndim and all(is_int_or_bool_iterable(s) for s in selection): return True # Case 2: selection contains either zero or one integer iterables. # All other selection elements are slices or integers return ( - len(selection) == ndim + len(selection) <= ndim and sum(is_int_or_bool_iterable(s) for s in selection) <= 1 and all(is_int_or_bool_iterable(s) or isinstance(s, int | slice) for s in selection) ) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index b0fa0febdb..0731a549bc 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1733,7 +1733,6 @@ def test_accessed_chunks(shape, chunks, ops): [1, ...], [slice(None)], [1, 3], - [(1, 3)], [[1, 2, 3], 9], [np.arange(1000)], [slice(5, 15)], From 9e7b53cd905f38a8cbe7578c1edf6d4a2016e925 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 17 Jun 2024 12:06:36 +0200 Subject: [PATCH 11/20] fix is_pure_orthogonal_indexing --- src/zarr/indexing.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 48c78ecba9..bd59f0f4b4 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -204,18 +204,22 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or if not ndim: return False - selection = (selection,) if not isinstance(selection, tuple) else selection + selection_normalized = (selection,) if not isinstance(selection, tuple) else selection # Case 1: Selection contains of iterable of integers or boolean - if len(selection) == ndim and all(is_int_or_bool_iterable(s) for s in selection): + if len(selection_normalized) == ndim and all( + is_int_or_bool_iterable(s) for s in selection_normalized + ): return True # Case 2: selection contains either zero or one integer iterables. # All other selection elements are slices or integers return ( - len(selection) <= ndim - and sum(is_int_or_bool_iterable(s) for s in selection) <= 1 - and all(is_int_or_bool_iterable(s) or isinstance(s, int | slice) for s in selection) + len(selection_normalized) <= ndim + and sum(is_int_or_bool_iterable(s) for s in selection_normalized) <= 1 + and all( + is_int_or_bool_iterable(s) or isinstance(s, int | slice) for s in selection_normalized + ) ) From b1a2ccfcccf2978a5dbeab194a06cac95aabcf84 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 17 Jun 2024 12:13:07 +0200 Subject: [PATCH 12/20] replace deprecated ~ by not --- src/zarr/array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/array.py b/src/zarr/array.py index 9ac1ce41ec..e14332377f 100644 --- a/src/zarr/array.py +++ b/src/zarr/array.py @@ -389,7 +389,7 @@ def attrs(self) -> dict[str, JSON]: @property def read_only(self) -> bool: - return bool(~self.store_path.store.writeable) + return bool(not self.store_path.store.writeable) @property def path(self) -> str: From 7849f41f6758c789e6dd5ccac491861bf93c1a3d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 17 Jun 2024 13:55:44 +0200 Subject: [PATCH 13/20] restrict is_integer to not bool --- src/zarr/indexing.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index bd59f0f4b4..f784fc9852 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -101,11 +101,8 @@ def ceildiv(a: float, b: float) -> int: def is_integer(x: Any) -> TypeGuard[int]: - """True if x is an integer (both pure Python or NumPy). - - Note that Python's bool is considered an integer too. - """ - return isinstance(x, numbers.Integral) + """True if x is an integer (both pure Python or NumPy).""" + return isinstance(x, numbers.Integral) and not is_bool(x) def is_bool(x: Any) -> TypeGuard[int]: From d52b9d0026b4109cb4275a1613b3de460075b85c Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Wed, 19 Jun 2024 10:53:59 +0200 Subject: [PATCH 14/20] correct typing Co-authored-by: Joe Hamman --- src/zarr/indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index f784fc9852..bd7638487a 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -120,7 +120,7 @@ def is_integer_list(x: Any) -> TypeGuard[list[int]]: return isinstance(x, list) and len(x) > 0 and is_integer(x[0]) -def is_bool_list(x: Any) -> TypeGuard[list[int]]: +def is_bool_list(x: Any) -> TypeGuard[list[bool]]: """True if x is a list of boolean. This function assumes ie *does not check* that all elements of the list From 1b27e653214a5a26c6a3762ca676c70d9eee3f41 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 19 Jun 2024 11:00:24 +0200 Subject: [PATCH 15/20] correct typing --- src/zarr/indexing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index db2122ba3b..1385e14231 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -91,7 +91,7 @@ def is_integer(x: Any) -> TypeGuard[int]: return isinstance(x, numbers.Integral) and not is_bool(x) -def is_bool(x: Any) -> TypeGuard[int]: +def is_bool(x: Any) -> TypeGuard[bool|np.bool_]: """True if x is a boolean (both pure Python or NumPy).""" return type(x) in [bool, np.bool_] @@ -106,7 +106,7 @@ def is_integer_list(x: Any) -> TypeGuard[list[int]]: return isinstance(x, list) and len(x) > 0 and is_integer(x[0]) -def is_bool_list(x: Any) -> TypeGuard[list[bool]]: +def is_bool_list(x: Any) -> TypeGuard[list[bool|np.bool_]]: """True if x is a list of boolean. This function assumes ie *does not check* that all elements of the list From ea6eddbd9c4e10c33342a1d80a72d8ceb53a26d0 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 19 Jun 2024 14:43:43 +0200 Subject: [PATCH 16/20] check if bool list has only bools --- src/zarr/indexing.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 1385e14231..223b9eeb32 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -97,23 +97,13 @@ def is_bool(x: Any) -> TypeGuard[bool|np.bool_]: def is_integer_list(x: Any) -> TypeGuard[list[int]]: - """True if x is a list of integers. - - This function assumes ie *does not check* that all elements of the list - have the same type. Mixed type lists will result in other errors that will - bubble up anyway. - """ - return isinstance(x, list) and len(x) > 0 and is_integer(x[0]) + """True if x is a list of integers.""" + return isinstance(x, list) and len(x) > 0 and all(is_integer(i) for i in x) def is_bool_list(x: Any) -> TypeGuard[list[bool|np.bool_]]: - """True if x is a list of boolean. - - This function assumes ie *does not check* that all elements of the list - have the same type. Mixed type lists will result in other errors that will - bubble up anyway. - """ - return isinstance(x, list) and len(x) > 0 and is_bool(x[0]) + """True if x is a list of boolean.""" + return isinstance(x, list) and len(x) > 0 and all(is_bool(i) for i in x) def is_integer_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.intp]]: From 31c1e7a142bf4d28ced8ae68db91e14931c30991 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 19 Jun 2024 14:44:46 +0200 Subject: [PATCH 17/20] check if bool list has only bools --- src/zarr/indexing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 223b9eeb32..0da0a48080 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -91,7 +91,7 @@ def is_integer(x: Any) -> TypeGuard[int]: return isinstance(x, numbers.Integral) and not is_bool(x) -def is_bool(x: Any) -> TypeGuard[bool|np.bool_]: +def is_bool(x: Any) -> TypeGuard[bool | np.bool_]: """True if x is a boolean (both pure Python or NumPy).""" return type(x) in [bool, np.bool_] @@ -101,7 +101,7 @@ def is_integer_list(x: Any) -> TypeGuard[list[int]]: return isinstance(x, list) and len(x) > 0 and all(is_integer(i) for i in x) -def is_bool_list(x: Any) -> TypeGuard[list[bool|np.bool_]]: +def is_bool_list(x: Any) -> TypeGuard[list[bool | np.bool_]]: """True if x is a list of boolean.""" return isinstance(x, list) and len(x) > 0 and all(is_bool(i) for i in x) From d525d7f5b6c4a06fff2c1aadc3e783a51270d794 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 19 Jun 2024 15:37:16 +0200 Subject: [PATCH 18/20] fix list unpacking in test for python3.10 --- tests/v3/test_indexing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 0731a549bc..dc792924f5 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -191,7 +191,6 @@ def test_get_basic_selection_0d(store: StorePath): slice(50, 150, 10), ] - basic_selections_1d_bad = [ # only positive step supported slice(None, None, -1), @@ -292,7 +291,6 @@ def test_get_basic_selection_1d(store: StorePath): (Ellipsis, slice(None), slice(None)), ] - basic_selections_2d_bad = [ # bad stuff 2.3, @@ -1749,8 +1747,9 @@ def test_accessed_chunks(shape, chunks, ops): def test_indexing_equals_numpy(store, selection): a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) - expected = a[*selection] - actual = z[*selection] + # note: in python 3.10 a[*selection] is no valid unpacking syntax + expected = a[(*selection,)] + actual = z[(*selection,)] assert_array_equal(expected, actual, err_msg=f"selection: {selection}") @@ -1767,5 +1766,6 @@ def test_orthogonal_bool_indexing_like_numpy_ix(store, selection): a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) expected = a[np.ix_(*selection)] - actual = z[*selection] + # note: in python 3.10 z[*selection] is no valid unpacking syntax + actual = z[(*selection,)] assert_array_equal(expected, actual, err_msg=f"{selection=}") From ef7492b937a64fd8359d7fe23f7df626f309a993 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:50:48 +0200 Subject: [PATCH 19/20] Apply spelling suggestions from code review Co-authored-by: Davis Bennett --- tests/v3/test_indexing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index dc792924f5..47dd2a88d0 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1747,7 +1747,7 @@ def test_accessed_chunks(shape, chunks, ops): def test_indexing_equals_numpy(store, selection): a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) - # note: in python 3.10 a[*selection] is no valid unpacking syntax + # note: in python 3.10 a[*selection] is not valid unpacking syntax expected = a[(*selection,)] actual = z[(*selection,)] assert_array_equal(expected, actual, err_msg=f"selection: {selection}") @@ -1766,6 +1766,6 @@ def test_orthogonal_bool_indexing_like_numpy_ix(store, selection): a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) expected = a[np.ix_(*selection)] - # note: in python 3.10 z[*selection] is no valid unpacking syntax + # note: in python 3.10 z[*selection] is not valid unpacking syntax actual = z[(*selection,)] assert_array_equal(expected, actual, err_msg=f"{selection=}") From 4f0321f04be1a494e393be6cae9566e06424c833 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 27 Jun 2024 15:27:33 +0200 Subject: [PATCH 20/20] fix mypy --- src/zarr/indexing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 0da0a48080..ae4aa0681b 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -1035,9 +1035,9 @@ def __init__(self, selection: CoordinateSelection, shape: ChunkCoords, chunk_gri # flatten selection selection_broadcast = tuple(dim_sel.reshape(-1) for dim_sel in selection_broadcast) - chunks_multi_index_broadcast = [ - dim_chunks.reshape(-1) for dim_chunks in chunks_multi_index_broadcast - ] + chunks_multi_index_broadcast = tuple( + [dim_chunks.reshape(-1) for dim_chunks in chunks_multi_index_broadcast] + ) # ravel chunk indices chunks_raveled_indices = np.ravel_multi_index(