From 10badfbfc44740823d670607c8c03e5e641a15d3 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 5 Apr 2024 10:35:46 -0600 Subject: [PATCH 01/23] Add hypothesis tests 1. Roundtrip a numpy array 2. Basic Indexing --- tests/test_properties.py | 142 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 tests/test_properties.py diff --git a/tests/test_properties.py b/tests/test_properties.py new file mode 100644 index 0000000000..036dd4f9ce --- /dev/null +++ b/tests/test_properties.py @@ -0,0 +1,142 @@ +import pytest +from numpy.testing import assert_array_equal + +import zarr +from zarr import Array +from zarr.store import MemoryStore + +pytest.importorskip("hypothesis") + +import hypothesis.extra.numpy as npst # noqa +import hypothesis.strategies as st # noqa +from hypothesis import given, settings # noqa + +#### TODO: Provide this in zarr.strategies +# Copied from Xarray +_attr_keys = st.text(st.characters(), min_size=1) +_attr_values = st.recursive( + st.none() | st.booleans() | st.text(st.characters(), max_size=5), + lambda children: st.lists(children) | st.dictionaries(_attr_keys, children), + max_leaves=3, +) + +# No '/' in array names? +# No '.' in paths? +zarr_key_chars = st.sampled_from("-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") + +# The following should be public strategies +attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) +paths = st.none() | st.text(zarr_key_chars, min_size=1) | st.just("/") +array_names = st.text(zarr_key_chars | st.just("."), min_size=1).filter( + lambda t: not t.startswith((".", "..")) +) +np_arrays = npst.arrays( + # FIXME: re-enable timedeltas once we figure out the fill_value issue. + dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), + shape=npst.array_shapes(max_dims=4), +) +stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) + + +@st.composite +def np_array_and_chunks(draw, *, arrays=np_arrays): + """A hypothesis strategy to generate small sized random arrays. + + Returns: a tuple of the array and a suitable random chunking for it. + """ + array = draw(arrays) + # We want this strategy to shrink towards arrays with smaller number of chunks + # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks + numchunks = draw(st.tuples(*[st.integers(min_value=1, max_value=size) for size in array.shape])) + # 2. and now generate the chunks tuple + chunks = tuple(size // nchunks for size, nchunks in zip(array.shape, numchunks, strict=True)) + return (array, chunks) + + +@st.composite +def arrays(draw, *, stores=stores, arrays=np_arrays, paths=paths, array_names=array_names): + store = draw(stores) + nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) + path = draw(paths) + name = draw(array_names) + attributes = draw(attrs) + + # TODO: clean this up + if path is None and name is None: + array_path = None + array_name = None + elif path is None and name is not None: + array_path = f"{name}" + array_name = f"/{name}" + elif path is not None and name is None: + array_path = path + array_name = None + elif path == "/": + assert name is not None + array_path = name + array_name = "/" + name + else: + assert name is not None + array_path = f"{path}/{name}" + array_name = "/" + array_path + + expected_attrs = {} if attributes is None else attributes + + root = zarr.Group.create(store) + a = root.create_array( + array_path, + shape=nparray.shape, + chunks=chunks, + dtype=nparray.dtype.str, + attributes=attributes, + # TODO: FIXME seems to break with booleans and timedelta + # fill_value=nparray.dtype.type(0), + ) + + assert isinstance(a, Array) + assert nparray.shape == a.shape + # assert chunks == a.chunks # TODO: adapt for v2, v3 + assert array_path == a.path + assert array_name == a.name + # assert a.basename is None # TODO + # assert a.store == normalize_store_arg(store) + assert dict(a.attrs) == expected_attrs + + a[:] = nparray + + store.close() + + return a + + +##### + + +# @pytest.mark.slow +@settings(max_examples=300) +@given(st.data()) +def test_roundtrip(data): + nparray = data.draw(np_arrays) + zarray = data.draw(arrays(arrays=st.just(nparray))) + assert_array_equal(nparray, zarray[:]) + + +# @pytest.mark.slow +@given(data=st.data()) +def test_basic_indexing(data): + def is_negative_slice(idx): + return isinstance(idx, slice) and idx.step is not None and idx.step < 0 + + zarray = data.draw(arrays()) + nparray = zarray[:] + indexer = data.draw( + npst.basic_indices(shape=nparray.shape).filter( + lambda idxr: ( + not ( + is_negative_slice(idxr) + or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) + ) + ) + ) + ) + assert_array_equal(nparray[indexer], zarray[indexer]) From c81be37fecb1dc852dd38b657df69cda81048ace Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 15:45:31 -0600 Subject: [PATCH 02/23] Add compressors This is important for #1931 --- tests/test_properties.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_properties.py b/tests/test_properties.py index 036dd4f9ce..93c2a16b3c 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -36,6 +36,7 @@ shape=npst.array_shapes(max_dims=4), ) stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) +compressors = st.sampled_from([None, "default"]) @st.composite @@ -54,12 +55,21 @@ def np_array_and_chunks(draw, *, arrays=np_arrays): @st.composite -def arrays(draw, *, stores=stores, arrays=np_arrays, paths=paths, array_names=array_names): +def arrays( + draw, + *, + compressors=compressors, + stores=stores, + arrays=np_arrays, + paths=paths, + array_names=array_names, +): store = draw(stores) nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) path = draw(paths) name = draw(array_names) attributes = draw(attrs) + compressor = draw(compressors) # TODO: clean this up if path is None and name is None: @@ -89,6 +99,7 @@ def arrays(draw, *, stores=stores, arrays=np_arrays, paths=paths, array_names=ar chunks=chunks, dtype=nparray.dtype.str, attributes=attributes, + compressor=compressor, # TODO: FIXME seems to break with booleans and timedelta # fill_value=nparray.dtype.type(0), ) @@ -122,6 +133,7 @@ def test_roundtrip(data): # @pytest.mark.slow +@settings(max_examples=500) @given(data=st.data()) def test_basic_indexing(data): def is_negative_slice(idx): From dfd0d1b9671f37797a33d643907ef6de037fe51d Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:16:46 -0600 Subject: [PATCH 03/23] Add more test --- .gitignore | 2 ++ tests/test_properties.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7d32026e13..ede320c9ba 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,5 @@ data/* src/fixture/ .DS_Store +tests/.hypothesis +.hypothesis/ diff --git a/tests/test_properties.py b/tests/test_properties.py index 93c2a16b3c..05845b58db 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,4 +1,5 @@ import pytest +import numpy as np from numpy.testing import assert_array_equal import zarr @@ -132,6 +133,13 @@ def test_roundtrip(data): assert_array_equal(nparray, zarray[:]) +@given(st.data()) +def test_roundtrip_object_array(data): + nparray = data.draw(np_arrays) + zarray = data.draw(arrays(arrays=st.just(nparray))) + assert_array_equal(nparray, zarray[:]) + + # @pytest.mark.slow @settings(max_examples=500) @given(data=st.data()) @@ -151,4 +159,10 @@ def is_negative_slice(idx): ) ) ) - assert_array_equal(nparray[indexer], zarray[indexer]) + actual = zarray[indexer] + assert_array_equal(nparray[indexer], actual) + + new_data = np.ones_like(actual) + zarray[indexer] = new_data + nparray[indexer] = new_data + assert_array_equal(nparray, zarray) From 2ad1b35cee0d3bba0199d8b8d6efd7cd7e626618 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:19:07 -0600 Subject: [PATCH 04/23] Add zarr_version --- tests/test_properties.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_properties.py b/tests/test_properties.py index 05845b58db..93cfa08c0f 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -38,6 +38,7 @@ ) stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) compressors = st.sampled_from([None, "default"]) +zarr_versions = st.sampled_from([2]) @st.composite @@ -64,6 +65,7 @@ def arrays( arrays=np_arrays, paths=paths, array_names=array_names, + zarr_versions=zarr_versions, ): store = draw(stores) nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) @@ -71,6 +73,7 @@ def arrays( name = draw(array_names) attributes = draw(attrs) compressor = draw(compressors) + zarr_version = draw(zarr_versions) # TODO: clean this up if path is None and name is None: @@ -103,6 +106,7 @@ def arrays( compressor=compressor, # TODO: FIXME seems to break with booleans and timedelta # fill_value=nparray.dtype.type(0), + zarr_version=zarr_version, ) assert isinstance(a, Array) From 4b30167ad2f1c861c3cb8797f0628c606f645816 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:23:13 -0600 Subject: [PATCH 05/23] Revert "Add zarr_version" This reverts commit 2ad1b35cee0d3bba0199d8b8d6efd7cd7e626618. --- tests/test_properties.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_properties.py b/tests/test_properties.py index 93cfa08c0f..05845b58db 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -38,7 +38,6 @@ ) stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) compressors = st.sampled_from([None, "default"]) -zarr_versions = st.sampled_from([2]) @st.composite @@ -65,7 +64,6 @@ def arrays( arrays=np_arrays, paths=paths, array_names=array_names, - zarr_versions=zarr_versions, ): store = draw(stores) nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) @@ -73,7 +71,6 @@ def arrays( name = draw(array_names) attributes = draw(attrs) compressor = draw(compressors) - zarr_version = draw(zarr_versions) # TODO: clean this up if path is None and name is None: @@ -106,7 +103,6 @@ def arrays( compressor=compressor, # TODO: FIXME seems to break with booleans and timedelta # fill_value=nparray.dtype.type(0), - zarr_version=zarr_version, ) assert isinstance(a, Array) From 361b2263a7ce31304185bc684fca0c84cf096d9b Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:28:48 -0600 Subject: [PATCH 06/23] ADapt for V3 --- tests/test_properties.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_properties.py b/tests/test_properties.py index 05845b58db..465f80c40a 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,5 +1,5 @@ -import pytest import numpy as np +import pytest from numpy.testing import assert_array_equal import zarr @@ -100,7 +100,7 @@ def arrays( chunks=chunks, dtype=nparray.dtype.str, attributes=attributes, - compressor=compressor, + # compressor=compressor, # TODO: FIXME # TODO: FIXME seems to break with booleans and timedelta # fill_value=nparray.dtype.type(0), ) @@ -165,4 +165,4 @@ def is_negative_slice(idx): new_data = np.ones_like(actual) zarray[indexer] = new_data nparray[indexer] = new_data - assert_array_equal(nparray, zarray) + assert_array_equal(nparray, zarray[:]) From a772880294b5d40dacf2c29b6cdaa460cf856149 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:37:26 -0600 Subject: [PATCH 07/23] Add workflow --- .github/workflows/hypothesis.yaml | 106 ++++++++++++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 107 insertions(+) create mode 100644 .github/workflows/hypothesis.yaml diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml new file mode 100644 index 0000000000..5462a4b971 --- /dev/null +++ b/.github/workflows/hypothesis.yaml @@ -0,0 +1,106 @@ +name: Slow Hypothesis CI +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + types: [opened, reopened, synchronize, labeled] + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: # allows you to trigger manually + +env: + FORCE_COLOR: 3 + +jobs: + detect-ci-trigger: + name: detect ci trigger + runs-on: ubuntu-latest + if: | + github.repository == 'zarr-developers/zarr=python' + && (github.event_name == 'push' || github.event_name == 'pull_request') + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1 + id: detect-trigger + with: + keyword: "[skip-ci]" + + hypothesis: + name: Slow Hypothesis Tests + runs-on: "ubuntu-latest" + needs: detect-ci-trigger + if: | + always() + && ( + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + || needs.detect-ci-trigger.outputs.triggered == 'true' + || contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis') + ) + defaults: + run: + shell: bash -l {0} + + strategy: + matrix: + python-version: ['3.11'] + numpy-version: ['1.26'] + dependency-set: ["optional"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: $PYTHON_VERSION + cache: 'pip' + - name: Install Hatch + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} + hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env + # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache + - name: Restore cached hypothesis directory + id: restore-hypothesis-cache + uses: actions/cache/restore@v4 + with: + path: .hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + cache-hypothesis- + + - name: Run slow Hypothesis tests + if: success() + id: status + run: | + hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-hypothesis + + # explicitly save the cache so it gets updated, also do this even if it fails. + - name: Save cached hypothesis directory + id: save-hypothesis-cache + if: always() && steps.status.outcome != 'skipped' + uses: actions/cache/save@v4 + with: + path: .hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository_owner == 'zarr-developers' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: output-${{ matrix.python-version }}-log.jsonl + issue-title: "Nightly Hypothesis tests failed" + issue-label: "topic-hypothesis" diff --git a/pyproject.toml b/pyproject.toml index 80e03322a6..7f61cd9477 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,7 @@ run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" +run-hypothesis = "pytest --hypothesis-show-statistics tests/test_properties.py" list-env = "pip list" [tool.hatch.envs.docs] From bd90c6a09759e52975a91f7c23a78a43df0581b5 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:38:27 -0600 Subject: [PATCH 08/23] Try again --- .github/workflows/hypothesis.yaml | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 5462a4b971..c54ed58583 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -3,9 +3,11 @@ on: push: branches: - "main" + - "v3" pull_request: branches: - "main" + - "v3" types: [opened, reopened, synchronize, labeled] schedule: - cron: "0 0 * * *" # Daily “At 00:00” UTC @@ -15,33 +17,15 @@ env: FORCE_COLOR: 3 jobs: - detect-ci-trigger: - name: detect ci trigger - runs-on: ubuntu-latest - if: | - github.repository == 'zarr-developers/zarr=python' - && (github.event_name == 'push' || github.event_name == 'pull_request') - outputs: - triggered: ${{ steps.detect-trigger.outputs.trigger-found }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - uses: xarray-contrib/ci-trigger@v1 - id: detect-trigger - with: - keyword: "[skip-ci]" hypothesis: name: Slow Hypothesis Tests runs-on: "ubuntu-latest" - needs: detect-ci-trigger if: | always() && ( (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - || needs.detect-ci-trigger.outputs.triggered == 'true' - || contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis') + # || contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis') ) defaults: run: From 2a9c15c10f13ee4c20a74f923bd8ba865f0981b2 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:40:49 -0600 Subject: [PATCH 09/23] always run --- .github/workflows/hypothesis.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index c54ed58583..9a27ce896a 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -21,12 +21,6 @@ jobs: hypothesis: name: Slow Hypothesis Tests runs-on: "ubuntu-latest" - if: | - always() - && ( - (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - # || contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis') - ) defaults: run: shell: bash -l {0} From 0e5747a6e1981865ddd42a3e71568e61e46c30b5 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:41:41 -0600 Subject: [PATCH 10/23] fix env --- .github/workflows/hypothesis.yaml | 2 +- pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 9a27ce896a..c5a239c274 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -36,7 +36,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: $PYTHON_VERSION + python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install Hatch run: | diff --git a/pyproject.toml b/pyproject.toml index 7f61cd9477..c85da931b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,8 @@ extra-dependencies = [ "msgpack", "lmdb", "pytest-asyncio", - "mypy" + "mypy", + "hypothesis", ] features = ["extra"] From f6b8d049161de9b6f829f06a7c33fc21636f7090 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 20:55:21 -0600 Subject: [PATCH 11/23] Try typing --- src/zarr/strategies.py | 118 +++++++++++++++++++++++++++++++++++++++ tests/test_properties.py | 116 +------------------------------------- 2 files changed, 119 insertions(+), 115 deletions(-) create mode 100644 src/zarr/strategies.py diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py new file mode 100644 index 0000000000..3bbbd05176 --- /dev/null +++ b/src/zarr/strategies.py @@ -0,0 +1,118 @@ +import hypothesis.extra.numpy as npst +import hypothesis.strategies as st +import numpy as np +from hypothesis import given, settings # noqa + +from .array import Array +from .group import Group +from .store import MemoryStore, StoreLike + +# Copied from Xarray +_attr_keys = st.text(st.characters(), min_size=1) +_attr_values = st.recursive( + st.none() | st.booleans() | st.text(st.characters(), max_size=5), + lambda children: st.lists(children) | st.dictionaries(_attr_keys, children), + max_leaves=3, +) + +# No '/' in array names? +# No '.' in paths? +zarr_key_chars = st.sampled_from("-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") + +# The following should be public strategies +attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) +paths = st.none() | st.text(zarr_key_chars, min_size=1) | st.just("/") +array_names = st.text(zarr_key_chars | st.just("."), min_size=1).filter( + lambda t: not t.startswith((".", "..")) +) +np_arrays = npst.arrays( + # FIXME: re-enable timedeltas once we figure out the fill_value issue. + dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), + shape=npst.array_shapes(max_dims=4), +) +stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) +compressors = st.sampled_from([None, "default"]) + + +@st.composite # type: ignore[misc] +def np_array_and_chunks( + draw: st.DrawFn, *, arrays: st.SearchStrategy[np.ndarray] = np_arrays +) -> tuple[np.ndarray, tuple[int]]: + """A hypothesis strategy to generate small sized random arrays. + + Returns: a tuple of the array and a suitable random chunking for it. + """ + array = draw(arrays) + # We want this strategy to shrink towards arrays with smaller number of chunks + # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks + numchunks = draw(st.tuples(*[st.integers(min_value=1, max_value=size) for size in array.shape])) + # 2. and now generate the chunks tuple + chunks = tuple(size // nchunks for size, nchunks in zip(array.shape, numchunks, strict=True)) + return (array, chunks) + + +@st.composite # type: ignore[misc] +def arrays( + draw: st.DrawFn, + *, + compressors: st.SearchStrategy = compressors, + stores: st.SearchStrategy[StoreLike] = stores, + arrays: st.SearchStrategy[np.ndarray] = np_arrays, + paths: st.SearchStrategy[None | str] = paths, + array_names: st.SearchStrategy = array_names, + attrs: st.SearchStrategy = attrs, +) -> Array: + store = draw(stores) + nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) + path = draw(paths) + name = draw(array_names) + attributes = draw(attrs) + # compressor = draw(compressors) + + # TODO: clean this up + if path is None and name is None: + array_path = None + array_name = None + elif path is None and name is not None: + array_path = f"{name}" + array_name = f"/{name}" + elif path is not None and name is None: + array_path = path + array_name = None + elif path == "/": + assert name is not None + array_path = name + array_name = "/" + name + else: + assert name is not None + array_path = f"{path}/{name}" + array_name = "/" + array_path + + expected_attrs = {} if attributes is None else attributes + + root = Group.create(store) + a = root.create_array( + array_path, + shape=nparray.shape, + chunks=chunks, + dtype=nparray.dtype.str, + attributes=attributes, + # compressor=compressor, # TODO: FIXME + # TODO: FIXME seems to break with booleans and timedelta + # fill_value=nparray.dtype.type(0), + ) + + assert isinstance(a, Array) + assert nparray.shape == a.shape + # assert chunks == a.chunks # TODO: adapt for v2, v3 + assert array_path == a.path + assert array_name == a.name + # assert a.basename is None # TODO + # assert a.store == normalize_store_arg(store) + assert dict(a.attrs) == expected_attrs + + a[:] = nparray + + store.close() + + return a diff --git a/tests/test_properties.py b/tests/test_properties.py index 465f80c40a..911aeb3e26 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -2,126 +2,12 @@ import pytest from numpy.testing import assert_array_equal -import zarr -from zarr import Array -from zarr.store import MemoryStore - pytest.importorskip("hypothesis") import hypothesis.extra.numpy as npst # noqa import hypothesis.strategies as st # noqa from hypothesis import given, settings # noqa - -#### TODO: Provide this in zarr.strategies -# Copied from Xarray -_attr_keys = st.text(st.characters(), min_size=1) -_attr_values = st.recursive( - st.none() | st.booleans() | st.text(st.characters(), max_size=5), - lambda children: st.lists(children) | st.dictionaries(_attr_keys, children), - max_leaves=3, -) - -# No '/' in array names? -# No '.' in paths? -zarr_key_chars = st.sampled_from("-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") - -# The following should be public strategies -attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) -paths = st.none() | st.text(zarr_key_chars, min_size=1) | st.just("/") -array_names = st.text(zarr_key_chars | st.just("."), min_size=1).filter( - lambda t: not t.startswith((".", "..")) -) -np_arrays = npst.arrays( - # FIXME: re-enable timedeltas once we figure out the fill_value issue. - dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), - shape=npst.array_shapes(max_dims=4), -) -stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) -compressors = st.sampled_from([None, "default"]) - - -@st.composite -def np_array_and_chunks(draw, *, arrays=np_arrays): - """A hypothesis strategy to generate small sized random arrays. - - Returns: a tuple of the array and a suitable random chunking for it. - """ - array = draw(arrays) - # We want this strategy to shrink towards arrays with smaller number of chunks - # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks - numchunks = draw(st.tuples(*[st.integers(min_value=1, max_value=size) for size in array.shape])) - # 2. and now generate the chunks tuple - chunks = tuple(size // nchunks for size, nchunks in zip(array.shape, numchunks, strict=True)) - return (array, chunks) - - -@st.composite -def arrays( - draw, - *, - compressors=compressors, - stores=stores, - arrays=np_arrays, - paths=paths, - array_names=array_names, -): - store = draw(stores) - nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) - path = draw(paths) - name = draw(array_names) - attributes = draw(attrs) - compressor = draw(compressors) - - # TODO: clean this up - if path is None and name is None: - array_path = None - array_name = None - elif path is None and name is not None: - array_path = f"{name}" - array_name = f"/{name}" - elif path is not None and name is None: - array_path = path - array_name = None - elif path == "/": - assert name is not None - array_path = name - array_name = "/" + name - else: - assert name is not None - array_path = f"{path}/{name}" - array_name = "/" + array_path - - expected_attrs = {} if attributes is None else attributes - - root = zarr.Group.create(store) - a = root.create_array( - array_path, - shape=nparray.shape, - chunks=chunks, - dtype=nparray.dtype.str, - attributes=attributes, - # compressor=compressor, # TODO: FIXME - # TODO: FIXME seems to break with booleans and timedelta - # fill_value=nparray.dtype.type(0), - ) - - assert isinstance(a, Array) - assert nparray.shape == a.shape - # assert chunks == a.chunks # TODO: adapt for v2, v3 - assert array_path == a.path - assert array_name == a.name - # assert a.basename is None # TODO - # assert a.store == normalize_store_arg(store) - assert dict(a.attrs) == expected_attrs - - a[:] = nparray - - store.close() - - return a - - -##### +from zarr.strategies import arrays, np_arrays # noqa # @pytest.mark.slow From 70b5269b6731b454013db69edaae6d0a2898d8c2 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 21:07:49 -0600 Subject: [PATCH 12/23] Cleanup --- src/zarr/strategies.py | 21 +++++++++++++++++++++ tests/test_properties.py | 22 ++++++++-------------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index 3bbbd05176..2f6fdc7d7e 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -1,3 +1,5 @@ +from typing import Any + import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np @@ -116,3 +118,22 @@ def arrays( store.close() return a + + +def is_negative_slice(idx: Any) -> bool: + return isinstance(idx, slice) and idx.step is not None and idx.step < 0 + + +@st.composite # type: ignore[misc] +def basic_indices(draw: st.DrawFn, *, shape: tuple[int]): + """Basic indices without unsupported negative slices.""" + return draw( + npst.basic_indices(shape=shape).filter( + lambda idxr: ( + not ( + is_negative_slice(idxr) + or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) + ) + ) + ) + ) diff --git a/tests/test_properties.py b/tests/test_properties.py index 911aeb3e26..6abb84a043 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -7,7 +7,7 @@ import hypothesis.extra.numpy as npst # noqa import hypothesis.strategies as st # noqa from hypothesis import given, settings # noqa -from zarr.strategies import arrays, np_arrays # noqa +from zarr.strategies import arrays, np_arrays, basic_indices # noqa # @pytest.mark.slow @@ -30,21 +30,9 @@ def test_roundtrip_object_array(data): @settings(max_examples=500) @given(data=st.data()) def test_basic_indexing(data): - def is_negative_slice(idx): - return isinstance(idx, slice) and idx.step is not None and idx.step < 0 - zarray = data.draw(arrays()) nparray = zarray[:] - indexer = data.draw( - npst.basic_indices(shape=nparray.shape).filter( - lambda idxr: ( - not ( - is_negative_slice(idxr) - or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) - ) - ) - ) - ) + indexer = data.draw(basic_indices(shape=nparray.shape)) actual = zarray[indexer] assert_array_equal(nparray[indexer], actual) @@ -52,3 +40,9 @@ def is_negative_slice(idx): zarray[indexer] = new_data nparray[indexer] = new_data assert_array_equal(nparray, zarray[:]) + + +@settings(max_examples=500) +@given(data=st.data()) +def test_advanced_indexing(data): + pass From 9c7768357a73aa6c96f69dbeb22581f362cea760 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 3 Jun 2024 21:43:46 -0600 Subject: [PATCH 13/23] Add vindex --- src/zarr/strategies.py | 4 ++-- tests/test_properties.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index 2f6fdc7d7e..418e34e458 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -125,10 +125,10 @@ def is_negative_slice(idx: Any) -> bool: @st.composite # type: ignore[misc] -def basic_indices(draw: st.DrawFn, *, shape: tuple[int]): +def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs): """Basic indices without unsupported negative slices.""" return draw( - npst.basic_indices(shape=shape).filter( + npst.basic_indices(shape=shape, **kwargs).filter( lambda idxr: ( not ( is_negative_slice(idxr) diff --git a/tests/test_properties.py b/tests/test_properties.py index 6abb84a043..4c4c1c21ee 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,3 +1,5 @@ +from itertools import zip_longest + import numpy as np import pytest from numpy.testing import assert_array_equal @@ -42,7 +44,33 @@ def test_basic_indexing(data): assert_array_equal(nparray, zarray[:]) +@st.composite +def advanced_indices(draw, *, shape): + basic_idxr = draw( + basic_indices( + shape=shape, min_dims=len(shape), max_dims=len(shape), allow_ellipsis=False + ).filter(lambda x: isinstance(x, tuple)) + ) + + int_idxr = draw( + npst.integer_array_indices(shape=shape, result_shape=npst.array_shapes(max_dims=1)) + ) + args = tuple( + st.sampled_from((l, r)) for l, r in zip_longest(basic_idxr, int_idxr, fillvalue=slice(None)) + ) + return draw(st.tuples(*args)) + + @settings(max_examples=500) @given(data=st.data()) -def test_advanced_indexing(data): - pass +def test_vindex(data): + zarray = data.draw(arrays()) + nparray = zarray[:] + + indexer = data.draw( + npst.integer_array_indices( + shape=nparray.shape, result_shape=npst.array_shapes(max_dims=None) + ) + ) + actual = zarray.vindex[indexer] + assert_array_equal(nparray[indexer], actual) From 14cc22d9e78d4ba5ae88a7b9e5805d8a9f19d8ee Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 25 Jun 2024 08:02:13 -0400 Subject: [PATCH 14/23] Review feedback --- src/zarr/strategies.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index 418e34e458..a00f42e334 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -17,16 +17,14 @@ max_leaves=3, ) -# No '/' in array names? -# No '.' in paths? -zarr_key_chars = st.sampled_from("-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") +# From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names +zarr_key_chars = st.sampled_from( + ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" +) # The following should be public strategies attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) paths = st.none() | st.text(zarr_key_chars, min_size=1) | st.just("/") -array_names = st.text(zarr_key_chars | st.just("."), min_size=1).filter( - lambda t: not t.startswith((".", "..")) -) np_arrays = npst.arrays( # FIXME: re-enable timedeltas once we figure out the fill_value issue. dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), @@ -35,6 +33,15 @@ stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) compressors = st.sampled_from([None, "default"]) +# 1. must not be the empty string ("") +# 2. must not be the empty string ("") +# 3. must not include the character "/" +# 4. must not be a string composed only of period characters, e.g. "." or ".." +# 5. must not start with the reserved prefix "__" +array_names = st.text(zarr_key_chars, min_size=1).filter( + lambda t: t not in (".", "..") and not t.startswith("__") +) + @st.composite # type: ignore[misc] def np_array_and_chunks( From e51185fd8a276c20977036ece0b504aa30dca7ae Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 25 Jun 2024 08:09:10 -0400 Subject: [PATCH 15/23] cleanup --- src/zarr/strategies.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index a00f42e334..ace7a18f89 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -18,11 +18,15 @@ ) # From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names +# 1. must not be the empty string ("") +# 2. must not be the empty string ("") +# 3. must not include the character "/" +# 4. must not be a string composed only of period characters, e.g. "." or ".." +# 5. must not start with the reserved prefix "__" zarr_key_chars = st.sampled_from( ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ) - -# The following should be public strategies +array_names = st.text(zarr_key_chars, min_size=1) attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) paths = st.none() | st.text(zarr_key_chars, min_size=1) | st.just("/") np_arrays = npst.arrays( @@ -33,15 +37,6 @@ stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) compressors = st.sampled_from([None, "default"]) -# 1. must not be the empty string ("") -# 2. must not be the empty string ("") -# 3. must not include the character "/" -# 4. must not be a string composed only of period characters, e.g. "." or ".." -# 5. must not start with the reserved prefix "__" -array_names = st.text(zarr_key_chars, min_size=1).filter( - lambda t: t not in (".", "..") and not t.startswith("__") -) - @st.composite # type: ignore[misc] def np_array_and_chunks( From 35ba251bcf02ce170718859237499f819cf3e202 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 25 Jun 2024 09:45:36 -0400 Subject: [PATCH 16/23] WIP --- src/zarr/strategies.py | 45 ++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index ace7a18f89..fb44f8502b 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -26,9 +26,11 @@ zarr_key_chars = st.sampled_from( ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ) -array_names = st.text(zarr_key_chars, min_size=1) +array_names = st.text(zarr_key_chars, min_size=1).filter(lambda t: t not in (".", "..")) attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) -paths = st.none() | st.text(zarr_key_chars, min_size=1) | st.just("/") +paths = st.lists(st.text(zarr_key_chars, min_size=1), min_size=1).map( + lambda x: "/".join(x) +) | st.just("/") np_arrays = npst.arrays( # FIXME: re-enable timedeltas once we figure out the fill_value issue. dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), @@ -74,26 +76,27 @@ def arrays( # compressor = draw(compressors) # TODO: clean this up - if path is None and name is None: - array_path = None - array_name = None - elif path is None and name is not None: - array_path = f"{name}" - array_name = f"/{name}" - elif path is not None and name is None: - array_path = path - array_name = None - elif path == "/": - assert name is not None - array_path = name - array_name = "/" + name - else: - assert name is not None - array_path = f"{path}/{name}" - array_name = "/" + array_path + # if path is None and name is None: + # array_path = None + # array_name = None + # elif path is None and name is not None: + # array_path = f"{name}" + # array_name = f"/{name}" + # elif path is not None and name is None: + # array_path = path + # array_name = None + # elif path == "/": + # assert name is not None + # array_path = name + # array_name = "/" + name + # else: + # assert name is not None + # array_path = f"{path}/{name}" + # array_name = "/" + array_path expected_attrs = {} if attributes is None else attributes + array_path = path + ("/" if not path.endswith("/") else "") + name root = Group.create(store) a = root.create_array( array_path, @@ -109,8 +112,8 @@ def arrays( assert isinstance(a, Array) assert nparray.shape == a.shape # assert chunks == a.chunks # TODO: adapt for v2, v3 - assert array_path == a.path - assert array_name == a.name + assert array_path == a.path, (path, name, array_path, a.name, a.path) + assert array_path == a.name, (path, name, array_path, a.name, a.path) # assert a.basename is None # TODO # assert a.store == normalize_store_arg(store) assert dict(a.attrs) == expected_attrs From c041fd4dd99cf97ef3ab16a13addc300d5ea25e7 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 24 Jul 2024 21:24:55 -0600 Subject: [PATCH 17/23] Cleanup --- .gitignore | 1 + pyproject.toml | 2 +- src/zarr/strategies.py | 17 ++++++++----- tests/test_properties.py | 55 ++++++++++++++++++---------------------- tests/v3/conftest.py | 16 ++++++++++++ 5 files changed, 52 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index ab0bd6f8fe..84bcb00ffb 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ src/zarr/_version.py #test_sync* data/* src/fixture/ +fixture/ .DS_Store tests/.hypothesis diff --git a/pyproject.toml b/pyproject.toml index 5a352a9464..58e0c7ebf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -140,7 +140,7 @@ run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-show-statistics tests/test_properties.py" +run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py" list-env = "pip list" [tool.hatch.envs.docs] diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index fb44f8502b..d1af40e090 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -32,7 +32,7 @@ lambda x: "/".join(x) ) | st.just("/") np_arrays = npst.arrays( - # FIXME: re-enable timedeltas once we figure out the fill_value issue. + # TODO: re-enable timedeltas once they are supported dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), shape=npst.array_shapes(max_dims=4), ) @@ -42,7 +42,7 @@ @st.composite # type: ignore[misc] def np_array_and_chunks( - draw: st.DrawFn, *, arrays: st.SearchStrategy[np.ndarray] = np_arrays + draw: st.DrawFn, *, arrays: st.SearchStrategy = np_arrays ) -> tuple[np.ndarray, tuple[int]]: """A hypothesis strategy to generate small sized random arrays. @@ -98,6 +98,10 @@ def arrays( array_path = path + ("/" if not path.endswith("/") else "") + name root = Group.create(store) + fill_value_args: tuple[Any, ...] = tuple() + if nparray.dtype.kind == "M": + fill_value_args = ("ns",) + a = root.create_array( array_path, shape=nparray.shape, @@ -105,15 +109,14 @@ def arrays( dtype=nparray.dtype.str, attributes=attributes, # compressor=compressor, # TODO: FIXME - # TODO: FIXME seems to break with booleans and timedelta - # fill_value=nparray.dtype.type(0), + fill_value=nparray.dtype.type(0, *fill_value_args), ) assert isinstance(a, Array) assert nparray.shape == a.shape - # assert chunks == a.chunks # TODO: adapt for v2, v3 + assert chunks == a.chunks assert array_path == a.path, (path, name, array_path, a.name, a.path) - assert array_path == a.name, (path, name, array_path, a.name, a.path) + # assert array_path == a.name, (path, name, array_path, a.name, a.path) # assert a.basename is None # TODO # assert a.store == normalize_store_arg(store) assert dict(a.attrs) == expected_attrs @@ -130,7 +133,7 @@ def is_negative_slice(idx: Any) -> bool: @st.composite # type: ignore[misc] -def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs): +def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs): # type: ignore[no-untyped-def] """Basic indices without unsupported negative slices.""" return draw( npst.basic_indices(shape=shape, **kwargs).filter( diff --git a/tests/test_properties.py b/tests/test_properties.py index 4c4c1c21ee..d339f5dff6 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,5 +1,3 @@ -from itertools import zip_longest - import numpy as np import pytest from numpy.testing import assert_array_equal @@ -12,8 +10,6 @@ from zarr.strategies import arrays, np_arrays, basic_indices # noqa -# @pytest.mark.slow -@settings(max_examples=300) @given(st.data()) def test_roundtrip(data): nparray = data.draw(np_arrays) @@ -21,15 +17,6 @@ def test_roundtrip(data): assert_array_equal(nparray, zarray[:]) -@given(st.data()) -def test_roundtrip_object_array(data): - nparray = data.draw(np_arrays) - zarray = data.draw(arrays(arrays=st.just(nparray))) - assert_array_equal(nparray, zarray[:]) - - -# @pytest.mark.slow -@settings(max_examples=500) @given(data=st.data()) def test_basic_indexing(data): zarray = data.draw(arrays()) @@ -44,24 +31,6 @@ def test_basic_indexing(data): assert_array_equal(nparray, zarray[:]) -@st.composite -def advanced_indices(draw, *, shape): - basic_idxr = draw( - basic_indices( - shape=shape, min_dims=len(shape), max_dims=len(shape), allow_ellipsis=False - ).filter(lambda x: isinstance(x, tuple)) - ) - - int_idxr = draw( - npst.integer_array_indices(shape=shape, result_shape=npst.array_shapes(max_dims=1)) - ) - args = tuple( - st.sampled_from((l, r)) for l, r in zip_longest(basic_idxr, int_idxr, fillvalue=slice(None)) - ) - return draw(st.tuples(*args)) - - -@settings(max_examples=500) @given(data=st.data()) def test_vindex(data): zarray = data.draw(arrays()) @@ -74,3 +43,27 @@ def test_vindex(data): ) actual = zarray.vindex[indexer] assert_array_equal(nparray[indexer], actual) + + +# @st.composite +# def advanced_indices(draw, *, shape): +# basic_idxr = draw( +# basic_indices( +# shape=shape, min_dims=len(shape), max_dims=len(shape), allow_ellipsis=False +# ).filter(lambda x: isinstance(x, tuple)) +# ) + +# int_idxr = draw( +# npst.integer_array_indices(shape=shape, result_shape=npst.array_shapes(max_dims=1)) +# ) +# args = tuple( +# st.sampled_from((l, r)) for l, r in zip_longest(basic_idxr, int_idxr, fillvalue=slice(None)) +# ) +# return draw(st.tuples(*args)) + + +# @given(st.data()) +# def test_roundtrip_object_array(data): +# nparray = data.draw(np_arrays) +# zarray = data.draw(arrays(arrays=st.just(nparray))) +# assert_array_equal(nparray, zarray[:]) diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index 8b75d9f2f8..a8c3d12086 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -111,3 +111,19 @@ def array_fixture(request: pytest.FixtureRequest) -> np.ndarray: .reshape(array_request.shape, order=array_request.order) .astype(array_request.dtype) ) + + +from hypothesis import HealthCheck, Verbosity, settings + +settings.register_profile( + "ci", + max_examples=1000, + deadline=None, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], +) +settings.register_profile( + "local", + max_examples=300, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], + verbosity=Verbosity.verbose, +) From e8175aaae268f7699769fb09ddb72732c83a1c49 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 24 Jul 2024 21:27:56 -0600 Subject: [PATCH 18/23] Move to v3/ --- pyproject.toml | 2 +- tests/v3/conftest.py | 3 +-- tests/{ => v3}/test_properties.py | 0 3 files changed, 2 insertions(+), 3 deletions(-) rename tests/{ => v3}/test_properties.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 58e0c7ebf8..93116bede8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -140,7 +140,7 @@ run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py" +run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py" list-env = "pip list" [tool.hatch.envs.docs] diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index a8c3d12086..74972ccae7 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -17,6 +17,7 @@ import numpy as np import pytest +from hypothesis import HealthCheck, Verbosity, settings from zarr.store import LocalStore, MemoryStore, StorePath from zarr.store.remote import RemoteStore @@ -113,8 +114,6 @@ def array_fixture(request: pytest.FixtureRequest) -> np.ndarray: ) -from hypothesis import HealthCheck, Verbosity, settings - settings.register_profile( "ci", max_examples=1000, diff --git a/tests/test_properties.py b/tests/v3/test_properties.py similarity index 100% rename from tests/test_properties.py rename to tests/v3/test_properties.py From 06c418f1adf7d040201aeed467dca9a7e595ad04 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 24 Jul 2024 21:30:58 -0600 Subject: [PATCH 19/23] another type ignore --- src/zarr/strategies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index d1af40e090..c1e00907ce 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -42,8 +42,8 @@ @st.composite # type: ignore[misc] def np_array_and_chunks( - draw: st.DrawFn, *, arrays: st.SearchStrategy = np_arrays -) -> tuple[np.ndarray, tuple[int]]: + draw: st.DrawFn, *, arrays: st.SearchStrategy[np.ndarray] = np_arrays +) -> tuple[np.ndarray, tuple[int]]: # type: ignore[type-arg] """A hypothesis strategy to generate small sized random arrays. Returns: a tuple of the array and a suitable random chunking for it. From b2408d010e26919ad3d78afccbd59112b44b38cc Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 5 Aug 2024 10:40:42 -0600 Subject: [PATCH 20/23] Add `_` --- src/zarr/strategies.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index c1e00907ce..a6c52d6bcd 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -19,18 +19,18 @@ # From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names # 1. must not be the empty string ("") -# 2. must not be the empty string ("") -# 3. must not include the character "/" -# 4. must not be a string composed only of period characters, e.g. "." or ".." -# 5. must not start with the reserved prefix "__" +# 2. must not include the character "/" +# 3. must not be a string composed only of period characters, e.g. "." or ".." +# 4. must not start with the reserved prefix "__" zarr_key_chars = st.sampled_from( - ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" + "_.-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ) -array_names = st.text(zarr_key_chars, min_size=1).filter(lambda t: t not in (".", "..")) +node_names = st.text(zarr_key_chars, min_size=1).filter( + lambda t: t not in (".", "..") and not t.startswith("__") +) +array_names = node_names attrs = st.none() | st.dictionaries(_attr_keys, _attr_values) -paths = st.lists(st.text(zarr_key_chars, min_size=1), min_size=1).map( - lambda x: "/".join(x) -) | st.just("/") +paths = st.lists(node_names, min_size=1).map(lambda x: "/".join(x)) | st.just("/") np_arrays = npst.arrays( # TODO: re-enable timedeltas once they are supported dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), From dda685718b55dbfdcc0e8973b8e35eb735eb626a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 8 Aug 2024 07:27:46 -0600 Subject: [PATCH 21/23] Update src/zarr/strategies.py --- src/zarr/strategies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index a6c52d6bcd..91808268f9 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -123,7 +123,6 @@ def arrays( a[:] = nparray - store.close() return a From 66a773781d627039b41dbb6d528a2844d4bcdefd Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 8 Aug 2024 07:28:35 -0600 Subject: [PATCH 22/23] Update src/zarr/strategies.py --- src/zarr/strategies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index 91808268f9..3cd3efdbbd 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -23,7 +23,7 @@ # 3. must not be a string composed only of period characters, e.g. "." or ".." # 4. must not start with the reserved prefix "__" zarr_key_chars = st.sampled_from( - "_.-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" + ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ) node_names = st.text(zarr_key_chars, min_size=1).filter( lambda t: t not in (".", "..") and not t.startswith("__") From c93dee0af04185401330686c890f59309582d4fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:30:12 +0000 Subject: [PATCH 23/23] style: pre-commit fixes --- src/zarr/strategies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index 3cd3efdbbd..91a8542ce9 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -123,7 +123,6 @@ def arrays( a[:] = nparray - return a