diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index d92f8d4e2e..3d9550f733 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -1,6 +1,6 @@ from abc import abstractmethod, ABC - from collections.abc import AsyncGenerator + from typing import List, Tuple, Optional diff --git a/src/zarr/array.py b/src/zarr/array.py index 128d7c58e1..a594b3dd11 100644 --- a/src/zarr/array.py +++ b/src/zarr/array.py @@ -28,6 +28,7 @@ ChunkCoords, Selection, SliceSelection, + ZarrFormat, concurrent_map, ) from zarr.config import config @@ -89,6 +90,7 @@ async def create( dimension_names: Optional[Iterable[str]] = None, attributes: Optional[Dict[str, Any]] = None, exists_ok: bool = False, + zarr_format: ZarrFormat = 3, ) -> AsyncArray: store_path = make_store_path(store) if not exists_ok: diff --git a/src/zarr/common.py b/src/zarr/common.py index ea26cae7b1..95cb8f4a3e 100644 --- a/src/zarr/common.py +++ b/src/zarr/common.py @@ -5,8 +5,6 @@ Union, Tuple, Iterable, - Dict, - List, TypeVar, overload, Any, @@ -18,7 +16,7 @@ import functools if TYPE_CHECKING: - from typing import Any, Awaitable, Callable, Iterator, Optional, Type + from typing import Awaitable, Callable, Iterator, Optional, Type import numpy as np @@ -27,25 +25,26 @@ ZGROUP_JSON = ".zgroup" ZATTRS_JSON = ".zattrs" -BytesLike = Union[bytes, bytearray, memoryview] -ChunkCoords = Tuple[int, ...] +BytesLike = bytes | bytearray | memoryview +ChunkCoords = tuple[int, ...] ChunkCoordsLike = Iterable[int] -SliceSelection = Tuple[slice, ...] -Selection = Union[slice, SliceSelection] -JSON = Union[str, None, int, float, Enum, Dict[str, "JSON"], List["JSON"], Tuple["JSON", ...]] +SliceSelection = tuple[slice, ...] +Selection = slice | SliceSelection +ZarrFormat = Literal[2, 3] +JSON = Union[str, None, int, float, Enum, dict[str, "JSON"], list["JSON"], tuple["JSON", ...]] def product(tup: ChunkCoords) -> int: return functools.reduce(lambda x, y: x * y, tup, 1) -T = TypeVar("T", bound=Tuple[Any, ...]) +T = TypeVar("T", bound=tuple[Any, ...]) V = TypeVar("V") async def concurrent_map( - items: List[T], func: Callable[..., Awaitable[V]], limit: Optional[int] = None -) -> List[V]: + items: list[T], func: Callable[..., Awaitable[V]], limit: Optional[int] = None +) -> list[V]: if limit is None: return await asyncio.gather(*[func(*item) for item in items]) @@ -127,18 +126,18 @@ def parse_configuration(data: JSON) -> JSON: @overload def parse_named_configuration( data: JSON, expected_name: Optional[str] = None -) -> Tuple[str, Dict[str, JSON]]: ... +) -> tuple[str, dict[str, JSON]]: ... @overload def parse_named_configuration( data: JSON, expected_name: Optional[str] = None, *, require_configuration: bool = True -) -> Tuple[str, Optional[Dict[str, JSON]]]: ... +) -> tuple[str, Optional[dict[str, JSON]]]: ... def parse_named_configuration( data: JSON, expected_name: Optional[str] = None, *, require_configuration: bool = True -) -> Tuple[str, Optional[JSON]]: +) -> tuple[str, Optional[JSON]]: if not isinstance(data, dict): raise TypeError(f"Expected dict, got {type(data)}") if "name" not in data: @@ -153,7 +152,7 @@ def parse_named_configuration( return name_parsed, configuration_parsed -def parse_shapelike(data: Any) -> Tuple[int, ...]: +def parse_shapelike(data: Any) -> tuple[int, ...]: if not isinstance(data, Iterable): raise TypeError(f"Expected an iterable. Got {data} instead.") data_tuple = tuple(data) diff --git a/src/zarr/group.py b/src/zarr/group.py index c71860b1b6..cce53d0a98 100644 --- a/src/zarr/group.py +++ b/src/zarr/group.py @@ -5,21 +5,19 @@ import asyncio import json import logging +import numpy.typing as npt if TYPE_CHECKING: - from typing import ( - Any, - AsyncGenerator, - Literal, - AsyncIterator, - ) + from typing import Any, AsyncGenerator, Literal, Iterable +from zarr.abc.codec import Codec from zarr.abc.metadata import Metadata from zarr.array import AsyncArray, Array from zarr.attributes import Attributes -from zarr.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, ZGROUP_JSON +from zarr.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, ZGROUP_JSON, ChunkCoords from zarr.store import StoreLike, StorePath, make_store_path from zarr.sync import SyncMixin, sync +from typing import overload logger = logging.getLogger("zarr.group") @@ -41,6 +39,26 @@ def parse_attributes(data: Any) -> dict[str, Any]: raise TypeError(msg) +@overload +def _parse_async_node(node: AsyncArray) -> Array: ... + + +@overload +def _parse_async_node(node: AsyncGroup) -> Group: ... + + +def _parse_async_node(node: AsyncArray | AsyncGroup) -> Array | Group: + """ + Wrap an AsyncArray in an Array, or an AsyncGroup in a Group. + """ + if isinstance(node, AsyncArray): + return Array(node) + elif isinstance(node, AsyncGroup): + return Group(node) + else: + assert False + + @dataclass(frozen=True) class GroupMetadata(Metadata): attributes: dict[str, Any] = field(default_factory=dict) @@ -53,7 +71,7 @@ def to_bytes(self) -> dict[str, bytes]: return {ZARR_JSON: json.dumps(self.to_dict()).encode()} else: return { - ZGROUP_JSON: json.dumps({"zarr_format": 2}).encode(), + ZGROUP_JSON: json.dumps({"zarr_format": self.zarr_format}).encode(), ZATTRS_JSON: json.dumps(self.attributes).encode(), } @@ -113,11 +131,11 @@ async def open( (store_path / ZGROUP_JSON).get(), (store_path / ZATTRS_JSON).get() ) if zgroup_bytes is None: - raise KeyError(store_path) # filenotfounderror? + raise FileNotFoundError(store_path) elif zarr_format == 3: zarr_json_bytes = await (store_path / ZARR_JSON).get() if zarr_json_bytes is None: - raise KeyError(store_path) # filenotfounderror? + raise FileNotFoundError(store_path) elif zarr_format is None: zarr_json_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( (store_path / ZARR_JSON).get(), @@ -168,6 +186,7 @@ async def getitem( key: str, ) -> AsyncArray | AsyncGroup: store_path = self.store_path / key + logger.warning("key=%s, store_path=%s", key, store_path) # Note: # in zarr-python v2, we first check if `key` references an Array, else if `key` references @@ -175,10 +194,6 @@ async def getitem( # are reusable, but for v3 they would perform redundant I/O operations. # Not clear how much of that strategy we want to keep here. - # if `key` names an object in storage, it cannot be an array or group - if await store_path.exists(): - raise KeyError(key) - if self.metadata.zarr_format == 3: zarr_json_bytes = await (store_path / ZARR_JSON).get() if zarr_json_bytes is None: @@ -248,16 +263,42 @@ def attrs(self): def info(self): return self.metadata.info - async def create_group(self, path: str, **kwargs) -> AsyncGroup: + async def create_group( + self, path: str, exists_ok: bool = False, attributes: dict[str, Any] = {} + ) -> AsyncGroup: return await type(self).create( self.store_path / path, - **kwargs, + attributes=attributes, + exists_ok=exists_ok, + zarr_format=self.metadata.zarr_format, ) - async def create_array(self, path: str, **kwargs) -> AsyncArray: + async def create_array( + self, + path: str, + shape: ChunkCoords, + dtype: npt.DTypeLike, + chunk_shape: ChunkCoords, + fill_value: Any | None = None, + chunk_key_encoding: tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] = ("default", "/"), + codecs: Iterable[Codec | dict[str, Any]] | None = None, + dimension_names: Iterable[str] | None = None, + attributes: dict[str, Any] | None = None, + exists_ok: bool = False, + ) -> AsyncArray: return await AsyncArray.create( self.store_path / path, - **kwargs, + shape=shape, + dtype=dtype, + chunk_shape=chunk_shape, + fill_value=fill_value, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + attributes=attributes, + exists_ok=exists_ok, + zarr_format=self.metadata.zarr_format, ) async def update_attributes(self, new_attributes: dict[str, Any]): @@ -348,7 +389,7 @@ async def array_keys(self) -> AsyncGenerator[str, None]: yield key # todo: decide if this method should be separate from `array_keys` - async def arrays(self) -> AsyncIterator[AsyncArray]: + async def arrays(self) -> AsyncGenerator[AsyncArray, None]: async for key, value in self.members(): if isinstance(value, AsyncArray): yield value @@ -472,19 +513,13 @@ def nmembers(self) -> int: @property def members(self) -> tuple[tuple[str, Array | Group], ...]: """ - Return the sub-arrays and sub-groups of this group as a `tuple` of (name, array | group) + Return the sub-arrays and sub-groups of this group as a tuple of (name, array | group) pairs """ - _members: list[tuple[str, AsyncArray | AsyncGroup]] = self._sync_iter( - self._async_group.members() - ) - ret: list[tuple[str, Array | Group]] = [] - for key, value in _members: - if isinstance(value, AsyncArray): - ret.append((key, Array(value))) - else: - ret.append((key, Group(value))) - return tuple(ret) + _members = self._sync_iter(self._async_group.members()) + + result = tuple(map(lambda kv: (kv[0], _parse_async_node(kv[1])), _members)) + return result def __contains__(self, member) -> bool: return self._sync(self._async_group.contains(member)) diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index e5021b6483..a3dd65979b 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -4,20 +4,20 @@ import shutil from collections.abc import AsyncGenerator from pathlib import Path -from typing import Union, Optional, List, Tuple from zarr.abc.store import Store from zarr.common import BytesLike, concurrent_map, to_thread -def _get(path: Path, byte_range: Optional[Tuple[int, Optional[int]]] = None) -> bytes: +def _get(path: Path, byte_range: tuple[int, int | None] | None) -> bytes: """ Fetch a contiguous region of bytes from a file. + Parameters ---------- path: Path The file to read bytes from. - byte_range: Optional[Tuple[int, Optional[int]]] = None + byte_range: tuple[int, int | None] | None = None The range of bytes to read. If `byte_range` is `None`, then the entire file will be read. If `byte_range` is a tuple, the first value specifies the index of the first byte to read, and the second value specifies the total number of bytes to read. If the total value is @@ -49,7 +49,7 @@ def _get(path: Path, byte_range: Optional[Tuple[int, Optional[int]]] = None) -> def _put( path: Path, value: BytesLike, - start: Optional[int] = None, + start: int | None = None, auto_mkdir: bool = True, ) -> int | None: if auto_mkdir: @@ -71,7 +71,7 @@ class LocalStore(Store): root: Path auto_mkdir: bool - def __init__(self, root: Union[Path, str], auto_mkdir: bool = True): + def __init__(self, root: Path | str, auto_mkdir: bool = True): if isinstance(root, str): root = Path(root) assert isinstance(root, Path) @@ -88,9 +88,7 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: return isinstance(other, type(self)) and self.root == other.root - async def get( - self, key: str, byte_range: Optional[Tuple[int, Optional[int]]] = None - ) -> Optional[bytes]: + async def get(self, key: str, byte_range: tuple[int, int | None] | None = None) -> bytes | None: assert isinstance(key, str) path = self.root / key @@ -100,8 +98,8 @@ async def get( return None async def get_partial_values( - self, key_ranges: List[Tuple[str, Tuple[int, int]]] - ) -> List[Optional[bytes]]: + self, key_ranges: list[tuple[str, tuple[int, int]]] + ) -> list[bytes | None]: """ Read byte ranges from multiple keys. Parameters @@ -124,7 +122,7 @@ async def set(self, key: str, value: BytesLike) -> None: path = self.root / key await to_thread(_put, path, value, auto_mkdir=self.auto_mkdir) - async def set_partial_values(self, key_start_values: List[Tuple[str, int, bytes]]) -> None: + async def set_partial_values(self, key_start_values: list[tuple[str, int, bytes]]) -> None: args = [] for key, start, value in key_start_values: assert isinstance(key, str) @@ -169,6 +167,9 @@ async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: ------- AsyncGenerator[str, None] """ + for p in (self.root / prefix).rglob("*"): + if p.is_file(): + yield str(p) to_strip = str(self.root) + "/" for p in (self.root / prefix).rglob("*"): diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 9f10ed22a3..9730d635d5 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -88,4 +88,4 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: else: for key in self._store_dict: if key.startswith(prefix + "/") and key != prefix: - yield key.strip(prefix + "/").split("/")[0] + yield key.removeprefix(prefix + "/").split("/")[0] diff --git a/tests/v2/conftest.py b/tests/v2/conftest.py index c84cdfa439..225f3fd563 100644 --- a/tests/v2/conftest.py +++ b/tests/v2/conftest.py @@ -1,5 +1,5 @@ -import pathlib import pytest +import pathlib @pytest.fixture(params=[str, pathlib.Path]) diff --git a/tests/v3/__init__.py b/tests/v3/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index 3dc55c0298..3588048906 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -1,7 +1,30 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from zarr.common import ZarrFormat +from zarr.group import AsyncGroup + +if TYPE_CHECKING: + from typing import Any, Literal +from dataclasses import dataclass, field import pathlib + import pytest -from zarr.store import LocalStore, StorePath, MemoryStore, RemoteStore +from zarr.store import LocalStore, StorePath, MemoryStore +from zarr.store.remote import RemoteStore + + +def parse_store( + store: Literal["local", "memory", "remote"], path: str +) -> LocalStore | MemoryStore | RemoteStore: + if store == "local": + return LocalStore(path) + if store == "memory": + return MemoryStore() + if store == "remote": + return RemoteStore() + assert False @pytest.fixture(params=[str, pathlib.Path]) @@ -30,3 +53,30 @@ def remote_store(): @pytest.fixture(scope="function") def memory_store(): return MemoryStore() + + +@pytest.fixture(scope="function") +def store(request: str, tmpdir): + param = request.param + return parse_store(param, str(tmpdir)) + + +@dataclass +class AsyncGroupRequest: + zarr_format: ZarrFormat + store: Literal["local", "remote", "memory"] + attributes: dict[str, Any] = field(default_factory=dict) + + +@pytest.fixture(scope="function") +async def async_group(request: pytest.FixtureRequest, tmpdir) -> AsyncGroup: + param: AsyncGroupRequest = request.param + + store = parse_store(param.store, str(tmpdir)) + agroup = await AsyncGroup.create( + store, + attributes=param.attributes, + zarr_format=param.zarr_format, + exists_ok=False, + ) + return agroup diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index e042c7f275..fc209bd5e6 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -233,7 +233,6 @@ def test_nested_sharding( @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) @pytest.mark.parametrize("with_sharding", [True, False]) -@pytest.mark.asyncio async def test_order( store: Store, input_order: Literal["F", "C"], @@ -344,7 +343,6 @@ def test_order_implicit( @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) @pytest.mark.parametrize("with_sharding", [True, False]) -@pytest.mark.asyncio async def test_transpose( store: Store, input_order: Literal["F", "C"], @@ -601,7 +599,6 @@ def test_write_partial_sharded_chunks(store: Store): assert np.array_equal(a[0:16, 0:16], data) -@pytest.mark.asyncio async def test_delete_empty_chunks(store: Store): data = np.ones((16, 16)) @@ -618,7 +615,6 @@ async def test_delete_empty_chunks(store: Store): assert await (store / "delete_empty_chunks/c0/0").get() is None -@pytest.mark.asyncio async def test_delete_empty_sharded_chunks(store: Store): a = await AsyncArray.create( store / "delete_empty_sharded_chunks", @@ -644,7 +640,6 @@ async def test_delete_empty_sharded_chunks(store: Store): assert chunk_bytes is not None and len(chunk_bytes) == 16 * 2 + 8 * 8 * 2 + 4 -@pytest.mark.asyncio async def test_zarr_compat(store: Store): data = np.zeros((16, 18), dtype="uint16") @@ -676,7 +671,6 @@ async def test_zarr_compat(store: Store): assert z2._store["1.1"] == await (store / "zarr_compat3/1.1").get() -@pytest.mark.asyncio async def test_zarr_compat_F(store: Store): data = np.zeros((16, 18), dtype="uint16", order="F") @@ -710,7 +704,6 @@ async def test_zarr_compat_F(store: Store): assert z2._store["1.1"] == await (store / "zarr_compatF3/1.1").get() -@pytest.mark.asyncio async def test_dimension_names(store: Store): data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) @@ -776,7 +769,6 @@ def test_zstd(store: Store, checksum: bool): @pytest.mark.parametrize("endian", ["big", "little"]) -@pytest.mark.asyncio async def test_endian(store: Store, endian: Literal["big", "little"]): data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) @@ -808,7 +800,6 @@ async def test_endian(store: Store, endian: Literal["big", "little"]): @pytest.mark.parametrize("dtype_input_endian", [">u2", "u2", " None: """ Test that `Group.members` returns correct values, i.e. the arrays and groups (explicit and implicit) contained in that group. """ - store: LocalStore | MemoryStore = request.getfixturevalue(store_type) path = "group" agroup = AsyncGroup( metadata=GroupMetadata(), @@ -50,14 +53,10 @@ def test_group_members(store_type, request): assert sorted(dict(members_observed)) == sorted(members_expected) -@pytest.mark.parametrize("store_type", (("local_store",))) -def test_group(store_type, request) -> None: - store = request.getfixturevalue(store_type) +@pytest.mark.parametrize("store", (("local", "memory")), indirect=["store"]) +def test_group(store: MemoryStore | LocalStore) -> None: store_path = StorePath(store) - agroup = AsyncGroup( - metadata=GroupMetadata(), - store_path=store_path, - ) + agroup = AsyncGroup(metadata=GroupMetadata(), store_path=store_path) group = Group(agroup) assert agroup.metadata is group.metadata @@ -93,10 +92,290 @@ def test_group(store_type, request) -> None: assert dict(bar3.attrs) == {"baz": "qux", "name": "bar"} -def test_group_sync_constructor(store_path) -> None: - group = Group.create( - store=store_path, - attributes={"title": "test 123"}, +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("exists_ok", (True, False)) +def test_group_create(store: MemoryStore | LocalStore, exists_ok: bool) -> None: + """ + Test that `Group.create` works as expected. + """ + attributes = {"foo": 100} + group = Group.create(store, attributes=attributes, exists_ok=exists_ok) + + assert group.attrs == attributes + + if not exists_ok: + with pytest.raises(AssertionError): + group = Group.create( + store, + attributes=attributes, + exists_ok=exists_ok, + ) + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +@pytest.mark.parametrize("exists_ok", (True, False)) +async def test_asyncgroup_create( + store: MemoryStore | LocalStore, + exists_ok: bool, + zarr_format: ZarrFormat, +) -> None: + """ + Test that `AsyncGroup.create` works as expected. + """ + attributes = {"foo": 100} + agroup = await AsyncGroup.create( + store, + attributes=attributes, + exists_ok=exists_ok, + zarr_format=zarr_format, + ) + + assert agroup.metadata == GroupMetadata(zarr_format=zarr_format, attributes=attributes) + assert agroup.store_path == make_store_path(store) + + if not exists_ok: + with pytest.raises(AssertionError): + agroup = await AsyncGroup.create( + store, + attributes=attributes, + exists_ok=exists_ok, + zarr_format=zarr_format, + ) + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_asyncgroup_attrs(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + attributes = {"foo": 100} + agroup = await AsyncGroup.create(store, zarr_format=zarr_format, attributes=attributes) + + assert agroup.attrs == agroup.metadata.attributes == attributes + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_asyncgroup_info(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + agroup = await AsyncGroup.create( # noqa + store, + zarr_format=zarr_format, + ) + pytest.xfail("Info is not implemented for metadata yet") + # assert agroup.info == agroup.metadata.info + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_asyncgroup_open( + store: LocalStore | MemoryStore, + zarr_format: ZarrFormat, +) -> None: + """ + Create an `AsyncGroup`, then ensure that we can open it using `AsyncGroup.open` + """ + attributes = {"foo": 100} + group_w = await AsyncGroup.create( + store=store, + attributes=attributes, + exists_ok=False, + zarr_format=zarr_format, + ) + + group_r = await AsyncGroup.open(store=store, zarr_format=zarr_format) + + assert group_w.attrs == group_w.attrs == attributes + assert group_w == group_r + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_asyncgroup_open_wrong_format( + store: LocalStore | MemoryStore, + zarr_format: ZarrFormat, +) -> None: + _ = await AsyncGroup.create(store=store, exists_ok=False, zarr_format=zarr_format) + + # try opening with the wrong zarr format + if zarr_format == 3: + zarr_format_wrong = 2 + elif zarr_format == 2: + zarr_format_wrong = 3 + else: + assert False + + with pytest.raises(FileNotFoundError): + await AsyncGroup.open(store=store, zarr_format=zarr_format_wrong) + + +# todo: replace the dict[str, Any] type with something a bit more specific +# should this be async? +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize( + "data", + ( + {"zarr_format": 3, "node_type": "group", "attributes": {"foo": 100}}, + {"zarr_format": 2, "attributes": {"foo": 100}}, + ), +) +def test_asyncgroup_from_dict(store: MemoryStore | LocalStore, data: dict[str, Any]) -> None: + """ + Test that we can create an AsyncGroup from a dict + """ + path = "test" + store_path = StorePath(store=store, path=path) + group = AsyncGroup.from_dict(store_path, data=data) + + assert group.metadata.zarr_format == data["zarr_format"] + assert group.metadata.attributes == data["attributes"] + + +# todo: replace this with a declarative API where we model a full hierarchy + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize( + "zarr_format", + (pytest.param(2, marks=pytest.mark.xfail(reason="V2 arrays cannot be created yet.")), 3), +) +async def test_asyncgroup_getitem(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + """ + Create an `AsyncGroup`, then create members of that group, and ensure that we can access those + members via the `AsyncGroup.getitem` method. + """ + agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + + sub_array_path = "sub_array" + sub_array = await agroup.create_array( + path=sub_array_path, shape=(10,), dtype="uint8", chunk_shape=(2,) ) + assert await agroup.getitem(sub_array_path) == sub_array + + sub_group_path = "sub_group" + sub_group = await agroup.create_group(sub_group_path, attributes={"foo": 100}) + assert await agroup.getitem(sub_group_path) == sub_group + + # check that asking for a nonexistent key raises KeyError + with pytest.raises(KeyError): + await agroup.getitem("foo") + + +# todo: replace this with a declarative API where we model a full hierarchy + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize( + "zarr_format", + (2, 3), +) +async def test_asyncgroup_delitem(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + sub_array_path = "sub_array" + _ = await agroup.create_array( + path=sub_array_path, shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} + ) + await agroup.delitem(sub_array_path) + + # todo: clean up the code duplication here + if zarr_format == 2: + assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zarray") + assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zattrs") + elif zarr_format == 3: + assert not await agroup.store_path.store.exists(sub_array_path + "/" + "zarr.json") + else: + assert False + + sub_group_path = "sub_group" + _ = await agroup.create_group(sub_group_path, attributes={"foo": 100}) + await agroup.delitem(sub_group_path) + if zarr_format == 2: + assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zgroup") + assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zattrs") + elif zarr_format == 3: + assert not await agroup.store_path.store.exists(sub_array_path + "/" + "zarr.json") + else: + assert False + - assert group._async_group.metadata.attributes["title"] == "test 123" +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_asyncgroup_create_group( + store: LocalStore | MemoryStore, + zarr_format: ZarrFormat, +) -> None: + agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + sub_node_path = "sub_group" + attributes = {"foo": 999} + subnode = await agroup.create_group(path=sub_node_path, attributes=attributes) + + assert isinstance(subnode, AsyncGroup) + assert subnode.attrs == attributes + assert subnode.store_path.path == sub_node_path + assert subnode.store_path.store == store + assert subnode.metadata.zarr_format == zarr_format + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize( + "zarr_format", + (pytest.param(2, marks=pytest.mark.xfail(reason="V2 arrays cannot be created yet")), 3), +) +async def test_asyncgroup_create_array( + store: LocalStore | MemoryStore, + zarr_format: ZarrFormat, +) -> None: + """ + Test that the AsyncGroup.create_array method works correctly. We ensure that array properties + specified in create_array are present on the resulting array. + """ + + agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + + shape = (10,) + dtype = "uint8" + chunk_shape = (4,) + attributes = {"foo": 100} + + sub_node_path = "sub_array" + subnode = await agroup.create_array( + path=sub_node_path, + shape=shape, + dtype=dtype, + chunk_shape=chunk_shape, + attributes=attributes, + ) + assert isinstance(subnode, AsyncArray) + assert subnode.attrs == attributes + assert subnode.store_path.path == sub_node_path + assert subnode.store_path.store == store + assert subnode.shape == shape + assert subnode.dtype == dtype + # todo: fix the type annotation of array.metadata.chunk_grid so that we get some autocomplete + # here. + assert subnode.metadata.chunk_grid.chunk_shape == chunk_shape + assert subnode.metadata.zarr_format == zarr_format + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_asyncgroup_update_attributes( + store: LocalStore | MemoryStore, zarr_format: ZarrFormat +) -> None: + """ + Test that the AsyncGroup.update_attributes method works correctly. + """ + attributes_old = {"foo": 10} + attributes_new = {"baz": "new"} + agroup = await AsyncGroup.create( + store=store, zarr_format=zarr_format, attributes=attributes_old + ) + + agroup_new_attributes = await agroup.update_attributes(attributes_new) + assert agroup_new_attributes.attrs == attributes_new + + +@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_group_init(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + agroup = sync(AsyncGroup.create(store=store, zarr_format=zarr_format)) + group = Group(agroup) + assert group._async_group == agroup diff --git a/tests/v3/test_storage.py b/tests/v3/test_storage.py deleted file mode 100644 index 2761e608e2..0000000000 --- a/tests/v3/test_storage.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest - -from zarr.testing.store import StoreTests -from zarr.store.local import LocalStore -from zarr.store.memory import MemoryStore - - -class TestMemoryStore(StoreTests): - store_cls = MemoryStore - - -class TestLocalStore(StoreTests): - store_cls = LocalStore - - @pytest.fixture(scope="function") - @pytest.mark.parametrize("auto_mkdir", (True, False)) - def store(self, tmpdir) -> LocalStore: - return self.store_cls(str(tmpdir)) diff --git a/tests/v3/test_store.py b/tests/v3/test_store.py new file mode 100644 index 0000000000..e514d505ce --- /dev/null +++ b/tests/v3/test_store.py @@ -0,0 +1,797 @@ +from __future__ import annotations +from zarr.store.local import LocalStore +from pathlib import Path +import pytest + +from zarr.testing.store import StoreTests +from zarr.store.memory import MemoryStore + + +@pytest.mark.parametrize("auto_mkdir", (True, False)) +def test_local_store_init(tmpdir, auto_mkdir: bool) -> None: + tmpdir_str = str(tmpdir) + tmpdir_path = Path(tmpdir_str) + store = LocalStore(root=tmpdir_str, auto_mkdir=auto_mkdir) + + assert store.root == tmpdir_path + assert store.auto_mkdir == auto_mkdir + + # ensure that str and pathlib.Path get normalized to the same output + assert store == LocalStore(root=tmpdir_path, auto_mkdir=auto_mkdir) + + store_str = f"file://{tmpdir_str}" + assert str(store) == store_str + assert repr(store) == f"LocalStore({store_str!r})" + + +@pytest.mark.parametrize("byte_range", (None, (0, None), (1, None), (1, 2), (None, 1))) +async def test_local_store_get( + local_store, byte_range: None | tuple[int | None, int | None] +) -> None: + payload = b"\x01\x02\x03\x04" + object_name = "foo" + (local_store.root / object_name).write_bytes(payload) + observed = await local_store.get(object_name, byte_range=byte_range) + + if byte_range is None: + start = 0 + length = len(payload) + else: + maybe_start, maybe_len = byte_range + if maybe_start is None: + start = 0 + else: + start = maybe_start + + if maybe_len is None: + length = len(payload) - start + else: + length = maybe_len + + expected = payload[start : start + length] + assert observed == expected + + # test that getting from a file that doesn't exist returns None + assert await local_store.get(object_name + "_absent", byte_range=byte_range) is None + + +@pytest.mark.parametrize( + "key_ranges", + ( + [], + [("key_0", (0, 1))], + [("dir/key_0", (0, 1)), ("key_1", (0, 2))], + [("key_0", (0, 1)), ("key_1", (0, 2)), ("key_1", (0, 2))], + ), +) +async def test_local_store_get_partial( + tmpdir, key_ranges: tuple[list[tuple[str, tuple[int, int]]]] +) -> None: + store = LocalStore(str(tmpdir), auto_mkdir=True) + # use the utf-8 encoding of the key as the bytes + for key, _ in key_ranges: + payload = bytes(key, encoding="utf-8") + target_path: Path = store.root / key + # create the parent directories + target_path.parent.mkdir(parents=True, exist_ok=True) + # write bytes + target_path.write_bytes(payload) + + results = await store.get_partial_values(key_ranges) + for idx, observed in enumerate(results): + key, byte_range = key_ranges[idx] + expected = await store.get(key, byte_range=byte_range) + assert observed == expected + + +@pytest.mark.parametrize("path", ("foo", "foo/bar")) +@pytest.mark.parametrize("auto_mkdir", (True, False)) +async def test_local_store_set(tmpdir, path: str, auto_mkdir: bool) -> None: + store = LocalStore(str(tmpdir), auto_mkdir=auto_mkdir) + payload = b"\x01\x02\x03\x04" + + if "/" in path and not auto_mkdir: + with pytest.raises(FileNotFoundError): + await store.set(path, payload) + else: + x = await store.set(path, payload) + + # this method should not return anything + assert x is None + + assert (store.root / path).read_bytes() == payload + + +# import zarr +# from zarr._storage.store import _get_hierarchy_metadata, v3_api_available, StorageTransformer +# from zarr._storage.v3_storage_transformers import ( +# DummyStorageTransfomer, +# ShardingStorageTransformer, +# v3_sharding_available, +# ) +# from zarr.core import Array +# from zarr.meta import _default_entry_point_metadata_v3 +# from zarr.storage import ( +# atexit_rmglob, +# atexit_rmtree, +# data_root, +# default_compressor, +# getsize, +# init_array, +# meta_root, +# normalize_store_arg, +# ) +# from zarr._storage.v3 import ( +# ABSStoreV3, +# ConsolidatedMetadataStoreV3, +# DBMStoreV3, +# DirectoryStoreV3, +# FSStoreV3, +# KVStore, +# KVStoreV3, +# LMDBStoreV3, +# LRUStoreCacheV3, +# MemoryStoreV3, +# MongoDBStoreV3, +# RedisStoreV3, +# SQLiteStoreV3, +# StoreV3, +# ZipStoreV3, +# ) +# from .util import CountingDictV3, have_fsspec, skip_test_env_var, mktemp + +# # pytest will fail to run if the following fixtures aren't imported here +# from .test_storage import StoreTests as _StoreTests +# from .test_storage import TestABSStore as _TestABSStore +# from .test_storage import TestConsolidatedMetadataStore as _TestConsolidatedMetadataStore +# from .test_storage import TestDBMStore as _TestDBMStore +# from .test_storage import TestDBMStoreBerkeleyDB as _TestDBMStoreBerkeleyDB +# from .test_storage import TestDBMStoreDumb as _TestDBMStoreDumb +# from .test_storage import TestDBMStoreGnu as _TestDBMStoreGnu +# from .test_storage import TestDBMStoreNDBM as _TestDBMStoreNDBM +# from .test_storage import TestDirectoryStore as _TestDirectoryStore +# from .test_storage import TestFSStore as _TestFSStore +# from .test_storage import TestLMDBStore as _TestLMDBStore +# from .test_storage import TestLRUStoreCache as _TestLRUStoreCache +# from .test_storage import TestMemoryStore as _TestMemoryStore +# from .test_storage import TestSQLiteStore as _TestSQLiteStore +# from .test_storage import TestSQLiteStoreInMemory as _TestSQLiteStoreInMemory +# from .test_storage import TestZipStore as _TestZipStore +# from .test_storage import dimension_separator_fixture, s3, skip_if_nested_chunks + + +# pytestmark = pytest.mark.skipif(not v3_api_available, reason="v3 api is not available") + + +# @pytest.fixture( +# params=[ +# (None, "/"), +# (".", "."), +# ("/", "/"), +# ] +# ) +# def dimension_separator_fixture_v3(request): +# return request.param + + +# class DummyStore: +# # contains all methods expected of Mutable Mapping + +# def keys(self): +# """keys""" + +# def values(self): +# """values""" + +# def get(self, value, default=None): +# """get""" + +# def __setitem__(self, key, value): +# """__setitem__""" + +# def __getitem__(self, key): +# """__getitem__""" + +# def __delitem__(self, key): +# """__delitem__""" + +# def __contains__(self, key): +# """__contains__""" + + +# class InvalidDummyStore: +# # does not contain expected methods of a MutableMapping + +# def keys(self): +# """keys""" + + +# def test_ensure_store_v3(): +# class InvalidStore: +# pass + +# with pytest.raises(ValueError): +# StoreV3._ensure_store(InvalidStore()) + +# # cannot initialize with a store from a different Zarr version +# with pytest.raises(ValueError): +# StoreV3._ensure_store(KVStore(dict())) + +# assert StoreV3._ensure_store(None) is None + +# # class with all methods of a MutableMapping will become a KVStoreV3 +# assert isinstance(StoreV3._ensure_store(DummyStore), KVStoreV3) + +# with pytest.raises(ValueError): +# # does not have the methods expected of a MutableMapping +# StoreV3._ensure_store(InvalidDummyStore) + + +# def test_valid_key(): +# store = KVStoreV3(dict) + +# # only ascii keys are valid +# assert not store._valid_key(5) +# assert not store._valid_key(2.8) + +# for key in store._valid_key_characters: +# assert store._valid_key(key) + +# # other characters not in store._valid_key_characters are not allowed +# assert not store._valid_key("*") +# assert not store._valid_key("~") +# assert not store._valid_key("^") + + +# def test_validate_key(): +# store = KVStoreV3(dict) + +# # zarr.json is a valid key +# store._validate_key("zarr.json") +# # but other keys not starting with meta/ or data/ are not +# with pytest.raises(ValueError): +# store._validate_key("zar.json") + +# # valid ascii keys +# for valid in [ +# meta_root + "arr1.array.json", +# data_root + "arr1.array.json", +# meta_root + "subfolder/item_1-0.group.json", +# ]: +# store._validate_key(valid) +# # but otherwise valid keys cannot end in / +# with pytest.raises(ValueError): +# assert store._validate_key(valid + "/") + +# for invalid in [0, "*", "~", "^", "&"]: +# with pytest.raises(ValueError): +# store._validate_key(invalid) + + +# class StoreV3Tests(_StoreTests): + +# version = 3 +# root = meta_root + +# def test_getsize(self): +# # TODO: determine proper getsize() behavior for v3 +# # Currently returns the combined size of entries under +# # meta/root/path and data/root/path. +# # Any path not under meta/root/ or data/root/ (including zarr.json) +# # returns size 0. + +# store = self.create_store() +# if isinstance(store, dict) or hasattr(store, "getsize"): +# assert 0 == getsize(store, "zarr.json") +# store[meta_root + "foo/a"] = b"x" +# assert 1 == getsize(store) +# assert 1 == getsize(store, "foo") +# store[meta_root + "foo/b"] = b"x" +# assert 2 == getsize(store, "foo") +# assert 1 == getsize(store, "foo/b") +# store[meta_root + "bar/a"] = b"yy" +# assert 2 == getsize(store, "bar") +# store[data_root + "bar/a"] = b"zzz" +# assert 5 == getsize(store, "bar") +# store[data_root + "baz/a"] = b"zzz" +# assert 3 == getsize(store, "baz") +# assert 10 == getsize(store) +# store[data_root + "quux"] = array.array("B", b"zzzz") +# assert 14 == getsize(store) +# assert 4 == getsize(store, "quux") +# store[data_root + "spong"] = np.frombuffer(b"zzzzz", dtype="u1") +# assert 19 == getsize(store) +# assert 5 == getsize(store, "spong") +# store.close() + +# def test_init_array(self, dimension_separator_fixture_v3): + +# pass_dim_sep, want_dim_sep = dimension_separator_fixture_v3 + +# store = self.create_store() +# path = "arr1" +# transformer = DummyStorageTransfomer( +# "dummy_type", test_value=DummyStorageTransfomer.TEST_CONSTANT +# ) +# init_array( +# store, +# path=path, +# shape=1000, +# chunks=100, +# dimension_separator=pass_dim_sep, +# storage_transformers=[transformer], +# ) + +# # check metadata +# mkey = meta_root + path + ".array.json" +# assert mkey in store +# meta = store._metadata_class.decode_array_metadata(store[mkey]) +# assert (1000,) == meta["shape"] +# assert (100,) == meta["chunk_grid"]["chunk_shape"] +# assert np.dtype(None) == meta["data_type"] +# assert default_compressor == meta["compressor"] +# assert meta["fill_value"] is None +# # Missing MUST be assumed to be "/" +# assert meta["chunk_grid"]["separator"] is want_dim_sep +# assert len(meta["storage_transformers"]) == 1 +# assert isinstance(meta["storage_transformers"][0], DummyStorageTransfomer) +# assert meta["storage_transformers"][0].test_value == DummyStorageTransfomer.TEST_CONSTANT +# store.close() + +# def test_list_prefix(self): + +# store = self.create_store() +# path = "arr1" +# init_array(store, path=path, shape=1000, chunks=100) + +# expected = [meta_root + "arr1.array.json", "zarr.json"] +# assert sorted(store.list_prefix("")) == expected + +# expected = [meta_root + "arr1.array.json"] +# assert sorted(store.list_prefix(meta_root.rstrip("/"))) == expected + +# # cannot start prefix with '/' +# with pytest.raises(ValueError): +# store.list_prefix(prefix="/" + meta_root.rstrip("/")) + +# def test_equal(self): +# store = self.create_store() +# assert store == store + +# def test_rename_nonexisting(self): +# store = self.create_store() +# if store.is_erasable(): +# with pytest.raises(ValueError): +# store.rename("a", "b") +# else: +# with pytest.raises(NotImplementedError): +# store.rename("a", "b") + +# def test_get_partial_values(self): +# store = self.create_store() +# store.supports_efficient_get_partial_values in [True, False] +# store[data_root + "foo"] = b"abcdefg" +# store[data_root + "baz"] = b"z" +# assert [b"a"] == store.get_partial_values([(data_root + "foo", (0, 1))]) +# assert [ +# b"d", +# b"b", +# b"z", +# b"abc", +# b"defg", +# b"defg", +# b"g", +# b"ef", +# ] == store.get_partial_values( +# [ +# (data_root + "foo", (3, 1)), +# (data_root + "foo", (1, 1)), +# (data_root + "baz", (0, 1)), +# (data_root + "foo", (0, 3)), +# (data_root + "foo", (3, 4)), +# (data_root + "foo", (3, None)), +# (data_root + "foo", (-1, None)), +# (data_root + "foo", (-3, 2)), +# ] +# ) + +# def test_set_partial_values(self): +# store = self.create_store() +# store.supports_efficient_set_partial_values() +# store[data_root + "foo"] = b"abcdefg" +# store.set_partial_values([(data_root + "foo", 0, b"hey")]) +# assert store[data_root + "foo"] == b"heydefg" + +# store.set_partial_values([(data_root + "baz", 0, b"z")]) +# assert store[data_root + "baz"] == b"z" +# store.set_partial_values( +# [ +# (data_root + "foo", 1, b"oo"), +# (data_root + "baz", 1, b"zzz"), +# (data_root + "baz", 4, b"aaaa"), +# (data_root + "foo", 6, b"done"), +# ] +# ) +# assert store[data_root + "foo"] == b"hoodefdone" +# assert store[data_root + "baz"] == b"zzzzaaaa" +# store.set_partial_values( +# [ +# (data_root + "foo", -2, b"NE"), +# (data_root + "baz", -5, b"q"), +# ] +# ) +# assert store[data_root + "foo"] == b"hoodefdoNE" +# assert store[data_root + "baz"] == b"zzzq" + + +# class TestMappingStoreV3(StoreV3Tests): +# def create_store(self, **kwargs): +# return KVStoreV3(dict()) + +# def test_set_invalid_content(self): +# # Generic mappings support non-buffer types +# pass + + +# class TestMemoryStoreV3(_TestMemoryStore, StoreV3Tests): +# def create_store(self, **kwargs): +# skip_if_nested_chunks(**kwargs) +# return MemoryStoreV3(**kwargs) + + +# class TestDirectoryStoreV3(_TestDirectoryStore, StoreV3Tests): +# def create_store(self, normalize_keys=False, **kwargs): +# # For v3, don't have to skip if nested. +# # skip_if_nested_chunks(**kwargs) + +# path = tempfile.mkdtemp() +# atexit.register(atexit_rmtree, path) +# store = DirectoryStoreV3(path, normalize_keys=normalize_keys, **kwargs) +# return store + +# def test_rename_nonexisting(self): +# store = self.create_store() +# with pytest.raises(FileNotFoundError): +# store.rename(meta_root + "a", meta_root + "b") + + +# @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +# class TestFSStoreV3(_TestFSStore, StoreV3Tests): +# def create_store(self, normalize_keys=False, dimension_separator=".", path=None, **kwargs): + +# if path is None: +# path = tempfile.mkdtemp() +# atexit.register(atexit_rmtree, path) + +# store = FSStoreV3( +# path, normalize_keys=normalize_keys, dimension_separator=dimension_separator, **kwargs +# ) +# return store + +# def test_init_array(self): +# store = self.create_store() +# path = "arr1" +# init_array(store, path=path, shape=1000, chunks=100) + +# # check metadata +# mkey = meta_root + path + ".array.json" +# assert mkey in store +# meta = store._metadata_class.decode_array_metadata(store[mkey]) +# assert (1000,) == meta["shape"] +# assert (100,) == meta["chunk_grid"]["chunk_shape"] +# assert np.dtype(None) == meta["data_type"] +# assert meta["chunk_grid"]["separator"] == "/" + + +# @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +# class TestFSStoreV3WithKeySeparator(StoreV3Tests): +# def create_store(self, normalize_keys=False, key_separator=".", **kwargs): + +# # Since the user is passing key_separator, that will take priority. +# skip_if_nested_chunks(**kwargs) + +# path = tempfile.mkdtemp() +# atexit.register(atexit_rmtree, path) +# return FSStoreV3(path, normalize_keys=normalize_keys, key_separator=key_separator) + + +# # TODO: enable once N5StoreV3 has been implemented +# # @pytest.mark.skipif(True, reason="N5StoreV3 not yet fully implemented") +# # class TestN5StoreV3(_TestN5Store, TestDirectoryStoreV3, StoreV3Tests): + + +# class TestZipStoreV3(_TestZipStore, StoreV3Tests): + +# ZipStoreClass = ZipStoreV3 + +# def create_store(self, **kwargs): +# path = mktemp(suffix=".zip") +# atexit.register(os.remove, path) +# store = ZipStoreV3(path, mode="w", **kwargs) +# return store + + +# class TestDBMStoreV3(_TestDBMStore, StoreV3Tests): +# def create_store(self, dimension_separator=None): +# path = mktemp(suffix=".anydbm") +# atexit.register(atexit_rmglob, path + "*") +# # create store using default dbm implementation +# store = DBMStoreV3(path, flag="n", dimension_separator=dimension_separator) +# return store + + +# class TestDBMStoreV3Dumb(_TestDBMStoreDumb, StoreV3Tests): +# def create_store(self, **kwargs): +# path = mktemp(suffix=".dumbdbm") +# atexit.register(atexit_rmglob, path + "*") + +# import dbm.dumb as dumbdbm + +# store = DBMStoreV3(path, flag="n", open=dumbdbm.open, **kwargs) +# return store + + +# class TestDBMStoreV3Gnu(_TestDBMStoreGnu, StoreV3Tests): +# def create_store(self, **kwargs): +# gdbm = pytest.importorskip("dbm.gnu") +# path = mktemp(suffix=".gdbm") # pragma: no cover +# atexit.register(os.remove, path) # pragma: no cover +# store = DBMStoreV3( +# path, flag="n", open=gdbm.open, write_lock=False, **kwargs +# ) # pragma: no cover +# return store # pragma: no cover + + +# class TestDBMStoreV3NDBM(_TestDBMStoreNDBM, StoreV3Tests): +# def create_store(self, **kwargs): +# ndbm = pytest.importorskip("dbm.ndbm") +# path = mktemp(suffix=".ndbm") # pragma: no cover +# atexit.register(atexit_rmglob, path + "*") # pragma: no cover +# store = DBMStoreV3(path, flag="n", open=ndbm.open, **kwargs) # pragma: no cover +# return store # pragma: no cover + + +# class TestDBMStoreV3BerkeleyDB(_TestDBMStoreBerkeleyDB, StoreV3Tests): +# def create_store(self, **kwargs): +# bsddb3 = pytest.importorskip("bsddb3") +# path = mktemp(suffix=".dbm") +# atexit.register(os.remove, path) +# store = DBMStoreV3(path, flag="n", open=bsddb3.btopen, write_lock=False, **kwargs) +# return store + + +# class TestLMDBStoreV3(_TestLMDBStore, StoreV3Tests): +# def create_store(self, **kwargs): +# pytest.importorskip("lmdb") +# path = mktemp(suffix=".lmdb") +# atexit.register(atexit_rmtree, path) +# buffers = True +# store = LMDBStoreV3(path, buffers=buffers, **kwargs) +# return store + + +# class TestSQLiteStoreV3(_TestSQLiteStore, StoreV3Tests): +# def create_store(self, **kwargs): +# pytest.importorskip("sqlite3") +# path = mktemp(suffix=".db") +# atexit.register(atexit_rmtree, path) +# store = SQLiteStoreV3(path, **kwargs) +# return store + + +# class TestSQLiteStoreV3InMemory(_TestSQLiteStoreInMemory, StoreV3Tests): +# def create_store(self, **kwargs): +# pytest.importorskip("sqlite3") +# store = SQLiteStoreV3(":memory:", **kwargs) +# return store + + +# @skip_test_env_var("ZARR_TEST_MONGO") +# class TestMongoDBStoreV3(StoreV3Tests): +# def create_store(self, **kwargs): +# pytest.importorskip("pymongo") +# store = MongoDBStoreV3( +# host="127.0.0.1", database="zarr_tests", collection="zarr_tests", **kwargs +# ) +# # start with an empty store +# store.clear() +# return store + + +# @skip_test_env_var("ZARR_TEST_REDIS") +# class TestRedisStoreV3(StoreV3Tests): +# def create_store(self, **kwargs): +# # TODO: this is the default host for Redis on Travis, +# # we probably want to generalize this though +# pytest.importorskip("redis") +# store = RedisStoreV3(host="localhost", port=6379, **kwargs) +# # start with an empty store +# store.clear() +# return store + + +# @pytest.mark.skipif(not v3_sharding_available, reason="sharding is disabled") +# class TestStorageTransformerV3(TestMappingStoreV3): +# def create_store(self, **kwargs): +# inner_store = super().create_store(**kwargs) +# dummy_transformer = DummyStorageTransfomer( +# "dummy_type", test_value=DummyStorageTransfomer.TEST_CONSTANT +# ) +# sharding_transformer = ShardingStorageTransformer( +# "indexed", +# chunks_per_shard=2, +# ) +# path = "bla" +# init_array( +# inner_store, +# path=path, +# shape=1000, +# chunks=100, +# dimension_separator=".", +# storage_transformers=[dummy_transformer, sharding_transformer], +# ) +# store = Array(store=inner_store, path=path).chunk_store +# store.erase_prefix("data/root/bla/") +# store.clear() +# return store + +# def test_method_forwarding(self): +# store = self.create_store() +# inner_store = store.inner_store.inner_store +# assert store.list() == inner_store.list() +# assert store.list_dir(data_root) == inner_store.list_dir(data_root) + +# assert store.is_readable() +# assert store.is_writeable() +# assert store.is_listable() +# inner_store._readable = False +# inner_store._writeable = False +# inner_store._listable = False +# assert not store.is_readable() +# assert not store.is_writeable() +# assert not store.is_listable() + + +# class TestLRUStoreCacheV3(_TestLRUStoreCache, StoreV3Tests): + +# CountingClass = CountingDictV3 +# LRUStoreClass = LRUStoreCacheV3 + + +# @skip_test_env_var("ZARR_TEST_ABS") +# class TestABSStoreV3(_TestABSStore, StoreV3Tests): + +# ABSStoreClass = ABSStoreV3 + + +# def test_normalize_store_arg_v3(tmpdir): + +# fn = tmpdir.join("store.zip") +# store = normalize_store_arg(str(fn), zarr_version=3, mode="w") +# assert isinstance(store, ZipStoreV3) +# assert "zarr.json" in store + +# # can't pass storage_options to non-fsspec store +# with pytest.raises(ValueError): +# normalize_store_arg(str(fn), zarr_version=3, mode="w", storage_options={"some": "kwargs"}) + +# if have_fsspec: +# import fsspec + +# path = tempfile.mkdtemp() +# store = normalize_store_arg("file://" + path, zarr_version=3, mode="w") +# assert isinstance(store, FSStoreV3) +# assert "zarr.json" in store + +# store = normalize_store_arg(fsspec.get_mapper("file://" + path), zarr_version=3) +# assert isinstance(store, FSStoreV3) + +# # regression for https://github.com/zarr-developers/zarr-python/issues/1382 +# # contents of zarr.json are not important for this test +# out = {"version": 1, "refs": {"zarr.json": "{...}"}} +# store = normalize_store_arg( +# "reference://", +# storage_options={"fo": out, "remote_protocol": "memory"}, zarr_version=3 +# ) +# assert isinstance(store, FSStoreV3) + +# fn = tmpdir.join("store.n5") +# with pytest.raises(NotImplementedError): +# normalize_store_arg(str(fn), zarr_version=3, mode="w") + +# # error on zarr_version=3 with a v2 store +# with pytest.raises(ValueError): +# normalize_store_arg(KVStore(dict()), zarr_version=3, mode="w") + +# # error on zarr_version=2 with a v3 store +# with pytest.raises(ValueError): +# normalize_store_arg(KVStoreV3(dict()), zarr_version=2, mode="w") + + +# class TestConsolidatedMetadataStoreV3(_TestConsolidatedMetadataStore): + +# version = 3 +# ConsolidatedMetadataClass = ConsolidatedMetadataStoreV3 + +# @property +# def metadata_key(self): +# return meta_root + "consolidated/.zmetadata" + +# def test_bad_store_version(self): +# with pytest.raises(ValueError): +# self.ConsolidatedMetadataClass(KVStore(dict())) + + +# def test_get_hierarchy_metadata(): +# store = KVStoreV3({}) + +# # error raised if 'jarr.json' is not in the store +# with pytest.raises(ValueError): +# _get_hierarchy_metadata(store) + +# store["zarr.json"] = _default_entry_point_metadata_v3 +# assert _get_hierarchy_metadata(store) == _default_entry_point_metadata_v3 + +# # ValueError if only a subset of keys are present +# store["zarr.json"] = {"zarr_format": "https://purl.org/zarr/spec/protocol/core/3.0"} +# with pytest.raises(ValueError): +# _get_hierarchy_metadata(store) + +# # ValueError if any unexpected keys are present +# extra_metadata = copy.copy(_default_entry_point_metadata_v3) +# extra_metadata["extra_key"] = "value" +# store["zarr.json"] = extra_metadata +# with pytest.raises(ValueError): +# _get_hierarchy_metadata(store) + + +# def test_top_level_imports(): +# for store_name in [ +# "ABSStoreV3", +# "DBMStoreV3", +# "KVStoreV3", +# "DirectoryStoreV3", +# "LMDBStoreV3", +# "LRUStoreCacheV3", +# "MemoryStoreV3", +# "MongoDBStoreV3", +# "RedisStoreV3", +# "SQLiteStoreV3", +# "ZipStoreV3", +# ]: +# if v3_api_available: +# assert hasattr(zarr, store_name) # pragma: no cover +# else: +# assert not hasattr(zarr, store_name) # pragma: no cover + + +# def _get_public_and_dunder_methods(some_class): +# return set( +# name +# for name, _ in inspect.getmembers(some_class, predicate=inspect.isfunction) +# if not name.startswith("_") or name.startswith("__") +# ) + + +# def test_storage_transformer_interface(): +# store_v3_methods = _get_public_and_dunder_methods(StoreV3) +# store_v3_methods.discard("__init__") +# # Note, getitems() isn't mandatory when get_partial_values() is available +# store_v3_methods.discard("getitems") +# storage_transformer_methods = _get_public_and_dunder_methods(StorageTransformer) +# storage_transformer_methods.discard("__init__") +# storage_transformer_methods.discard("get_config") +# assert storage_transformer_methods == store_v3_methods + + +class TestMemoryStore(StoreTests): + store_cls = MemoryStore + + +class TestLocalStore(StoreTests): + store_cls = LocalStore + + @pytest.fixture(scope="function") + @pytest.mark.parametrize("auto_mkdir", (True, False)) + def store(self, tmpdir) -> LocalStore: + return self.store_cls(str(tmpdir))