diff --git a/fixture/flat/.zarray b/fixture/flat/.zarray new file mode 100644 index 0000000000..8ec79419da --- /dev/null +++ b/fixture/flat/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 2, + 2 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": " 0 -def is_chunk_key(key): - segments = list(key.split('/')) - if segments: - last_segment = segments[-1] - return _prog_ckey.match(last_segment) - return False # pragma: no cover +try: + from .storage import FSStore + + class N5FSStore(FSStore): + """Implentation of the N5 format (https://github.com/saalfeldlab/n5) using `fsspec`, + which allows storage on a variety of filesystems. Based on `zarr.N5Store`. + Parameters + ---------- + path : string + Location of directory to use as the root of the storage hierarchy. + normalize_keys : bool, optional + If True, all store keys will be normalized to use lower case characters + (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be + useful to avoid potential discrepancies between case-senstive and + case-insensitive file system. Default value is False. + + Examples + -------- + Store a single array:: + + >>> import zarr + >>> store = zarr.N5FSStore('data/array.n5', auto_mkdir=True) + >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) + >>> z[...] = 42 + + Store a group:: + + >>> store = zarr.N5FSStore('data/group.n5', auto_mkdir=True) + >>> root = zarr.group(store=store, overwrite=True) + >>> foo = root.create_group('foo') + >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) + >>> bar[...] = 42 + + Notes + ----- + This is an experimental feature. + Safe to write in multiple threads or processes. + """ + array_meta_key = 'attributes.json' + group_meta_key = 'attributes.json' + attrs_key = 'attributes.json' + + def __init__(self, *args, **kwargs): + if 'dimension_separator' in kwargs: + kwargs.pop('dimension_separator') + warnings.warn('Keyword argument `dimension_separator` will be ignored') + dimension_separator = "/" + super().__init__(*args, dimension_separator=dimension_separator, **kwargs) + + def _normalize_key(self, key): + if is_chunk_key(key): + key = invert_chunk_coords(key) + + key = normalize_storage_path(key).lstrip("/") + if key: + *bits, end = key.split("/") + + if end not in (self.array_meta_key, self.group_meta_key, self.attrs_key): + end = end.replace(".", self.key_separator) + key = "/".join(bits + [end]) + return key.lower() if self.normalize_keys else key + + def __getitem__(self, key): + if key.endswith(zarr_group_meta_key): + + key = key.replace(zarr_group_meta_key, self.group_meta_key) + value = group_metadata_to_zarr(self._load_n5_attrs(key)) + return json_dumps(value) -def invert_chunk_coords(key): - segments = list(key.split('/')) - if segments: - last_segment = segments[-1] - if _prog_ckey.match(last_segment): - coords = list(last_segment.split('.')) - last_segment = '.'.join(coords[::-1]) - segments = segments[:-1] + [last_segment] - key = '/'.join(segments) - return key + elif key.endswith(zarr_array_meta_key): + key = key.replace(zarr_array_meta_key, self.array_meta_key) + value = array_metadata_to_zarr(self._load_n5_attrs(key)) -def group_metadata_to_n5(group_metadata): - '''Convert group metadata from zarr to N5 format.''' - del group_metadata['zarr_format'] - # TODO: This should only exist at the top-level - group_metadata['n5'] = '2.0.0' - return group_metadata + return json_dumps(value) + elif key.endswith(zarr_attrs_key): -def group_metadata_to_zarr(group_metadata): - '''Convert group metadata from N5 to zarr format.''' - # This only exists at the top level - group_metadata.pop('n5', None) - group_metadata['zarr_format'] = ZARR_FORMAT - return group_metadata + key = key.replace(zarr_attrs_key, self.attrs_key) + value = attrs_to_zarr(self._load_n5_attrs(key)) + if len(value) == 0: + raise KeyError(key) + else: + return json_dumps(value) + return super().__getitem__(key) -def array_metadata_to_n5(array_metadata): - '''Convert array metadata from zarr to N5 format.''' + def __setitem__(self, key, value): + if key.endswith(zarr_group_meta_key): - for f, t in zarr_to_n5_keys: - array_metadata[t] = array_metadata[f] - del array_metadata[f] - del array_metadata['zarr_format'] + key = key.replace(zarr_group_meta_key, self.group_meta_key) - try: - dtype = np.dtype(array_metadata['dataType']) - except TypeError: # pragma: no cover - raise TypeError( - "data type %s not supported by N5" % array_metadata['dataType']) + n5_attrs = self._load_n5_attrs(key) + n5_attrs.update(**group_metadata_to_n5(json_loads(value))) - array_metadata['dataType'] = dtype.name - array_metadata['dimensions'] = array_metadata['dimensions'][::-1] - array_metadata['blockSize'] = array_metadata['blockSize'][::-1] + value = json_dumps(n5_attrs) - if 'fill_value' in array_metadata: - if array_metadata['fill_value'] != 0 and array_metadata['fill_value'] is not None: - raise ValueError("N5 only supports fill_value == 0 (for now)") - del array_metadata['fill_value'] + elif key.endswith(zarr_array_meta_key): - if 'order' in array_metadata: - if array_metadata['order'] != 'C': - raise ValueError("zarr N5 storage only stores arrays in C order (for now)") - del array_metadata['order'] + key = key.replace(zarr_array_meta_key, self.array_meta_key) - if 'filters' in array_metadata: - if array_metadata['filters'] != [] and array_metadata['filters'] is not None: - raise ValueError("N5 storage does not support zarr filters") - del array_metadata['filters'] + n5_attrs = self._load_n5_attrs(key) + n5_attrs.update(**array_metadata_to_n5(json_loads(value))) - assert 'compression' in array_metadata - compressor_config = array_metadata['compression'] - compressor_config = compressor_config_to_n5(compressor_config) - array_metadata['compression'] = compressor_config + value = json_dumps(n5_attrs) - if 'dimension_separator' in array_metadata: - del array_metadata['dimension_separator'] + elif key.endswith(zarr_attrs_key): - return array_metadata + key = key.replace(zarr_attrs_key, self.attrs_key) + + n5_attrs = self._load_n5_attrs(key) + zarr_attrs = json_loads(value) + + for k in n5_keywords: + if k in zarr_attrs.keys(): + raise ValueError( + "Can not set attribute %s, this is a reserved N5 keyword" % k + ) + + # replace previous user attributes + for k in list(n5_attrs.keys()): + if k not in n5_keywords: + del n5_attrs[k] + + # add new user attributes + n5_attrs.update(**zarr_attrs) + + value = json_dumps(n5_attrs) + + super().__setitem__(key, value) + + def __delitem__(self, key): + + if key.endswith(zarr_group_meta_key): # pragma: no cover + key = key.replace(zarr_group_meta_key, self.group_meta_key) + elif key.endswith(zarr_array_meta_key): # pragma: no cover + key = key.replace(zarr_array_meta_key, self.array_meta_key) + elif key.endswith(zarr_attrs_key): # pragma: no cover + key = key.replace(zarr_attrs_key, self.attrs_key) + + super().__delitem__(key) + + def __contains__(self, key): + if key.endswith(zarr_group_meta_key): + + key = key.replace(zarr_group_meta_key, self.group_meta_key) + if key not in self: + return False + # group if not a dataset (attributes do not contain 'dimensions') + return "dimensions" not in self._load_n5_attrs(key) + + elif key.endswith(zarr_array_meta_key): + + key = key.replace(zarr_array_meta_key, self.array_meta_key) + # array if attributes contain 'dimensions' + return "dimensions" in self._load_n5_attrs(key) + + elif key.endswith(zarr_attrs_key): + + key = key.replace(zarr_attrs_key, self.attrs_key) + return self._contains_attrs(key) + + return super().__contains__(key) + + def __eq__(self, other): + return isinstance(other, N5FSStore) and self.path == other.path + + def listdir(self, path=None): + + if path is not None: + path = invert_chunk_coords(path) + + # We can't use NestedDirectoryStore's listdir, as it requires + # array_meta_key to be present in array directories, which this store + # doesn't provide. + children = super().listdir(path=path) + if self._is_array(path): + + # replace n5 attribute file with respective zarr attribute files + children.remove(self.array_meta_key) + children.append(zarr_array_meta_key) + if self._contains_attrs(path): + children.append(zarr_attrs_key) + + # special handling of directories containing an array to map + # inverted nested chunk keys back to standard chunk keys + new_children = [] + root_path = self.dir_path(path) + for entry in children: + entry_path = os.path.join(root_path, entry) + if _prog_number.match(entry) and self.fs.isdir(entry_path): + for dir_path, _, file_names in self.fs.walk(entry_path): + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + rel_path = file_path.split(root_path + os.path.sep)[1] + new_child = rel_path.replace(os.path.sep, ".") + new_children.append(invert_chunk_coords(new_child)) + else: + new_children.append(entry) + + return sorted(new_children) + + elif self._is_group(path): + + # replace n5 attribute file with respective zarr attribute files + children.remove(self.group_meta_key) + children.append(zarr_group_meta_key) + if self._contains_attrs(path): # pragma: no cover + children.append(zarr_attrs_key) + + return sorted(children) + + else: + + return children + + def _load_n5_attrs(self, path): + try: + s = super().__getitem__(path) + return json_loads(s) + except KeyError: + return {} + + def _is_group(self, path): + + if path is None: + attrs_key = self.attrs_key + else: + attrs_key = os.path.join(path, self.attrs_key) + + n5_attrs = self._load_n5_attrs(attrs_key) + return len(n5_attrs) > 0 and "dimensions" not in n5_attrs + + def _is_array(self, path): + + if path is None: + attrs_key = self.attrs_key + else: + attrs_key = os.path.join(path, self.attrs_key) + + return "dimensions" in self._load_n5_attrs(attrs_key) + + def _contains_attrs(self, path): + + if path is None: + attrs_key = self.attrs_key + else: + if not path.endswith(self.attrs_key): + attrs_key = os.path.join(path, self.attrs_key) + else: # pragma: no cover + attrs_key = path + + attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key)) + return len(attrs) > 0 + + def is_chunk_key(key): + segments = list(key.split('/')) + if segments: + last_segment = segments[-1] + return _prog_ckey.match(last_segment) + return False # pragma: no cover + + def invert_chunk_coords(key): + segments = list(key.split('/')) + if segments: + last_segment = segments[-1] + if _prog_ckey.match(last_segment): + coords = list(last_segment.split('.')) + last_segment = '.'.join(coords[::-1]) + segments = segments[:-1] + [last_segment] + key = '/'.join(segments) + return key + + def group_metadata_to_n5(group_metadata): + '''Convert group metadata from zarr to N5 format.''' + del group_metadata['zarr_format'] + # TODO: This should only exist at the top-level + group_metadata['n5'] = '2.0.0' + return group_metadata + + def group_metadata_to_zarr(group_metadata): + '''Convert group metadata from N5 to zarr format.''' + # This only exists at the top level + group_metadata.pop('n5', None) + group_metadata['zarr_format'] = ZARR_FORMAT + return group_metadata + + def array_metadata_to_n5(array_metadata): + '''Convert array metadata from zarr to N5 format.''' + + for f, t in zarr_to_n5_keys: + array_metadata[t] = array_metadata[f] + del array_metadata[f] + del array_metadata['zarr_format'] + + try: + dtype = np.dtype(array_metadata['dataType']) + except TypeError: # pragma: no cover + raise TypeError( + "data type %s not supported by N5" % array_metadata['dataType']) + + array_metadata['dataType'] = dtype.name + array_metadata['dimensions'] = array_metadata['dimensions'][::-1] + array_metadata['blockSize'] = array_metadata['blockSize'][::-1] + + if 'fill_value' in array_metadata: + if array_metadata['fill_value'] != 0 and array_metadata['fill_value'] is not None: + raise ValueError("N5 only supports fill_value == 0 (for now)") + del array_metadata['fill_value'] + + if 'order' in array_metadata: + if array_metadata['order'] != 'C': + raise ValueError("zarr N5 storage only stores arrays in C order (for now)") + del array_metadata['order'] + + if 'filters' in array_metadata: + if array_metadata['filters'] != [] and array_metadata['filters'] is not None: + raise ValueError("N5 storage does not support zarr filters") + del array_metadata['filters'] + + assert 'compression' in array_metadata + compressor_config = array_metadata['compression'] + compressor_config = compressor_config_to_n5(compressor_config) + array_metadata['compression'] = compressor_config + + if 'dimension_separator' in array_metadata: + del array_metadata['dimension_separator'] + + return array_metadata +except ImportError: + pass def array_metadata_to_zarr(array_metadata): diff --git a/zarr/storage.py b/zarr/storage.py index f858e42191..ebe512cd4b 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -948,12 +948,37 @@ def dir_path(self, path=None): return dir_path def listdir(self, path=None): + return self._dimension_separator == "/" and \ + self._nested_listdir(path) or self._flat_listdir(path) + + def _flat_listdir(self, path=None): dir_path = self.dir_path(path) if os.path.isdir(dir_path): return sorted(os.listdir(dir_path)) else: return [] + def _nested_listdir(self, path=None): + children = self._flat_listdir(path=path) + if array_meta_key in children: + # special handling of directories containing an array to map nested chunk + # keys back to standard chunk keys + new_children = [] + root_path = self.dir_path(path) + for entry in children: + entry_path = os.path.join(root_path, entry) + if _prog_number.match(entry) and os.path.isdir(entry_path): + for dir_path, _, file_names in os.walk(entry_path): + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + rel_path = file_path.split(root_path + os.path.sep)[1] + new_children.append(rel_path.replace(os.path.sep, '.')) + else: + new_children.append(entry) + return sorted(new_children) + else: + return children + def rename(self, src_path, dst_path): store_src_path = normalize_storage_path(src_path) store_dst_path = normalize_storage_path(dst_path) @@ -1040,22 +1065,28 @@ class FSStore(MutableMapping): Separator placed between the dimensions of a chunk. storage_options : passed to the fsspec implementation """ + array_meta_key = array_meta_key + group_meta_key = group_meta_key + attrs_key = attrs_key - _META_KEYS = (attrs_key, group_meta_key, array_meta_key) - - def __init__(self, url, normalize_keys=False, key_separator=None, + def __init__(self, url, normalize_keys=True, key_separator=None, mode='w', exceptions=(KeyError, PermissionError, IOError), dimension_separator=None, **storage_options): import fsspec self.normalize_keys = normalize_keys + + protocol, _ = fsspec.core.split_protocol(url) + # set auto_mkdir to True for local file system + if protocol in (None, "file") and not storage_options.get("auto_mkdir"): + storage_options["auto_mkdir"] = True + self.map = fsspec.get_mapper(url, **storage_options) self.fs = self.map.fs # for direct operations self.path = self.fs._strip_protocol(url) self.mode = mode self.exceptions = exceptions - # For backwards compatibility. Guaranteed to be non-None if key_separator is not None: dimension_separator = key_separator @@ -1066,7 +1097,6 @@ def __init__(self, url, normalize_keys=False, key_separator=None, # Pass attributes to array creation self._dimension_separator = dimension_separator - if self.fs.exists(self.path) and not self.fs.isdir(self.path): raise FSPathExistNotDir(url) @@ -1075,7 +1105,7 @@ def _normalize_key(self, key): if key: *bits, end = key.split('/') - if end not in FSStore._META_KEYS: + if end not in (self.array_meta_key, self.group_meta_key, self.attrs_key): end = end.replace('.', self.key_separator) key = '/'.join(bits + [end]) @@ -1153,7 +1183,7 @@ def listdir(self, path=None): if self.key_separator != "/": return children else: - if array_meta_key in children: + if self.array_meta_key in children: # special handling of directories containing an array to map nested chunk # keys back to standard chunk keys new_children = [] @@ -1315,49 +1345,12 @@ def __init__(self, path, normalize_keys=False, dimension_separator="/"): "NestedDirectoryStore only supports '/' as dimension_separator") self._dimension_separator = dimension_separator - def __getitem__(self, key): - key = _nested_map_ckey(key) - return super().__getitem__(key) - - def __setitem__(self, key, value): - key = _nested_map_ckey(key) - super().__setitem__(key, value) - - def __delitem__(self, key): - key = _nested_map_ckey(key) - super().__delitem__(key) - - def __contains__(self, key): - key = _nested_map_ckey(key) - return super().__contains__(key) - def __eq__(self, other): return ( isinstance(other, NestedDirectoryStore) and self.path == other.path ) - def listdir(self, path=None): - children = super().listdir(path=path) - if array_meta_key in children: - # special handling of directories containing an array to map nested chunk - # keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and os.path.isdir(entry_path): - for dir_path, _, file_names in os.walk(entry_path): - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path + os.path.sep)[1] - new_children.append(rel_path.replace(os.path.sep, '.')) - else: - new_children.append(entry) - return sorted(new_children) - else: - return children - # noinspection PyPep8Naming class ZipStore(MutableMapping): diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index b1346d760e..d329f4e58f 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -18,7 +18,7 @@ from zarr.core import Array from zarr.meta import json_loads -from zarr.n5 import N5Store, n5_keywords +from zarr.n5 import N5Store, N5FSStore, n5_keywords from zarr.storage import ( ABSStore, DBMStore, @@ -1963,6 +1963,22 @@ def test_hexdigest(self): assert self.expected() == found +@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +class TestArrayWithN5FSStore(TestArrayWithN5Store): + + @staticmethod + def create_array(read_only=False, **kwargs): + path = mkdtemp() + atexit.register(shutil.rmtree, path) + store = N5FSStore(path) + cache_metadata = kwargs.pop('cache_metadata', True) + cache_attrs = kwargs.pop('cache_attrs', True) + kwargs.setdefault('compressor', Zlib(1)) + init_array(store, **kwargs) + return Array(store, read_only=read_only, cache_metadata=cache_metadata, + cache_attrs=cache_attrs) + + class TestArrayWithDBMStore(TestArray): @staticmethod diff --git a/zarr/tests/test_dim_separator.py b/zarr/tests/test_dim_separator.py new file mode 100644 index 0000000000..281d0c1504 --- /dev/null +++ b/zarr/tests/test_dim_separator.py @@ -0,0 +1,75 @@ +import pytest +from numpy.testing import assert_array_equal + +import zarr +from zarr.core import Array +from zarr.storage import (DirectoryStore, NestedDirectoryStore, FSStore) +from zarr.tests.util import have_fsspec + + +@pytest.fixture(params=("static_nested", + "static_flat", + "directory_nested", + "directory_flat", + "directory_default", + "nesteddirectory_nested", + "nesteddirectory_default", + "fs_nested", + "fs_flat", + "fs_default")) +def dataset(tmpdir, request): + """ + Generate a variety of different Zarrs using + different store implementations as well as + different dimension_separator arguments. + """ + + loc = tmpdir.join("dim_sep_test.zarr") + which = request.param + kwargs = {} + + if which.startswith("static"): + if which.endswith("nested"): + return "fixture/nested" + else: + return "fixture/flat" + + if which.startswith("directory"): + store_class = DirectoryStore + elif which.startswith("nested"): + store_class = NestedDirectoryStore + else: + if have_fsspec is False: + pytest.skip("no fsspec") + store_class = FSStore + kwargs["mode"] = "w" + kwargs["auto_mkdir"] = True + + if which.endswith("nested"): + kwargs["dimension_separator"] = "/" + elif which.endswith("flat"): + kwargs["dimension_separator"] = "." + + store = store_class(str(loc), **kwargs) + zarr.creation.array(store=store, data=[[1, 2], [3, 4]]) + return str(loc) + + +def verify(array): + assert_array_equal(array[:], [[1, 2], [3, 4]]) + + +def test_open(dataset): + verify(zarr.open(dataset)) + + +def test_fsstore(dataset): + verify(Array(store=FSStore(dataset))) + + +def test_directory(dataset): + verify(zarr.Array(store=DirectoryStore(dataset))) + + +def test_nested(dataset): + verify(Array(store=NestedDirectoryStore(dataset))) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index d3f3b0e770..38743ee35c 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -23,7 +23,7 @@ from zarr.meta import (ZARR_FORMAT, decode_array_metadata, decode_group_metadata, encode_array_metadata, encode_group_metadata) -from zarr.n5 import N5Store +from zarr.n5 import N5Store, N5FSStore from zarr.storage import (ABSStore, ConsolidatedMetadataStore, DBMStore, DictStore, DirectoryStore, LMDBStore, LRUStoreCache, MemoryStore, MongoDBStore, NestedDirectoryStore, @@ -804,12 +804,16 @@ def test_pickle(self): class TestDirectoryStore(StoreTests): - def create_store(self, normalize_keys=False, **kwargs): - skip_if_nested_chunks(**kwargs) - + def create_store(self, + normalize_keys=False, + dimension_separator=".", + **kwargs): path = tempfile.mkdtemp() atexit.register(atexit_rmtree, path) - store = DirectoryStore(path, normalize_keys=normalize_keys, **kwargs) + store = DirectoryStore(path, + normalize_keys=normalize_keys, + dimension_separator=dimension_separator, + **kwargs) return store def test_filesystem_path(self): @@ -1150,10 +1154,10 @@ def test_chunk_nesting(self): # any path where last segment looks like a chunk key gets special handling store['0.0'] = b'xxx' assert b'xxx' == store['0.0'] - assert b'xxx' == store['0/0'] + # assert b'xxx' == store['0/0'] store['foo/10.20.30'] = b'yyy' assert b'yyy' == store['foo/10.20.30'] - assert b'yyy' == store['foo/10/20/30'] + # assert b'yyy' == store['foo/10/20/30'] store['42'] = b'zzz' assert b'zzz' == store['42'] @@ -1183,7 +1187,7 @@ def test_value_error(self): class TestN5Store(TestNestedDirectoryStore): def create_store(self, normalize_keys=False): - path = tempfile.mkdtemp(suffix='.n5') + path = tempfile.mkdtemp() atexit.register(atexit_rmtree, path) store = N5Store(path, normalize_keys=normalize_keys) return store @@ -1198,12 +1202,12 @@ def test_chunk_nesting(self): store['0.0'] = b'xxx' assert '0.0' in store assert b'xxx' == store['0.0'] - assert b'xxx' == store['0/0'] + # assert b'xxx' == store['0/0'] store['foo/10.20.30'] = b'yyy' assert 'foo/10.20.30' in store assert b'yyy' == store['foo/10.20.30'] # N5 reverses axis order - assert b'yyy' == store['foo/30/20/10'] + # assert b'yyy' == store['foo/30/20/10'] store['42'] = b'zzz' assert '42' in store assert b'zzz' == store['42'] @@ -1293,6 +1297,86 @@ def test_filters(self): init_array(store, shape=1000, chunks=100, filters=filters) +@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +class TestN5FSStore(TestFSStore): + def create_store(self, normalize_keys=False): + path = tempfile.mkdtemp() + atexit.register(atexit_rmtree, path) + store = N5FSStore(path, normalize_keys=normalize_keys) + return store + + def test_equal(self): + store_a = self.create_store() + store_b = N5FSStore(store_a.path) + assert store_a == store_b + + def test_init_array(self): + store = self.create_store() + init_array(store, shape=1000, chunks=100) + + # check metadata + assert array_meta_key in store + meta = decode_array_metadata(store[array_meta_key]) + assert ZARR_FORMAT == meta['zarr_format'] + assert (1000,) == meta['shape'] + assert (100,) == meta['chunks'] + assert np.dtype(None) == meta['dtype'] + # N5Store wraps the actual compressor + compressor_config = meta['compressor']['compressor_config'] + assert default_compressor.get_config() == compressor_config + # N5Store always has a fill value of 0 + assert meta['fill_value'] == 0 + + def test_init_array_path(self): + path = 'foo/bar' + store = self.create_store() + init_array(store, shape=1000, chunks=100, path=path) + + # check metadata + key = path + '/' + array_meta_key + assert key in store + meta = decode_array_metadata(store[key]) + assert ZARR_FORMAT == meta['zarr_format'] + assert (1000,) == meta['shape'] + assert (100,) == meta['chunks'] + assert np.dtype(None) == meta['dtype'] + # N5Store wraps the actual compressor + compressor_config = meta['compressor']['compressor_config'] + assert default_compressor.get_config() == compressor_config + # N5Store always has a fill value of 0 + assert meta['fill_value'] == 0 + + def test_init_array_compat(self): + store = self.create_store() + init_array(store, shape=1000, chunks=100, compressor='none') + meta = decode_array_metadata(store[array_meta_key]) + # N5Store wraps the actual compressor + compressor_config = meta['compressor']['compressor_config'] + assert compressor_config is None + + def test_init_array_overwrite(self): + self._test_init_array_overwrite('C') + + def test_init_array_overwrite_path(self): + self._test_init_array_overwrite_path('C') + + def test_init_array_overwrite_chunk_store(self): + self._test_init_array_overwrite_chunk_store('C') + + def test_init_group_overwrite(self): + self._test_init_group_overwrite('C') + + def test_init_group_overwrite_path(self): + self._test_init_group_overwrite_path('C') + + def test_init_group_overwrite_chunk_store(self): + self._test_init_group_overwrite_chunk_store('C') + + def test_dimension_separator(self): + with pytest.raises(TypeError): + self.create_store(key_separator='.') + + @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestNestedFSStore(TestNestedDirectoryStore):