From 16d5f71d7288491258435eeb059a9c7f4430a57c Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Sun, 30 Oct 2022 16:06:45 -0400 Subject: [PATCH 1/2] Make sure fs exceptions are raised if not Missing --- zarr/storage.py | 7 ++++++- zarr/tests/test_storage.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/zarr/storage.py b/zarr/storage.py index f5459990ba..b1b671d964 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1365,9 +1365,14 @@ def _normalize_key(self, key): def getitems(self, keys, **kwargs): keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") + results = self.map.getitems(keys_transformed, on_error="return") # The function calling this method may not recognize the transformed keys # So we send the values returned by self.map.getitems back into the original key space. + for k, v in results.copy().items(): + if isinstance(v, self.exceptions): + results.pop(k) + elif isinstance(v, Exception): + raise v return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} def __getitem__(self, key): diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 39d4b5988d..5527974eda 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1321,6 +1321,21 @@ def test_s3_complex(self): ) assert (a[:] == -np.ones((8, 8, 8))).all() + def test_exceptions(self): + import fsspec + m = fsspec.filesystem("memory") + g = zarr.open_group("memory://test/out.zarr", mode='w') + arr = g.create_dataset("data", data=[1, 2, 3, 4], + dtype="i4", compression=None, chunks=[2]) + m.store["/test/out.zarr/data/0"] = None + del m.store["/test/out.zarr/data/1"] + assert g.store.getitems(["data/1"]) == {} # not found + with pytest.raises(Exception): + # None is bad daa, as opposed to missing + g.store.getitems(["data/0", "data/1"]) + with pytest.raises(Exception): + # None is bad daa, as opposed to missing + arr[:] @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStoreWithKeySeparator(StoreTests): From e8f883364be4c4f3c7bb0a24439b7cca8402e1e0 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Mon, 21 Nov 2022 15:23:20 -0500 Subject: [PATCH 2/2] lint --- zarr/tests/test_storage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 5527974eda..08e9e816eb 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1331,12 +1331,13 @@ def test_exceptions(self): del m.store["/test/out.zarr/data/1"] assert g.store.getitems(["data/1"]) == {} # not found with pytest.raises(Exception): - # None is bad daa, as opposed to missing + # None is bad data, as opposed to missing g.store.getitems(["data/0", "data/1"]) with pytest.raises(Exception): - # None is bad daa, as opposed to missing + # None is bad data, as opposed to missing arr[:] + @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStoreWithKeySeparator(StoreTests):