Zarr to irirs Cube: add tests and Zarr sample data

.github/workflows/ci-tests.yml

@@ -11,7 +11,6 @@ on:
   push:
     branches:
       - "main"
-      - "v*x"
     tags:
       - "v*"
   pull_request:
@@ -49,7 +48,7 @@ jobs:
 
       - name: "Install dependencies"
         run: |
-          conda install --yes numpy pytest pytest-mock iris xarray filelock requests
+          conda install --yes numpy pytest pytest-mock iris xarray filelock requests zarr aiohttp fsspec
 
       - name: "Install *latest* Iris"
         run: |

lib/ncdata/utils/_compare_nc_datasets.py

@@ -14,7 +14,6 @@
 import netCDF4
 import netCDF4 as nc
 import numpy as np
-
 from ncdata import NcData, NcVariable
 
 

lib/ncdata/utils/_save_errors.py

@@ -3,7 +3,6 @@
 
 import netCDF4 as nc
 import numpy as np
-
 from ncdata import NcData, NcVariable
 
 

tests/integration/equivalence_testing_utils.py

@@ -5,11 +5,10 @@
 ncdata and other types of data preserve information.
 """
 import dask.array as da
+import iris.mesh
 import numpy as np
 import pytest
 
-import iris.mesh
-
 
 def cubes_equal__corrected(c1, c2):
     """

tests/integration/example_scripts/ex_dataset_print.py

@@ -1,8 +1,8 @@
 """Temporary integrational proof-of-concept example for dataset printout."""
 import iris
-
 import ncdata.iris as nci
 from ncdata import NcData, NcDimension, NcVariable
+
 from tests import testdata_dir
 
 

tests/integration/example_scripts/ex_iris_saveto_ncdata.py

@@ -4,8 +4,8 @@
 Check that conversion succeeds and print the resulting dataset.
 """
 import iris
-
 from ncdata.iris import from_iris
+
 from tests import testdata_dir
 
 

tests/integration/example_scripts/ex_iris_xarray_conversion.py

@@ -7,8 +7,8 @@
 import iris
 import numpy as np
 import xarray as xr
-
 from ncdata.iris_xarray import cubes_from_xarray, cubes_to_xarray
+
 from tests import testdata_dir
 
 

tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py

@@ -9,10 +9,10 @@
 
 import netCDF4 as nc
 import numpy as np
-
 from ncdata import NcData, NcDimension, NcVariable
 from ncdata.netcdf4 import from_nc4, to_nc4
 from ncdata.utils import dataset_differences
+
 from tests import testdata_dir
 
 

tests/integration/test_iris_load_and_save_equivalence.py

@@ -9,9 +9,9 @@
 
 import iris
 import pytest
-
 from ncdata.netcdf4 import from_nc4, to_nc4
 from ncdata.utils import dataset_differences
+
 from tests.data_testcase_schemas import session_testdir, standard_testcase
 from tests.integration.equivalence_testing_utils import (
     adjust_chunks,

tests/integration/test_iris_xarray_roundtrips.py

@@ -13,13 +13,13 @@
 import numpy as np
 import pytest
 import xarray
-
 from ncdata.iris import from_iris
 from ncdata.iris_xarray import cubes_to_xarray
 from ncdata.netcdf4 import from_nc4
 from ncdata.threadlock_sharing import lockshare_context
 from ncdata.utils import dataset_differences
 from ncdata.xarray import from_xarray
+
 from tests.data_testcase_schemas import (
     BAD_LOADSAVE_TESTCASES,
     session_testdir,

tests/integration/test_netcdf_roundtrips.py

@@ -5,6 +5,7 @@
 
 from ncdata.netcdf4 import from_nc4, to_nc4
 from ncdata.utils import dataset_differences
+
 from tests.data_testcase_schemas import session_testdir, standard_testcase
 
 # Avoid complaints that the imported fixtures are "unused"

tests/integration/test_xarray_load_and_save_equivalence.py

@@ -7,11 +7,11 @@
 """
 import pytest
 import xarray
-
 from ncdata.netcdf4 import from_nc4, to_nc4
 from ncdata.threadlock_sharing import lockshare_context
 from ncdata.utils import dataset_differences
 from ncdata.xarray import from_xarray, to_xarray
+
 from tests.data_testcase_schemas import (
     BAD_LOADSAVE_TESTCASES,
     session_testdir,

tests/integration/test_zarr_to_iris.py

@@ -0,0 +1,111 @@
+"""Test conversion of remote and local Zarr store to iris Cube."""
+
+from importlib.resources import files as importlib_files
+from pathlib import Path
+
+import fsspec
+import iris
+import pytest
+import xarray as xr
+from ncdata.iris_xarray import cubes_from_xarray as conversion_func
+
+
+def _return_kwargs():
+    time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+    return {
+        "consolidated": True,
+        "decode_times": time_coder,
+        "engine": "zarr",
+        "chunks": {},
+        "backend_kwargs": {},
+    }
+
+
+def _run_checks(cube):
+    """Run some standard checks."""
+    assert cube.var_name == "q"
+    assert cube.standard_name == "specific_humidity"
+    assert cube.long_name is None
+    coords = cube.coords()
+    coord_names = [coord.standard_name for coord in coords]
+    assert "longitude" in coord_names
+    assert "latitude" in coord_names
+
+
+def test_load_zarr2_local():
+    """Test loading a Zarr2 store from local FS."""
+    zarr_path = (
+        Path(importlib_files("tests"))
+        / "testdata"
+        / "zarr-sample-data"
+        / "example_field_0.zarr2"
+    )
+
+    xr_kwargs = _return_kwargs()
+    zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs)
+
+    cubes = conversion_func(zarr_xr)
+
+    assert len(cubes) == 1
+    cube = cubes[0]
+    _run_checks(cube)
+
+
+def test_load_zarr3_local():
+    """Test loading a Zarr3 store from local FS."""
+    zarr_path = (
+        Path(importlib_files("tests"))
+        / "testdata"
+        / "zarr-sample-data"
+        / "example_field_0.zarr3"
+    )
+
+    xr_kwargs = _return_kwargs()
+    zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs)
+
+    cubes = conversion_func(zarr_xr)
+
+    assert len(cubes) == 1
+    cube = cubes[0]
+    _run_checks(cube)
+
+
+def _is_url_ok(url):
+    fs = fsspec.filesystem("http")
+    valid_zarr = True
+    try:
+        fs.open(str(url) + "/zarr.json", "rb")  # Zarr3
+    except Exception:  # noqa: BLE001
+        try:
+            fs.open(str(url) + "/.zmetadata", "rb")  # Zarr2
+        except Exception:  # noqa: BLE001
+            valid_zarr = False
+
+    return valid_zarr
+
+
+S3_TEST_PATH = (
+    "https://uor-aces-o.s3-ext.jc.rl.ac.uk/"
+    "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3"
+)
+_S3_accessible = _is_url_ok(S3_TEST_PATH)
+
+
+@pytest.mark.skipif(not _S3_accessible, reason="S3 url not accessible")
+def test_load_remote_zarr():
+    """Test loading a remote Zarr store.
+
+    This is a ~250MB compressed Zarr in an S3 bucket.
+    Conversion is done fully lazily, by passing chunks={}
+    to Xarray loader. Test takes ~3-4s and needs ~400MB res mem.
+    """
+    zarr_path = S3_TEST_PATH
+
+    xr_kwargs = _return_kwargs()
+    zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs)
+
+    cubes = conversion_func(zarr_xr)
+
+    assert isinstance(cubes, iris.cube.CubeList)
+    assert len(cubes) == 1
+    assert cubes[0].has_lazy_data()

tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs

@@ -0,0 +1,3 @@
+{
+    "Conventions": "CF-1.12"
+}

tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup

@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}