diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c01b881f..a30842a2 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -11,7 +11,6 @@ on: push: branches: - "main" - - "v*x" tags: - "v*" pull_request: @@ -49,7 +48,7 @@ jobs: - name: "Install dependencies" run: | - conda install --yes numpy pytest pytest-mock iris xarray filelock requests + conda install --yes numpy pytest pytest-mock iris xarray filelock requests zarr aiohttp fsspec - name: "Install *latest* Iris" run: | diff --git a/docs/changelog_fragments/145.dev.rst b/docs/changelog_fragments/145.dev.rst new file mode 100644 index 00000000..02433cbb --- /dev/null +++ b/docs/changelog_fragments/145.dev.rst @@ -0,0 +1 @@ +@valeriupredoi added test for Zarr conversion to Iris cubes. diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py new file mode 100644 index 00000000..5b8dc33b --- /dev/null +++ b/tests/integration/test_zarr_to_iris.py @@ -0,0 +1,111 @@ +"""Test conversion of remote and local Zarr store to iris Cube.""" + +from importlib.resources import files as importlib_files +from pathlib import Path + +import fsspec +import iris +import pytest +import xarray as xr +from ncdata.iris_xarray import cubes_from_xarray as conversion_func + + +def _return_kwargs(): + time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) + return { + "consolidated": True, + "decode_times": time_coder, + "engine": "zarr", + "chunks": {}, + "backend_kwargs": {}, + } + + +def _run_checks(cube): + """Run some standard checks.""" + assert cube.var_name == "q" + assert cube.standard_name == "specific_humidity" + assert cube.long_name is None + coords = cube.coords() + coord_names = [coord.standard_name for coord in coords] + assert "longitude" in coord_names + assert "latitude" in coord_names + + +def test_load_zarr2_local(): + """Test loading a Zarr2 store from local FS.""" + zarr_path = ( + Path(importlib_files("tests")) + / "testdata" + / "zarr-sample-data" + / "example_field_0.zarr2" + ) + + xr_kwargs = _return_kwargs() + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + + cubes = conversion_func(zarr_xr) + + assert len(cubes) == 1 + cube = cubes[0] + _run_checks(cube) + + +def test_load_zarr3_local(): + """Test loading a Zarr3 store from local FS.""" + zarr_path = ( + Path(importlib_files("tests")) + / "testdata" + / "zarr-sample-data" + / "example_field_0.zarr3" + ) + + xr_kwargs = _return_kwargs() + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + + cubes = conversion_func(zarr_xr) + + assert len(cubes) == 1 + cube = cubes[0] + _run_checks(cube) + + +def _is_url_ok(url): + fs = fsspec.filesystem("http") + valid_zarr = True + try: + fs.open(str(url) + "/zarr.json", "rb") # Zarr3 + except Exception: # noqa: BLE001 + try: + fs.open(str(url) + "/.zmetadata", "rb") # Zarr2 + except Exception: # noqa: BLE001 + valid_zarr = False + + return valid_zarr + + +S3_TEST_PATH = ( + "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" + "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" +) +_S3_accessible = _is_url_ok(S3_TEST_PATH) + + +@pytest.mark.skipif(not _S3_accessible, reason="S3 url not accessible") +def test_load_remote_zarr(): + """Test loading a remote Zarr store. + + This is a ~250MB compressed Zarr in an S3 bucket. + Conversion is done fully lazily, by passing chunks={} + to Xarray loader. Test takes ~3-4s and needs ~400MB res mem. + """ + zarr_path = S3_TEST_PATH + + xr_kwargs = _return_kwargs() + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + + cubes = conversion_func(zarr_xr) + + assert isinstance(cubes, iris.cube.CubeList) + assert len(cubes) == 1 + assert cubes[0].has_lazy_data() diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs new file mode 100644 index 00000000..bb815dea --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs @@ -0,0 +1,3 @@ +{ + "Conventions": "CF-1.12" +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup new file mode 100644 index 00000000..3f3fad2d --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata new file mode 100644 index 00000000..ab417b34 --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata @@ -0,0 +1,171 @@ +{ + "metadata": { + ".zattrs": { + "Conventions": "CF-1.12" + }, + ".zgroup": { + "zarr_format": 2 + }, + "lat/.zarray": { + "chunks": [ + 5 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "