diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c01b881f..a30842a2 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -11,7 +11,6 @@ on: push: branches: - "main" - - "v*x" tags: - "v*" pull_request: @@ -49,7 +48,7 @@ jobs: - name: "Install dependencies" run: | - conda install --yes numpy pytest pytest-mock iris xarray filelock requests + conda install --yes numpy pytest pytest-mock iris xarray filelock requests zarr aiohttp fsspec - name: "Install *latest* Iris" run: | diff --git a/lib/ncdata/utils/_compare_nc_datasets.py b/lib/ncdata/utils/_compare_nc_datasets.py index 8c7053db..b9892a39 100644 --- a/lib/ncdata/utils/_compare_nc_datasets.py +++ b/lib/ncdata/utils/_compare_nc_datasets.py @@ -14,7 +14,6 @@ import netCDF4 import netCDF4 as nc import numpy as np - from ncdata import NcData, NcVariable diff --git a/lib/ncdata/utils/_save_errors.py b/lib/ncdata/utils/_save_errors.py index aa2e7228..f8cfcaf8 100644 --- a/lib/ncdata/utils/_save_errors.py +++ b/lib/ncdata/utils/_save_errors.py @@ -3,7 +3,6 @@ import netCDF4 as nc import numpy as np - from ncdata import NcData, NcVariable diff --git a/tests/integration/equivalence_testing_utils.py b/tests/integration/equivalence_testing_utils.py index 8693bc6d..87a11dca 100644 --- a/tests/integration/equivalence_testing_utils.py +++ b/tests/integration/equivalence_testing_utils.py @@ -5,11 +5,10 @@ ncdata and other types of data preserve information. """ import dask.array as da +import iris.mesh import numpy as np import pytest -import iris.mesh - def cubes_equal__corrected(c1, c2): """ diff --git a/tests/integration/example_scripts/ex_dataset_print.py b/tests/integration/example_scripts/ex_dataset_print.py index 5f3251fb..83ea63eb 100644 --- a/tests/integration/example_scripts/ex_dataset_print.py +++ b/tests/integration/example_scripts/ex_dataset_print.py @@ -1,8 +1,8 @@ """Temporary integrational proof-of-concept example for dataset printout.""" import iris - import ncdata.iris as nci from ncdata import NcData, NcDimension, NcVariable + from tests import testdata_dir diff --git a/tests/integration/example_scripts/ex_iris_saveto_ncdata.py b/tests/integration/example_scripts/ex_iris_saveto_ncdata.py index c4a3c14b..ec24eeb2 100644 --- a/tests/integration/example_scripts/ex_iris_saveto_ncdata.py +++ b/tests/integration/example_scripts/ex_iris_saveto_ncdata.py @@ -4,8 +4,8 @@ Check that conversion succeeds and print the resulting dataset. """ import iris - from ncdata.iris import from_iris + from tests import testdata_dir diff --git a/tests/integration/example_scripts/ex_iris_xarray_conversion.py b/tests/integration/example_scripts/ex_iris_xarray_conversion.py index 018c1548..361a9c08 100644 --- a/tests/integration/example_scripts/ex_iris_xarray_conversion.py +++ b/tests/integration/example_scripts/ex_iris_xarray_conversion.py @@ -7,8 +7,8 @@ import iris import numpy as np import xarray as xr - from ncdata.iris_xarray import cubes_from_xarray, cubes_to_xarray + from tests import testdata_dir diff --git a/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py b/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py index 7f2e614e..becc646c 100644 --- a/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py +++ b/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py @@ -9,10 +9,10 @@ import netCDF4 as nc import numpy as np - from ncdata import NcData, NcDimension, NcVariable from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences + from tests import testdata_dir diff --git a/tests/integration/test_iris_load_and_save_equivalence.py b/tests/integration/test_iris_load_and_save_equivalence.py index 7ad3eb95..5117fca5 100644 --- a/tests/integration/test_iris_load_and_save_equivalence.py +++ b/tests/integration/test_iris_load_and_save_equivalence.py @@ -9,9 +9,9 @@ import iris import pytest - from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences + from tests.data_testcase_schemas import session_testdir, standard_testcase from tests.integration.equivalence_testing_utils import ( adjust_chunks, diff --git a/tests/integration/test_iris_xarray_roundtrips.py b/tests/integration/test_iris_xarray_roundtrips.py index 9ece022b..7cfeabac 100644 --- a/tests/integration/test_iris_xarray_roundtrips.py +++ b/tests/integration/test_iris_xarray_roundtrips.py @@ -13,13 +13,13 @@ import numpy as np import pytest import xarray - from ncdata.iris import from_iris from ncdata.iris_xarray import cubes_to_xarray from ncdata.netcdf4 import from_nc4 from ncdata.threadlock_sharing import lockshare_context from ncdata.utils import dataset_differences from ncdata.xarray import from_xarray + from tests.data_testcase_schemas import ( BAD_LOADSAVE_TESTCASES, session_testdir, diff --git a/tests/integration/test_netcdf_roundtrips.py b/tests/integration/test_netcdf_roundtrips.py index 6fe635d5..db9ea467 100644 --- a/tests/integration/test_netcdf_roundtrips.py +++ b/tests/integration/test_netcdf_roundtrips.py @@ -5,6 +5,7 @@ from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences + from tests.data_testcase_schemas import session_testdir, standard_testcase # Avoid complaints that the imported fixtures are "unused" diff --git a/tests/integration/test_xarray_load_and_save_equivalence.py b/tests/integration/test_xarray_load_and_save_equivalence.py index d7fb3164..f36eb06b 100644 --- a/tests/integration/test_xarray_load_and_save_equivalence.py +++ b/tests/integration/test_xarray_load_and_save_equivalence.py @@ -7,11 +7,11 @@ """ import pytest import xarray - from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.threadlock_sharing import lockshare_context from ncdata.utils import dataset_differences from ncdata.xarray import from_xarray, to_xarray + from tests.data_testcase_schemas import ( BAD_LOADSAVE_TESTCASES, session_testdir, diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py new file mode 100644 index 00000000..5b8dc33b --- /dev/null +++ b/tests/integration/test_zarr_to_iris.py @@ -0,0 +1,111 @@ +"""Test conversion of remote and local Zarr store to iris Cube.""" + +from importlib.resources import files as importlib_files +from pathlib import Path + +import fsspec +import iris +import pytest +import xarray as xr +from ncdata.iris_xarray import cubes_from_xarray as conversion_func + + +def _return_kwargs(): + time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) + return { + "consolidated": True, + "decode_times": time_coder, + "engine": "zarr", + "chunks": {}, + "backend_kwargs": {}, + } + + +def _run_checks(cube): + """Run some standard checks.""" + assert cube.var_name == "q" + assert cube.standard_name == "specific_humidity" + assert cube.long_name is None + coords = cube.coords() + coord_names = [coord.standard_name for coord in coords] + assert "longitude" in coord_names + assert "latitude" in coord_names + + +def test_load_zarr2_local(): + """Test loading a Zarr2 store from local FS.""" + zarr_path = ( + Path(importlib_files("tests")) + / "testdata" + / "zarr-sample-data" + / "example_field_0.zarr2" + ) + + xr_kwargs = _return_kwargs() + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + + cubes = conversion_func(zarr_xr) + + assert len(cubes) == 1 + cube = cubes[0] + _run_checks(cube) + + +def test_load_zarr3_local(): + """Test loading a Zarr3 store from local FS.""" + zarr_path = ( + Path(importlib_files("tests")) + / "testdata" + / "zarr-sample-data" + / "example_field_0.zarr3" + ) + + xr_kwargs = _return_kwargs() + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + + cubes = conversion_func(zarr_xr) + + assert len(cubes) == 1 + cube = cubes[0] + _run_checks(cube) + + +def _is_url_ok(url): + fs = fsspec.filesystem("http") + valid_zarr = True + try: + fs.open(str(url) + "/zarr.json", "rb") # Zarr3 + except Exception: # noqa: BLE001 + try: + fs.open(str(url) + "/.zmetadata", "rb") # Zarr2 + except Exception: # noqa: BLE001 + valid_zarr = False + + return valid_zarr + + +S3_TEST_PATH = ( + "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" + "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" +) +_S3_accessible = _is_url_ok(S3_TEST_PATH) + + +@pytest.mark.skipif(not _S3_accessible, reason="S3 url not accessible") +def test_load_remote_zarr(): + """Test loading a remote Zarr store. + + This is a ~250MB compressed Zarr in an S3 bucket. + Conversion is done fully lazily, by passing chunks={} + to Xarray loader. Test takes ~3-4s and needs ~400MB res mem. + """ + zarr_path = S3_TEST_PATH + + xr_kwargs = _return_kwargs() + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + + cubes = conversion_func(zarr_xr) + + assert isinstance(cubes, iris.cube.CubeList) + assert len(cubes) == 1 + assert cubes[0].has_lazy_data() diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs new file mode 100644 index 00000000..bb815dea --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs @@ -0,0 +1,3 @@ +{ + "Conventions": "CF-1.12" +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup new file mode 100644 index 00000000..3f3fad2d --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata new file mode 100644 index 00000000..ab417b34 --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata @@ -0,0 +1,171 @@ +{ + "metadata": { + ".zattrs": { + "Conventions": "CF-1.12" + }, + ".zgroup": { + "zarr_format": 2 + }, + "lat/.zarray": { + "chunks": [ + 5 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "