From eed755f02ad4e75c1ecbfde19e3772d6c8706a98 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 15:21:22 +0100 Subject: [PATCH 01/37] add a user warning when data is not lazy --- lib/ncdata/xarray.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index cf92ce74..5178ff4f 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -9,6 +9,7 @@ # Hopefully a minimal amount. # The structure of an NcData object makes it fairly painless. # +import warnings from pathlib import Path from typing import AnyStr, Union @@ -96,6 +97,14 @@ def store( # Install variables, creating dimensions as we go. for varname, var in new_variables.items(): + if isinstance(var.data, np.ndarray) and \ + var.attrs["axis"] not in ["X", "Y", "Z", "T"]: + warn_msg = ( + f"Variable {var} has fully realized " + "data, if you need lazy data, then add " + "chunks={} as argument to Xarray open_dataset." + ) + warnings.warn(warn_msg, UserWarning, stacklevel=2) if varname in self.ncdata.variables: raise ValueError(f'duplicate variable : "{varname}"') From f13b64214a97de9ea6dbaabb046bf3081145f565 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 15:21:42 +0100 Subject: [PATCH 02/37] add a test module for zarrs --- tests/integration/test_zarr_to_iris.py | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/integration/test_zarr_to_iris.py diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py new file mode 100644 index 00000000..0b18dca7 --- /dev/null +++ b/tests/integration/test_zarr_to_iris.py @@ -0,0 +1,37 @@ +"""Test conversion of remote and local Zarr store to iris Cube.""" +import iris +import xarray as xr +import ncdata +import ncdata.iris_xarray +import zarr + + +def test_load_remote_zarr(): + """Test loading a remote Zarr store. + + This is a ~250MB compressed Zarr in an S3 bucket. + Conversion is done fully lazily, by passing chunks={} + to Xarray loader. Test takes ~3-4s and needs ~400MB res mem. + """ + zarr_path = ( + "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" + "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" + ) + + time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) + zarr_xr = xr.open_dataset( + zarr_path, + consolidated=True, + decode_times=time_coder, + engine="zarr", + chunks={}, + backend_kwargs={}, + ) + zarr_xr.unify_chunks() + + conversion_func = ncdata.iris_xarray.cubes_from_xarray + cubes = conversion_func(zarr_xr) + + assert isinstance(cubes, iris.cube.CubeList) + assert len(cubes) == 1 + assert cubes[0].has_lazy_data() From 3f7cf4a80d73496c4a6eac07bd50b30346528ae7 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 15:22:59 +0100 Subject: [PATCH 03/37] add tiny Zarr sample data --- tests/zarr-sample-data/example_field_0.zarr17 | 1 + .../example_field_0.zarr2/.zattrs | 3 + .../example_field_0.zarr2/.zgroup | 3 + .../example_field_0.zarr2/.zmetadata | 171 ++++++++++++++++++ .../example_field_0.zarr2/lat/.zarray | 20 ++ .../example_field_0.zarr2/lat/.zattrs | 8 + .../example_field_0.zarr2/lat/0 | Bin 0 -> 56 bytes .../example_field_0.zarr2/lat_bnds/.zarray | 22 +++ .../example_field_0.zarr2/lat_bnds/.zattrs | 6 + .../example_field_0.zarr2/lat_bnds/0.0 | Bin 0 -> 64 bytes .../example_field_0.zarr2/lat_bnds/1.0 | Bin 0 -> 64 bytes .../example_field_0.zarr2/lon/.zarray | 20 ++ .../example_field_0.zarr2/lon/.zattrs | 8 + .../example_field_0.zarr2/lon/0 | Bin 0 -> 80 bytes .../example_field_0.zarr2/lon_bnds/.zarray | 22 +++ .../example_field_0.zarr2/lon_bnds/.zattrs | 6 + .../example_field_0.zarr2/lon_bnds/0.0 | Bin 0 -> 80 bytes .../example_field_0.zarr2/lon_bnds/1.0 | Bin 0 -> 80 bytes .../example_field_0.zarr2/q/.zarray | 22 +++ .../example_field_0.zarr2/q/.zattrs | 11 ++ .../example_field_0.zarr2/q/0.0 | Bin 0 -> 112 bytes .../example_field_0.zarr2/q/0.1 | Bin 0 -> 112 bytes .../example_field_0.zarr2/q/1.0 | Bin 0 -> 112 bytes .../example_field_0.zarr2/q/1.1 | Bin 0 -> 112 bytes .../example_field_0.zarr2/time/.zarray | 10 + .../example_field_0.zarr2/time/.zattrs | 5 + .../example_field_0.zarr2/time/0 | Bin 0 -> 8 bytes 27 files changed, 338 insertions(+) create mode 100644 tests/zarr-sample-data/example_field_0.zarr17 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/.zattrs create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/.zgroup create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/.zmetadata create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat/.zarray create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat/.zattrs create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat/0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zarray create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zattrs create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/0.0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/1.0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon/.zarray create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon/.zattrs create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon/0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/.zarray create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/.zattrs create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/0.0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/.zarray create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/.zattrs create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/0.0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/0.1 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/1.0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/1.1 create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/time/.zarray create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/time/.zattrs create mode 100644 tests/zarr-sample-data/example_field_0.zarr2/time/0 diff --git a/tests/zarr-sample-data/example_field_0.zarr17 b/tests/zarr-sample-data/example_field_0.zarr17 new file mode 100644 index 00000000..9abbe8a4 --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr17 @@ -0,0 +1 @@ +This is not a Zarr file. Go grab lunch! diff --git a/tests/zarr-sample-data/example_field_0.zarr2/.zattrs b/tests/zarr-sample-data/example_field_0.zarr2/.zattrs new file mode 100644 index 00000000..bb815dea --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr2/.zattrs @@ -0,0 +1,3 @@ +{ + "Conventions": "CF-1.12" +} diff --git a/tests/zarr-sample-data/example_field_0.zarr2/.zgroup b/tests/zarr-sample-data/example_field_0.zarr2/.zgroup new file mode 100644 index 00000000..3f3fad2d --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr2/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} diff --git a/tests/zarr-sample-data/example_field_0.zarr2/.zmetadata b/tests/zarr-sample-data/example_field_0.zarr2/.zmetadata new file mode 100644 index 00000000..ab417b34 --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr2/.zmetadata @@ -0,0 +1,171 @@ +{ + "metadata": { + ".zattrs": { + "Conventions": "CF-1.12" + }, + ".zgroup": { + "zarr_format": 2 + }, + "lat/.zarray": { + "chunks": [ + 5 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "u#KAaAN@}cyD1PC8SH>5cL02`bMZ2$lO literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 b/tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 new file mode 100644 index 0000000000000000000000000000000000000000..4461e510e92cd3bb68c020a0b129a8a1fd18cf71 GIT binary patch literal 80 rcmZQ#H0E$%U|;~@03Zf~hBOB-t&oFCKPZ6kVf2P#2p>i_lsNzZV$2HZ literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr2/q/.zarray b/tests/zarr-sample-data/example_field_0.zarr2/q/.zarray new file mode 100644 index 00000000..d03af818 --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr2/q/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 3, + 4 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "+iS-3ANs1X$Nop~(PRE3&8|;Oe51d-)y3Ky#+Zub*xLx)q%l6He(qC)8%aeym IyS>dG0J;D#egFUf literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr2/q/1.0 b/tests/zarr-sample-data/example_field_0.zarr2/q/1.0 new file mode 100644 index 0000000000000000000000000000000000000000..b71f5dc59971cee259c58961f1f3f0e0ad83f4f1 GIT binary patch literal 112 zcmZQ#H0DTPU|;~@0w9)T3FFh+6m7zv{B~ hluWL$x4!S+J3Via{q3J%0jT^!d!P|O@S`3V4FI+jCtd&m literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr2/q/1.1 b/tests/zarr-sample-data/example_field_0.zarr2/q/1.1 new file mode 100644 index 0000000000000000000000000000000000000000..377eb07cb782bc18c866ac4ad785aaa5ad77e488 GIT binary patch literal 112 zcmZQ#H0DTPU|;~@0w9)T3FFh Date: Fri, 8 Aug 2025 15:43:12 +0100 Subject: [PATCH 04/37] add test --- tests/integration/test_zarr_to_iris.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 0b18dca7..d4b11f89 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -1,4 +1,7 @@ """Test conversion of remote and local Zarr store to iris Cube.""" +from importlib.resources import files as importlib_files +from pathlib import Path + import iris import xarray as xr import ncdata @@ -6,6 +9,39 @@ import zarr +def test_load_zarr2_local(): + """Test loading a Zarr2 store from local FS.""" + zarr_path = ( + Path(importlib_files("tests")) + / "zarr-sample-data" + / "example_field_0.zarr2" + ) + + time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) + zarr_xr = xr.open_dataset( + zarr_path, + consolidated=True, + decode_times=time_coder, + engine="zarr", + chunks={}, + backend_kwargs={}, + ) + zarr_xr.unify_chunks() + + conversion_func = ncdata.iris_xarray.cubes_from_xarray + cubes = conversion_func(zarr_xr) + + assert len(cubes) == 1 + cube = cubes[0] + assert cube.var_name == "q" + assert cube.standard_name == "specific_humidity" + assert cube.long_name is None + coords = cube.coords() + coord_names = [coord.standard_name for coord in coords] + assert "longitude" in coord_names + assert "latitude" in coord_names + + def test_load_remote_zarr(): """Test loading a remote Zarr store. From edd6c365cecdee3429127877467e3b7a175663c4 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 15:43:26 +0100 Subject: [PATCH 05/37] make warning more robust --- lib/ncdata/xarray.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index 5178ff4f..9f05c23d 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -22,6 +22,16 @@ from . import NcAttribute, NcData, NcDimension, NcVariable +def _raise_warning(var): + """Raise a warnings.warning if variable data not lazy.""" + warn_msg = ( + f"Variable {var} has fully realized " + "data, if you need lazy data, then add " + "chunks={} as argument to Xarray open_dataset." + ) + warnings.warn(warn_msg, UserWarning, stacklevel=2) + + class _XarrayNcDataStore(NetCDF4DataStore): """ An adapter class presenting ncdata as an xarray datastore. @@ -97,14 +107,15 @@ def store( # Install variables, creating dimensions as we go. for varname, var in new_variables.items(): - if isinstance(var.data, np.ndarray) and \ - var.attrs["axis"] not in ["X", "Y", "Z", "T"]: - warn_msg = ( - f"Variable {var} has fully realized " - "data, if you need lazy data, then add " - "chunks={} as argument to Xarray open_dataset." - ) - warnings.warn(warn_msg, UserWarning, stacklevel=2) + if "axis" not in var.attrs: + std_axes = ["latitude", "longitude", "time"] + if isinstance(var.data, np.ndarray) and \ + var.attrs["standard_name"] not in std_axes: + _raise_warning(var) + else: + if isinstance(var.data, np.ndarray) and \ + var.attrs["axis"] not in ["X", "Y", "Z", "T"]: + _raise_warning(var) if varname in self.ncdata.variables: raise ValueError(f'duplicate variable : "{varname}"') From 105031f4e2870a1866511a2f5a3a2e7607f7258f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 16:16:38 +0100 Subject: [PATCH 06/37] add Zarr3 test data --- .../example_field_0.zarr3/lat/c/0 | Bin 0 -> 41 bytes .../example_field_0.zarr3/lat/zarr.json | 47 +++ .../example_field_0.zarr3/lat_bnds/c/0/0 | Bin 0 -> 39 bytes .../example_field_0.zarr3/lat_bnds/zarr.json | 47 +++ .../example_field_0.zarr3/lon/c/0 | Bin 0 -> 50 bytes .../example_field_0.zarr3/lon/zarr.json | 47 +++ .../example_field_0.zarr3/lon_bnds/c/0/0 | Bin 0 -> 50 bytes .../example_field_0.zarr3/lon_bnds/zarr.json | 47 +++ .../example_field_0.zarr3/q/c/0/0 | Bin 0 -> 235 bytes .../example_field_0.zarr3/q/zarr.json | 52 ++++ .../example_field_0.zarr3/time/c | Bin 0 -> 17 bytes .../example_field_0.zarr3/time/zarr.json | 39 +++ .../example_field_0.zarr3/zarr.json | 292 ++++++++++++++++++ 13 files changed, 571 insertions(+) create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat/c/0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat/zarr.json create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat_bnds/c/0/0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat_bnds/zarr.json create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon/c/0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/q/c/0/0 create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/time/c create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json create mode 100644 tests/zarr-sample-data/example_field_0.zarr3/zarr.json diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lat/c/0 b/tests/zarr-sample-data/example_field_0.zarr3/lat/c/0 new file mode 100644 index 0000000000000000000000000000000000000000..7ef1fc24eaf0f39d9d6c29cc892774561768449f GIT binary patch literal 41 pcmdPcs{dC(gO!nC0|N*g2s*&f;MU;g0OdI_Gc+_zJ;-Rm1OVQa3XA{% literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lat/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/lat/zarr.json new file mode 100644 index 00000000..42da08d9 --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr3/lat/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 5 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "standard_name": "latitude", + "bounds": "lat_bnds", + }, + "dimension_names": [ + "lat" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lat_bnds/c/0/0 b/tests/zarr-sample-data/example_field_0.zarr3/lat_bnds/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..a2317cbf52c38ba0b3a4d13bd977420f72f866d2 GIT binary patch literal 39 scmdPcs{dCZ;41?|0|N*&gdJe;vvcrk2yN0ra>71 DamfvK literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json new file mode 100644 index 00000000..42f96572 --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "standard_name": "longitude", + "bounds": "lon_bnds", + }, + "dimension_names": [ + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 b/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..3a05139fabd77d1a74d4826bad22504dba20c39a GIT binary patch literal 50 zcmdPcs{dD^!IzO?0RuyWn*&4GgM@}Og`5Wk8;Tps95@*qjzOWDLZz$jh9e9S0MM2a A%>V!Z literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json new file mode 100644 index 00000000..ed74f33e --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 8, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + }, + "dimension_names": [ + "lon", + "bounds2" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/q/c/0/0 b/tests/zarr-sample-data/example_field_0.zarr3/q/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..924dc5043c1f5bd21df06523b786c973e1bb19a3 GIT binary patch literal 235 zcmVFAwm_Hc=Rt_z)oj+P+qCX1Pqi{V6r$1OZw8(b8uctrbrJKYcu(EhROgD)^ zVCAqsDO``j^(eeQ_*1~Y!aov&z{;UTwm*^Wd_Of_x`}NLh3m1cvVxI#+xek6w8(b8pNc;f04qztz|I@O1(FFIn!+H<+~iwj lHZoZpDwk-xSs49O<=ZNmqV^6U%R^;YDs53M9m*{hn`0yfZNLBk literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json new file mode 100644 index 00000000..7b895a0e --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 5, + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "project": "research", + "standard_name": "specific_humidity", + "units": "1", + "cell_methods": "area: mean", + "coordinates": "time", + }, + "dimension_names": [ + "lat", + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/time/c b/tests/zarr-sample-data/example_field_0.zarr3/time/c new file mode 100644 index 0000000000000000000000000000000000000000..16e658b14d90c05cf1c523593ca4c6f8efd0b32d GIT binary patch literal 17 ScmdPcs{dDk!;t|B>>U6kDFX5U literal 0 HcmV?d00001 diff --git a/tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json new file mode 100644 index 00000000..32ebdf51 --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json @@ -0,0 +1,39 @@ +{ + "shape": [], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "standard_name": "time", + "units": "days since 2018-12-01" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/zarr.json new file mode 100644 index 00000000..54ac972c --- /dev/null +++ b/tests/zarr-sample-data/example_field_0.zarr3/zarr.json @@ -0,0 +1,292 @@ +{ + "attributes": { + "Conventions": "CF-1.12" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "lon": { + "shape": [ + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "standard_name": "longitude", + "bounds": "lon_bnds" + }, + "dimension_names": [ + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lat_bnds": { + "shape": [ + 5, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + }, + "dimension_names": [ + "lat", + "bounds2" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lat": { + "shape": [ + 5 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "standard_name": "latitude", + "bounds": "lat_bnds" + }, + "dimension_names": [ + "lat" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "standard_name": "time", + "units": "days since 2018-12-01" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lon_bnds": { + "shape": [ + 8, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + }, + "dimension_names": [ + "lon", + "bounds2" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "q": { + "shape": [ + 5, + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "project": "research", + "standard_name": "specific_humidity", + "units": "1", + "cell_methods": "area: mean", + "coordinates": "time" + }, + "dimension_names": [ + "lat", + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} From 16fb58811ea3aa5d931af9d2ea4b3d0edc462497 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 16:17:40 +0100 Subject: [PATCH 07/37] rm erroneous file --- tests/zarr-sample-data/example_field_0.zarr17 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/zarr-sample-data/example_field_0.zarr17 diff --git a/tests/zarr-sample-data/example_field_0.zarr17 b/tests/zarr-sample-data/example_field_0.zarr17 deleted file mode 100644 index 9abbe8a4..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr17 +++ /dev/null @@ -1 +0,0 @@ -This is not a Zarr file. Go grab lunch! From f6c2766cd9aa9933fec47119951f56a57a861228 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 16:17:54 +0100 Subject: [PATCH 08/37] add zarr3 test --- tests/integration/test_zarr_to_iris.py | 54 +++++++++++++++++--------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index d4b11f89..e3569656 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -9,6 +9,16 @@ import zarr +time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) +xr_kwargs = { + "consolidated": True, + "decode_times": time_coder, + "engine": "zarr", + "chunks": {}, + "backend_kwargs": {}, +} + + def test_load_zarr2_local(): """Test loading a Zarr2 store from local FS.""" zarr_path = ( @@ -17,15 +27,7 @@ def test_load_zarr2_local(): / "example_field_0.zarr2" ) - time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) - zarr_xr = xr.open_dataset( - zarr_path, - consolidated=True, - decode_times=time_coder, - engine="zarr", - chunks={}, - backend_kwargs={}, - ) + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() conversion_func = ncdata.iris_xarray.cubes_from_xarray @@ -42,6 +44,30 @@ def test_load_zarr2_local(): assert "latitude" in coord_names +def test_load_zarr3_local(): + """Test loading a Zarr3 store from local FS.""" + zarr_path = ( + Path(importlib_files("tests")) + / "zarr-sample-data" + / "example_field_0.zarr3" + ) + + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + zarr_xr.unify_chunks() + + conversion_func = ncdata.iris_xarray.cubes_from_xarray + cubes = conversion_func(zarr_xr) + + assert len(cubes) == 1 + cube = cubes[0] + assert cube.var_name == "q" + assert cube.standard_name == "specific_humidity" + assert cube.long_name is None + coords = cube.coords() + coord_names = [coord.standard_name for coord in coords] + assert "longitude" in coord_names + assert "latitude" in coord_names + def test_load_remote_zarr(): """Test loading a remote Zarr store. @@ -54,15 +80,7 @@ def test_load_remote_zarr(): "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" ) - time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) - zarr_xr = xr.open_dataset( - zarr_path, - consolidated=True, - decode_times=time_coder, - engine="zarr", - chunks={}, - backend_kwargs={}, - ) + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() conversion_func = ncdata.iris_xarray.cubes_from_xarray From 7846f776ff17c07d75f062c1e69259511233be4a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 16:18:06 +0100 Subject: [PATCH 09/37] make the warning better --- lib/ncdata/xarray.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index 9f05c23d..b5d33529 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -107,15 +107,16 @@ def store( # Install variables, creating dimensions as we go. for varname, var in new_variables.items(): - if "axis" not in var.attrs: - std_axes = ["latitude", "longitude", "time"] - if isinstance(var.data, np.ndarray) and \ - var.attrs["standard_name"] not in std_axes: - _raise_warning(var) - else: - if isinstance(var.data, np.ndarray) and \ - var.attrs["axis"] not in ["X", "Y", "Z", "T"]: - _raise_warning(var) + if isinstance(var.data, np.ndarray): + # Zarr2 metadata + if "axis" not in var.attrs: + std_axes = ["latitude", "longitude", "time"] + if var.attrs["standard_name"] not in std_axes: + _raise_warning(var) + # Zarr3 metadata + else: + if var.attrs["axis"] not in ["X", "Y", "Z", "T"]: + _raise_warning(var) if varname in self.ncdata.variables: raise ValueError(f'duplicate variable : "{varname}"') From 2d2c661dc0fe385b974d910ca4af24fd7e51c021 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 16:55:51 +0100 Subject: [PATCH 10/37] full test suite --- tests/integration/test_zarr_to_iris.py | 71 ++++++++++++++++++-------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index e3569656..3580ce64 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -3,20 +3,35 @@ from pathlib import Path import iris +import pytest import xarray as xr import ncdata import ncdata.iris_xarray import zarr -time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) -xr_kwargs = { - "consolidated": True, - "decode_times": time_coder, - "engine": "zarr", - "chunks": {}, - "backend_kwargs": {}, -} +def _return_kwargs(): + time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) + xr_kwargs = { + "consolidated": True, + "decode_times": time_coder, + "engine": "zarr", + "chunks": {}, + "backend_kwargs": {}, + } + + return xr_kwargs + + +def _run_checks(cube): + """Run some standard checks.""" + assert cube.var_name == "q" + assert cube.standard_name == "specific_humidity" + assert cube.long_name is None + coords = cube.coords() + coord_names = [coord.standard_name for coord in coords] + assert "longitude" in coord_names + assert "latitude" in coord_names def test_load_zarr2_local(): @@ -27,6 +42,7 @@ def test_load_zarr2_local(): / "example_field_0.zarr2" ) + xr_kwargs = _return_kwargs() zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() @@ -35,13 +51,7 @@ def test_load_zarr2_local(): assert len(cubes) == 1 cube = cubes[0] - assert cube.var_name == "q" - assert cube.standard_name == "specific_humidity" - assert cube.long_name is None - coords = cube.coords() - coord_names = [coord.standard_name for coord in coords] - assert "longitude" in coord_names - assert "latitude" in coord_names + _run_checks(cube) def test_load_zarr3_local(): @@ -52,6 +62,7 @@ def test_load_zarr3_local(): / "example_field_0.zarr3" ) + xr_kwargs = _return_kwargs() zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() @@ -60,13 +71,8 @@ def test_load_zarr3_local(): assert len(cubes) == 1 cube = cubes[0] - assert cube.var_name == "q" - assert cube.standard_name == "specific_humidity" - assert cube.long_name is None - coords = cube.coords() - coord_names = [coord.standard_name for coord in coords] - assert "longitude" in coord_names - assert "latitude" in coord_names + _run_checks(cube) + def test_load_remote_zarr(): """Test loading a remote Zarr store. @@ -80,6 +86,7 @@ def test_load_remote_zarr(): "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" ) + xr_kwargs = _return_kwargs() zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() @@ -89,3 +96,23 @@ def test_load_remote_zarr(): assert isinstance(cubes, iris.cube.CubeList) assert len(cubes) == 1 assert cubes[0].has_lazy_data() + + +def test_load_remote_zarr_realized_data(): + """Test with the same remote Zarr store but chunks=None.""" + zarr_path = ( + "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" + "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" + ) + + xr_kwargs = _return_kwargs() + xr_kwargs["chunks"] = None + zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) + + conversion_func = ncdata.iris_xarray.cubes_from_xarray + msg = ( + "has fully realized data, if you need lazy data, " + "then add chunks={} as argument to Xarray open_dataset." + ) + with pytest.warns(UserWarning, match=msg) as w: + cubes = conversion_func(zarr_xr) From 179610005e8f8e0239b7dad493f70b4da316acb0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 8 Aug 2025 16:57:11 +0100 Subject: [PATCH 11/37] more general search meth --- lib/ncdata/xarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index b5d33529..e746a440 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -111,7 +111,7 @@ def store( # Zarr2 metadata if "axis" not in var.attrs: std_axes = ["latitude", "longitude", "time"] - if var.attrs["standard_name"] not in std_axes: + if not list(set(var.attrs.values()) & set(std_axes)): _raise_warning(var) # Zarr3 metadata else: From ba10ec74334caa5b8301b85b8f0cbd3b2fe3cac3 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 11 Aug 2025 14:00:55 +0100 Subject: [PATCH 12/37] add extra deps and trigger GHA locally --- .github/workflows/ci-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index b0eef1af..1e41b6fd 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -11,7 +11,7 @@ on: push: branches: - "main" - - "v*x" + - add_warning_and_tests tags: - "v*" pull_request: @@ -49,7 +49,7 @@ jobs: - name: "Install dependencies" run: | - conda install --yes numpy pytest pytest-mock iris xarray filelock requests + conda install --yes numpy pytest pytest-mock iris xarray filelock requests zarr aiohttp - name: "Install *latest* Iris" run: | From 74a7c0eff83108f9ab6594dfd90e0def4ce23d38 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 11 Aug 2025 14:18:03 +0100 Subject: [PATCH 13/37] run just a simple pytest session --- .github/workflows/ci-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1e41b6fd..2e6a5714 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -79,6 +79,11 @@ jobs: run: | ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests + - name: "Run simple pytest" + if: matrix.session == 'tests' + run: | + pytest + pytest tests/integration/test_xarray_load_and_save_equivalence.py::test_save_direct_vs_viancdata[testdata____lambert_conformal__test_lcc] - name: "Run doctests: Docs" if: matrix.session == 'doctests-docs' From d7a2698fe1a11a44f92a7cb05782cd1a50336c96 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 11 Aug 2025 14:25:11 +0100 Subject: [PATCH 14/37] pop a conda env file --- environment.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..7fad220d --- /dev/null +++ b/environment.yml @@ -0,0 +1,9 @@ +--- +name: ncdata +channels: + - conda-forge + - nodefaults + +dependencies: + - dask + - netCDF4 From 16e28cab55972cb4dd7666e8153575ff7848f128 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 11 Aug 2025 14:25:24 +0100 Subject: [PATCH 15/37] use conda env file in GHA --- .github/workflows/ci-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2e6a5714..3cf6c4ab 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -43,6 +43,7 @@ jobs: with: miniforge-version: latest activate-environment: testenv + environment-file: environment.yml python-version: ${{ matrix.version }} channels: conda-forge show-channel-urls: true From 6fdb0b9712b6da743a378fd71280db312fa1dcdd Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 11 Aug 2025 14:30:56 +0100 Subject: [PATCH 16/37] run just simple pytest --- .github/workflows/ci-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 3cf6c4ab..c46ae309 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -75,11 +75,11 @@ jobs: mkdir --parents ${GITHUB_WORKSPACE}/iris_test_data mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/iris_test_data_download - - name: "Run tests" - if: matrix.session == 'tests' - run: | - ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data - OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests + #- name: "Run tests" + # if: matrix.session == 'tests' + # run: | + # ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data + # OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests - name: "Run simple pytest" if: matrix.session == 'tests' run: | From 01cce51a4b477249a297d639e02fad16280e47ad Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 12:17:13 +0100 Subject: [PATCH 17/37] Update lib/ncdata/xarray.py Co-authored-by: Patrick Peglar --- lib/ncdata/xarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index e746a440..7e9bbb2c 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -25,7 +25,7 @@ def _raise_warning(var): """Raise a warnings.warning if variable data not lazy.""" warn_msg = ( - f"Variable {var} has fully realized " + f"Variable {var.name}{var.dims} has fully realized " "data, if you need lazy data, then add " "chunks={} as argument to Xarray open_dataset." ) From b7ad9f4d79f15d585462a1eaecfb26ede31a3b36 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 12:38:19 +0100 Subject: [PATCH 18/37] unreference uneeded conda env file --- environment.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 environment.yml diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 7fad220d..00000000 --- a/environment.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: ncdata -channels: - - conda-forge - - nodefaults - -dependencies: - - dask - - netCDF4 From cc79ff1aff7edaded666ab0542afbbfd86a63fb6 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 12:39:48 +0100 Subject: [PATCH 19/37] restore GHA workflow to original --- .github/workflows/ci-tests.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c46ae309..1e41b6fd 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -43,7 +43,6 @@ jobs: with: miniforge-version: latest activate-environment: testenv - environment-file: environment.yml python-version: ${{ matrix.version }} channels: conda-forge show-channel-urls: true @@ -75,16 +74,11 @@ jobs: mkdir --parents ${GITHUB_WORKSPACE}/iris_test_data mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/iris_test_data_download - #- name: "Run tests" - # if: matrix.session == 'tests' - # run: | - # ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data - # OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests - - name: "Run simple pytest" + - name: "Run tests" if: matrix.session == 'tests' run: | - pytest - pytest tests/integration/test_xarray_load_and_save_equivalence.py::test_save_direct_vs_viancdata[testdata____lambert_conformal__test_lcc] + ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data + OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests - name: "Run doctests: Docs" if: matrix.session == 'doctests-docs' From e2f607ff84a56e7c91543e012e5fe13c6fba00f5 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 12:40:33 +0100 Subject: [PATCH 20/37] add comment --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1e41b6fd..48d21c08 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -11,7 +11,7 @@ on: push: branches: - "main" - - add_warning_and_tests + - add_warning_and_tests # remove before merge tags: - "v*" pull_request: From 773de45b0158e9ba837ba924e8eb204b1ba0ab3e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 12:42:32 +0100 Subject: [PATCH 21/37] Update lib/ncdata/xarray.py Co-authored-by: Patrick Peglar --- lib/ncdata/xarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index 7e9bbb2c..a9c960a1 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -111,7 +111,7 @@ def store( # Zarr2 metadata if "axis" not in var.attrs: std_axes = ["latitude", "longitude", "time"] - if not list(set(var.attrs.values()) & set(std_axes)): + if not [s for s in std_axes if s in str(var.attrs.values())]: _raise_warning(var) # Zarr3 metadata else: From edf06acfb22e6465404794dd6848d44ea09217ad Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 13:01:06 +0100 Subject: [PATCH 22/37] remove user warning --- lib/ncdata/xarray.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index 7e9bbb2c..598c4435 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -9,8 +9,6 @@ # Hopefully a minimal amount. # The structure of an NcData object makes it fairly painless. # -import warnings - from pathlib import Path from typing import AnyStr, Union @@ -22,16 +20,6 @@ from . import NcAttribute, NcData, NcDimension, NcVariable -def _raise_warning(var): - """Raise a warnings.warning if variable data not lazy.""" - warn_msg = ( - f"Variable {var.name}{var.dims} has fully realized " - "data, if you need lazy data, then add " - "chunks={} as argument to Xarray open_dataset." - ) - warnings.warn(warn_msg, UserWarning, stacklevel=2) - - class _XarrayNcDataStore(NetCDF4DataStore): """ An adapter class presenting ncdata as an xarray datastore. @@ -107,16 +95,6 @@ def store( # Install variables, creating dimensions as we go. for varname, var in new_variables.items(): - if isinstance(var.data, np.ndarray): - # Zarr2 metadata - if "axis" not in var.attrs: - std_axes = ["latitude", "longitude", "time"] - if not list(set(var.attrs.values()) & set(std_axes)): - _raise_warning(var) - # Zarr3 metadata - else: - if var.attrs["axis"] not in ["X", "Y", "Z", "T"]: - _raise_warning(var) if varname in self.ncdata.variables: raise ValueError(f'duplicate variable : "{varname}"') From aba9fa69f85d5c2d02bb6d46729b2385f3c683ce Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 13:02:15 +0100 Subject: [PATCH 23/37] remove test for warning --- tests/integration/test_zarr_to_iris.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 3580ce64..d8fd8265 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -96,23 +96,3 @@ def test_load_remote_zarr(): assert isinstance(cubes, iris.cube.CubeList) assert len(cubes) == 1 assert cubes[0].has_lazy_data() - - -def test_load_remote_zarr_realized_data(): - """Test with the same remote Zarr store but chunks=None.""" - zarr_path = ( - "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" - "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" - ) - - xr_kwargs = _return_kwargs() - xr_kwargs["chunks"] = None - zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) - - conversion_func = ncdata.iris_xarray.cubes_from_xarray - msg = ( - "has fully realized data, if you need lazy data, " - "then add chunks={} as argument to Xarray open_dataset." - ) - with pytest.warns(UserWarning, match=msg) as w: - cubes = conversion_func(zarr_xr) From 1fb1db7abd36b824d8e9adc0605c6e157c5f8092 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 13:04:43 +0100 Subject: [PATCH 24/37] readd empty line --- lib/ncdata/xarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index 598c4435..cf92ce74 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -9,6 +9,7 @@ # Hopefully a minimal amount. # The structure of an NcData object makes it fairly painless. # + from pathlib import Path from typing import AnyStr, Union From 75e24cfa7b9d63893d19c9a2b6a09292fa717330 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 13:16:18 +0100 Subject: [PATCH 25/37] unrun GHA on push --- .github/workflows/ci-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 48d21c08..11cba8c3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -11,7 +11,6 @@ on: push: branches: - "main" - - add_warning_and_tests # remove before merge tags: - "v*" pull_request: From b48088a2c009c1ba37d202f5110d59b2c38a1fb5 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 15:23:21 +0100 Subject: [PATCH 26/37] Update tests/integration/test_zarr_to_iris.py Co-authored-by: Patrick Peglar --- tests/integration/test_zarr_to_iris.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index d8fd8265..13fa398e 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -7,7 +7,6 @@ import xarray as xr import ncdata import ncdata.iris_xarray -import zarr def _return_kwargs(): From 736572989bea6f146cfc67cf8a8bd0d0b19be977 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 15:26:26 +0100 Subject: [PATCH 27/37] Update tests/integration/test_zarr_to_iris.py Co-authored-by: Patrick Peglar --- tests/integration/test_zarr_to_iris.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 13fa398e..72cc3e8f 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -73,6 +73,25 @@ def test_load_zarr3_local(): _run_checks(cube) +#TODO: imports at the top +import requests +from requests.exceptions import RequestException + +def _is_url_ok(url): + try: + req = requests.get(url) + result = req.status_code == 200 + except RequestException: + result = False + return result + +S3_TEST_PATH = ( + "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" + "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" +) +_S3_accessible = _is_url_ok(S3_TEST_PATH) + +@pytest.mark.skipif(not _S3_accessible, reason="S3 url not accessible") def test_load_remote_zarr(): """Test loading a remote Zarr store. From 33a9c80661ba5879db0e5c2392906e7bc27bb8cf Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 15:43:32 +0100 Subject: [PATCH 28/37] Update tests/integration/test_zarr_to_iris.py Co-authored-by: Patrick Peglar --- tests/integration/test_zarr_to_iris.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 72cc3e8f..dc1da624 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -99,10 +99,7 @@ def test_load_remote_zarr(): Conversion is done fully lazily, by passing chunks={} to Xarray loader. Test takes ~3-4s and needs ~400MB res mem. """ - zarr_path = ( - "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" - "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" - ) + zarr_path = S3_TEST_PATH xr_kwargs = _return_kwargs() zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) From 0518afec427927ea7b4e95bc4f054e4ec86d33ab Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 16:00:16 +0100 Subject: [PATCH 29/37] shorten imports --- tests/integration/test_zarr_to_iris.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 72cc3e8f..9d5d4f38 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -5,9 +5,10 @@ import iris import pytest import xarray as xr -import ncdata import ncdata.iris_xarray +from ncdata.iris_xarray import cubes_from_xarray as conversion_func + def _return_kwargs(): time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) @@ -45,7 +46,6 @@ def test_load_zarr2_local(): zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() - conversion_func = ncdata.iris_xarray.cubes_from_xarray cubes = conversion_func(zarr_xr) assert len(cubes) == 1 @@ -65,7 +65,6 @@ def test_load_zarr3_local(): zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() - conversion_func = ncdata.iris_xarray.cubes_from_xarray cubes = conversion_func(zarr_xr) assert len(cubes) == 1 @@ -108,7 +107,6 @@ def test_load_remote_zarr(): zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) zarr_xr.unify_chunks() - conversion_func = ncdata.iris_xarray.cubes_from_xarray cubes = conversion_func(zarr_xr) assert isinstance(cubes, iris.cube.CubeList) From 8f4e4dae487d48f334c29e1208052d1627da305e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 16:37:24 +0100 Subject: [PATCH 30/37] correct test for s3 connection --- tests/integration/test_zarr_to_iris.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 623e86e5..128fb1a9 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -2,6 +2,7 @@ from importlib.resources import files as importlib_files from pathlib import Path +import fsspec import iris import pytest import xarray as xr @@ -72,17 +73,18 @@ def test_load_zarr3_local(): _run_checks(cube) -#TODO: imports at the top -import requests -from requests.exceptions import RequestException - def _is_url_ok(url): + fs = fsspec.filesystem("http") + valid_zarr = True try: - req = requests.get(url) - result = req.status_code == 200 - except RequestException: - result = False - return result + fs.open(str(url) + "/zarr.json", "rb") # Zarr3 + except Exception: # noqa: BLE001 + try: + fs.open(str(url) + "/.zmetadata", "rb") # Zarr2 + except Exception: # noqa: BLE001 + valid_zarr = False + + return valid_zarr S3_TEST_PATH = ( "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" From 1c21daaf26e21fdd8f481e29a7514d03980dd5bb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 16:37:55 +0100 Subject: [PATCH 31/37] add dependency --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 11cba8c3..707cb552 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -48,7 +48,7 @@ jobs: - name: "Install dependencies" run: | - conda install --yes numpy pytest pytest-mock iris xarray filelock requests zarr aiohttp + conda install --yes numpy pytest pytest-mock iris xarray filelock requests zarr aiohttp fsspec - name: "Install *latest* Iris" run: | From ec917451e38cd56f48feb6737f340f5f89c27b8c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 16:41:11 +0100 Subject: [PATCH 32/37] remove zarr sample data --- .../example_field_0.zarr2/.zattrs | 3 - .../example_field_0.zarr2/.zgroup | 3 - .../example_field_0.zarr2/.zmetadata | 171 ---------- .../example_field_0.zarr2/lat/.zarray | 20 -- .../example_field_0.zarr2/lat/.zattrs | 8 - .../example_field_0.zarr2/lat/0 | Bin 56 -> 0 bytes .../example_field_0.zarr2/lat_bnds/.zarray | 22 -- .../example_field_0.zarr2/lat_bnds/.zattrs | 6 - .../example_field_0.zarr2/lat_bnds/0.0 | Bin 64 -> 0 bytes .../example_field_0.zarr2/lat_bnds/1.0 | Bin 64 -> 0 bytes .../example_field_0.zarr2/lon/.zarray | 20 -- .../example_field_0.zarr2/lon/.zattrs | 8 - .../example_field_0.zarr2/lon/0 | Bin 80 -> 0 bytes .../example_field_0.zarr2/lon_bnds/.zarray | 22 -- .../example_field_0.zarr2/lon_bnds/.zattrs | 6 - .../example_field_0.zarr2/lon_bnds/0.0 | Bin 80 -> 0 bytes .../example_field_0.zarr2/lon_bnds/1.0 | Bin 80 -> 0 bytes .../example_field_0.zarr2/q/.zarray | 22 -- .../example_field_0.zarr2/q/.zattrs | 11 - .../example_field_0.zarr2/q/0.0 | Bin 112 -> 0 bytes .../example_field_0.zarr2/q/0.1 | Bin 112 -> 0 bytes .../example_field_0.zarr2/q/1.0 | Bin 112 -> 0 bytes .../example_field_0.zarr2/q/1.1 | Bin 112 -> 0 bytes .../example_field_0.zarr2/time/.zarray | 10 - .../example_field_0.zarr2/time/.zattrs | 5 - .../example_field_0.zarr2/time/0 | Bin 8 -> 0 bytes .../example_field_0.zarr3/lat/c/0 | Bin 41 -> 0 bytes .../example_field_0.zarr3/lat/zarr.json | 47 --- .../example_field_0.zarr3/lat_bnds/c/0/0 | Bin 39 -> 0 bytes .../example_field_0.zarr3/lat_bnds/zarr.json | 47 --- .../example_field_0.zarr3/lon/c/0 | Bin 50 -> 0 bytes .../example_field_0.zarr3/lon/zarr.json | 47 --- .../example_field_0.zarr3/lon_bnds/c/0/0 | Bin 50 -> 0 bytes .../example_field_0.zarr3/lon_bnds/zarr.json | 47 --- .../example_field_0.zarr3/q/c/0/0 | Bin 235 -> 0 bytes .../example_field_0.zarr3/q/zarr.json | 52 ---- .../example_field_0.zarr3/time/c | Bin 17 -> 0 bytes .../example_field_0.zarr3/time/zarr.json | 39 --- .../example_field_0.zarr3/zarr.json | 292 ------------------ 39 files changed, 908 deletions(-) delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/.zattrs delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/.zgroup delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/.zmetadata delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat/.zarray delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat/.zattrs delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zarray delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zattrs delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/0.0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lat_bnds/1.0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon/.zarray delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon/.zattrs delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/.zarray delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/.zattrs delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/0.0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/.zarray delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/.zattrs delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/0.0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/0.1 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/1.0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/q/1.1 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/time/.zarray delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/time/.zattrs delete mode 100644 tests/zarr-sample-data/example_field_0.zarr2/time/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat/c/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat/zarr.json delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat_bnds/c/0/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lat_bnds/zarr.json delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon/c/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/q/c/0/0 delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/time/c delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json delete mode 100644 tests/zarr-sample-data/example_field_0.zarr3/zarr.json diff --git a/tests/zarr-sample-data/example_field_0.zarr2/.zattrs b/tests/zarr-sample-data/example_field_0.zarr2/.zattrs deleted file mode 100644 index bb815dea..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr2/.zattrs +++ /dev/null @@ -1,3 +0,0 @@ -{ - "Conventions": "CF-1.12" -} diff --git a/tests/zarr-sample-data/example_field_0.zarr2/.zgroup b/tests/zarr-sample-data/example_field_0.zarr2/.zgroup deleted file mode 100644 index 3f3fad2d..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr2/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} diff --git a/tests/zarr-sample-data/example_field_0.zarr2/.zmetadata b/tests/zarr-sample-data/example_field_0.zarr2/.zmetadata deleted file mode 100644 index ab417b34..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr2/.zmetadata +++ /dev/null @@ -1,171 +0,0 @@ -{ - "metadata": { - ".zattrs": { - "Conventions": "CF-1.12" - }, - ".zgroup": { - "zarr_format": 2 - }, - "lat/.zarray": { - "chunks": [ - 5 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "u#KAaAN@}cyD1PC8SH>5cL02`bMZ2$lO diff --git a/tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 b/tests/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 deleted file mode 100644 index 4461e510e92cd3bb68c020a0b129a8a1fd18cf71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 rcmZQ#H0E$%U|;~@03Zf~hBOB-t&oFCKPZ6kVf2P#2p>i_lsNzZV$2HZ diff --git a/tests/zarr-sample-data/example_field_0.zarr2/q/.zarray b/tests/zarr-sample-data/example_field_0.zarr2/q/.zarray deleted file mode 100644 index d03af818..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr2/q/.zarray +++ /dev/null @@ -1,22 +0,0 @@ -{ - "chunks": [ - 3, - 4 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "+iS-3ANs1X$Nop~(PRE3&8|;Oe51d-)y3Ky#+Zub*xLx)q%l6He(qC)8%aeym IyS>dG0J;D#egFUf diff --git a/tests/zarr-sample-data/example_field_0.zarr2/q/1.0 b/tests/zarr-sample-data/example_field_0.zarr2/q/1.0 deleted file mode 100644 index b71f5dc59971cee259c58961f1f3f0e0ad83f4f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmZQ#H0DTPU|;~@0w9)T3FFh+6m7zv{B~ hluWL$x4!S+J3Via{q3J%0jT^!d!P|O@S`3V4FI+jCtd&m diff --git a/tests/zarr-sample-data/example_field_0.zarr2/q/1.1 b/tests/zarr-sample-data/example_field_0.zarr2/q/1.1 deleted file mode 100644 index 377eb07cb782bc18c866ac4ad785aaa5ad77e488..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmZQ#H0DTPU|;~@0w9)T3FFhN0ra>71 DamfvK diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json deleted file mode 100644 index 42f96572..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr3/lon/zarr.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "shape": [ - 8 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 8 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - "units": "degrees_east", - "standard_name": "longitude", - "bounds": "lon_bnds", - }, - "dimension_names": [ - "lon" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] -} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 b/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 deleted file mode 100644 index 3a05139fabd77d1a74d4826bad22504dba20c39a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 50 zcmdPcs{dD^!IzO?0RuyWn*&4GgM@}Og`5Wk8;Tps95@*qjzOWDLZz$jh9e9S0MM2a A%>V!Z diff --git a/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json deleted file mode 100644 index ed74f33e..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "shape": [ - 8, - 2 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 8, - 2 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - }, - "dimension_names": [ - "lon", - "bounds2" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] -} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/q/c/0/0 b/tests/zarr-sample-data/example_field_0.zarr3/q/c/0/0 deleted file mode 100644 index 924dc5043c1f5bd21df06523b786c973e1bb19a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 235 zcmVFAwm_Hc=Rt_z)oj+P+qCX1Pqi{V6r$1OZw8(b8uctrbrJKYcu(EhROgD)^ zVCAqsDO``j^(eeQ_*1~Y!aov&z{;UTwm*^Wd_Of_x`}NLh3m1cvVxI#+xek6w8(b8pNc;f04qztz|I@O1(FFIn!+H<+~iwj lHZoZpDwk-xSs49O<=ZNmqV^6U%R^;YDs53M9m*{hn`0yfZNLBk diff --git a/tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json deleted file mode 100644 index 7b895a0e..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr3/q/zarr.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "shape": [ - 5, - 8 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 5, - 8 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - "project": "research", - "standard_name": "specific_humidity", - "units": "1", - "cell_methods": "area: mean", - "coordinates": "time", - }, - "dimension_names": [ - "lat", - "lon" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] -} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/time/c b/tests/zarr-sample-data/example_field_0.zarr3/time/c deleted file mode 100644 index 16e658b14d90c05cf1c523593ca4c6f8efd0b32d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17 ScmdPcs{dDk!;t|B>>U6kDFX5U diff --git a/tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json deleted file mode 100644 index 32ebdf51..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr3/time/zarr.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "shape": [], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - "standard_name": "time", - "units": "days since 2018-12-01" - }, - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] -} diff --git a/tests/zarr-sample-data/example_field_0.zarr3/zarr.json b/tests/zarr-sample-data/example_field_0.zarr3/zarr.json deleted file mode 100644 index 54ac972c..00000000 --- a/tests/zarr-sample-data/example_field_0.zarr3/zarr.json +++ /dev/null @@ -1,292 +0,0 @@ -{ - "attributes": { - "Conventions": "CF-1.12" - }, - "zarr_format": 3, - "consolidated_metadata": { - "kind": "inline", - "must_understand": false, - "metadata": { - "lon": { - "shape": [ - 8 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 8 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - "units": "degrees_east", - "standard_name": "longitude", - "bounds": "lon_bnds" - }, - "dimension_names": [ - "lon" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] - }, - "lat_bnds": { - "shape": [ - 5, - 2 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 5, - 2 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - }, - "dimension_names": [ - "lat", - "bounds2" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] - }, - "lat": { - "shape": [ - 5 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 5 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - "units": "degrees_north", - "standard_name": "latitude", - "bounds": "lat_bnds" - }, - "dimension_names": [ - "lat" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] - }, - "time": { - "shape": [], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - "standard_name": "time", - "units": "days since 2018-12-01" - }, - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] - }, - "lon_bnds": { - "shape": [ - 8, - 2 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 8, - 2 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - }, - "dimension_names": [ - "lon", - "bounds2" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] - }, - "q": { - "shape": [ - 5, - 8 - ], - "data_type": "float64", - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 5, - 8 - ] - } - }, - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "fill_value": 0.0, - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - }, - { - "name": "zstd", - "configuration": { - "level": 0, - "checksum": false - } - } - ], - "attributes": { - "project": "research", - "standard_name": "specific_humidity", - "units": "1", - "cell_methods": "area: mean", - "coordinates": "time" - }, - "dimension_names": [ - "lat", - "lon" - ], - "zarr_format": 3, - "node_type": "array", - "storage_transformers": [] - } - } - }, - "node_type": "group" -} From 0d74e97a4c826a64ce7b0e35848388fc03eae1da Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 16:41:24 +0100 Subject: [PATCH 33/37] move zzarr sample data --- .../example_field_0.zarr2/.zattrs | 3 + .../example_field_0.zarr2/.zgroup | 3 + .../example_field_0.zarr2/.zmetadata | 171 ++++++++++ .../example_field_0.zarr2/lat/.zarray | 20 ++ .../example_field_0.zarr2/lat/.zattrs | 8 + .../example_field_0.zarr2/lat/0 | Bin 0 -> 56 bytes .../example_field_0.zarr2/lat_bnds/.zarray | 22 ++ .../example_field_0.zarr2/lat_bnds/.zattrs | 6 + .../example_field_0.zarr2/lat_bnds/0.0 | Bin 0 -> 64 bytes .../example_field_0.zarr2/lat_bnds/1.0 | Bin 0 -> 64 bytes .../example_field_0.zarr2/lon/.zarray | 20 ++ .../example_field_0.zarr2/lon/.zattrs | 8 + .../example_field_0.zarr2/lon/0 | Bin 0 -> 80 bytes .../example_field_0.zarr2/lon_bnds/.zarray | 22 ++ .../example_field_0.zarr2/lon_bnds/.zattrs | 6 + .../example_field_0.zarr2/lon_bnds/0.0 | Bin 0 -> 80 bytes .../example_field_0.zarr2/lon_bnds/1.0 | Bin 0 -> 80 bytes .../example_field_0.zarr2/q/.zarray | 22 ++ .../example_field_0.zarr2/q/.zattrs | 11 + .../example_field_0.zarr2/q/0.0 | Bin 0 -> 112 bytes .../example_field_0.zarr2/q/0.1 | Bin 0 -> 112 bytes .../example_field_0.zarr2/q/1.0 | Bin 0 -> 112 bytes .../example_field_0.zarr2/q/1.1 | Bin 0 -> 112 bytes .../example_field_0.zarr2/time/.zarray | 10 + .../example_field_0.zarr2/time/.zattrs | 5 + .../example_field_0.zarr2/time/0 | Bin 0 -> 8 bytes .../example_field_0.zarr3/lat/c/0 | Bin 0 -> 41 bytes .../example_field_0.zarr3/lat/zarr.json | 47 +++ .../example_field_0.zarr3/lat_bnds/c/0/0 | Bin 0 -> 39 bytes .../example_field_0.zarr3/lat_bnds/zarr.json | 47 +++ .../example_field_0.zarr3/lon/c/0 | Bin 0 -> 50 bytes .../example_field_0.zarr3/lon/zarr.json | 47 +++ .../example_field_0.zarr3/lon_bnds/c/0/0 | Bin 0 -> 50 bytes .../example_field_0.zarr3/lon_bnds/zarr.json | 47 +++ .../example_field_0.zarr3/q/c/0/0 | Bin 0 -> 235 bytes .../example_field_0.zarr3/q/zarr.json | 52 ++++ .../example_field_0.zarr3/time/c | Bin 0 -> 17 bytes .../example_field_0.zarr3/time/zarr.json | 39 +++ .../example_field_0.zarr3/zarr.json | 292 ++++++++++++++++++ 39 files changed, 908 insertions(+) create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lat/.zarray create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lat/.zattrs create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lat/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zarray create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zattrs create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lat_bnds/0.0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lat_bnds/1.0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lon/.zarray create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lon/.zattrs create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lon/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lon_bnds/.zarray create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lon_bnds/.zattrs create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lon_bnds/0.0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/q/.zarray create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/q/.zattrs create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/q/0.0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/q/0.1 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/q/1.0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/q/1.1 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/time/.zarray create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/time/.zattrs create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr2/time/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lat/c/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lat/zarr.json create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lat_bnds/c/0/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lat_bnds/zarr.json create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lon/c/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lon/zarr.json create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/q/c/0/0 create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/q/zarr.json create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/time/c create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/time/zarr.json create mode 100644 tests/testdata/zarr-sample-data/example_field_0.zarr3/zarr.json diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs new file mode 100644 index 00000000..bb815dea --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zattrs @@ -0,0 +1,3 @@ +{ + "Conventions": "CF-1.12" +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup new file mode 100644 index 00000000..3f3fad2d --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata new file mode 100644 index 00000000..ab417b34 --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/.zmetadata @@ -0,0 +1,171 @@ +{ + "metadata": { + ".zattrs": { + "Conventions": "CF-1.12" + }, + ".zgroup": { + "zarr_format": 2 + }, + "lat/.zarray": { + "chunks": [ + 5 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "u#KAaAN@}cyD1PC8SH>5cL02`bMZ2$lO literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 b/tests/testdata/zarr-sample-data/example_field_0.zarr2/lon_bnds/1.0 new file mode 100644 index 0000000000000000000000000000000000000000..4461e510e92cd3bb68c020a0b129a8a1fd18cf71 GIT binary patch literal 80 rcmZQ#H0E$%U|;~@03Zf~hBOB-t&oFCKPZ6kVf2P#2p>i_lsNzZV$2HZ literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/q/.zarray b/tests/testdata/zarr-sample-data/example_field_0.zarr2/q/.zarray new file mode 100644 index 00000000..d03af818 --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr2/q/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 3, + 4 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "+iS-3ANs1X$Nop~(PRE3&8|;Oe51d-)y3Ky#+Zub*xLx)q%l6He(qC)8%aeym IyS>dG0J;D#egFUf literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/q/1.0 b/tests/testdata/zarr-sample-data/example_field_0.zarr2/q/1.0 new file mode 100644 index 0000000000000000000000000000000000000000..b71f5dc59971cee259c58961f1f3f0e0ad83f4f1 GIT binary patch literal 112 zcmZQ#H0DTPU|;~@0w9)T3FFh+6m7zv{B~ hluWL$x4!S+J3Via{q3J%0jT^!d!P|O@S`3V4FI+jCtd&m literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr2/q/1.1 b/tests/testdata/zarr-sample-data/example_field_0.zarr2/q/1.1 new file mode 100644 index 0000000000000000000000000000000000000000..377eb07cb782bc18c866ac4ad785aaa5ad77e488 GIT binary patch literal 112 zcmZQ#H0DTPU|;~@0w9)T3FFhN0ra>71 DamfvK literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon/zarr.json b/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon/zarr.json new file mode 100644 index 00000000..42f96572 --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "standard_name": "longitude", + "bounds": "lon_bnds", + }, + "dimension_names": [ + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 b/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon_bnds/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..3a05139fabd77d1a74d4826bad22504dba20c39a GIT binary patch literal 50 zcmdPcs{dD^!IzO?0RuyWn*&4GgM@}Og`5Wk8;Tps95@*qjzOWDLZz$jh9e9S0MM2a A%>V!Z literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json b/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json new file mode 100644 index 00000000..ed74f33e --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr3/lon_bnds/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 8, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + }, + "dimension_names": [ + "lon", + "bounds2" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/q/c/0/0 b/tests/testdata/zarr-sample-data/example_field_0.zarr3/q/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..924dc5043c1f5bd21df06523b786c973e1bb19a3 GIT binary patch literal 235 zcmVFAwm_Hc=Rt_z)oj+P+qCX1Pqi{V6r$1OZw8(b8uctrbrJKYcu(EhROgD)^ zVCAqsDO``j^(eeQ_*1~Y!aov&z{;UTwm*^Wd_Of_x`}NLh3m1cvVxI#+xek6w8(b8pNc;f04qztz|I@O1(FFIn!+H<+~iwj lHZoZpDwk-xSs49O<=ZNmqV^6U%R^;YDs53M9m*{hn`0yfZNLBk literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/q/zarr.json b/tests/testdata/zarr-sample-data/example_field_0.zarr3/q/zarr.json new file mode 100644 index 00000000..7b895a0e --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr3/q/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 5, + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "project": "research", + "standard_name": "specific_humidity", + "units": "1", + "cell_methods": "area: mean", + "coordinates": "time", + }, + "dimension_names": [ + "lat", + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/time/c b/tests/testdata/zarr-sample-data/example_field_0.zarr3/time/c new file mode 100644 index 0000000000000000000000000000000000000000..16e658b14d90c05cf1c523593ca4c6f8efd0b32d GIT binary patch literal 17 ScmdPcs{dDk!;t|B>>U6kDFX5U literal 0 HcmV?d00001 diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/time/zarr.json b/tests/testdata/zarr-sample-data/example_field_0.zarr3/time/zarr.json new file mode 100644 index 00000000..32ebdf51 --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr3/time/zarr.json @@ -0,0 +1,39 @@ +{ + "shape": [], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "standard_name": "time", + "units": "days since 2018-12-01" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} diff --git a/tests/testdata/zarr-sample-data/example_field_0.zarr3/zarr.json b/tests/testdata/zarr-sample-data/example_field_0.zarr3/zarr.json new file mode 100644 index 00000000..54ac972c --- /dev/null +++ b/tests/testdata/zarr-sample-data/example_field_0.zarr3/zarr.json @@ -0,0 +1,292 @@ +{ + "attributes": { + "Conventions": "CF-1.12" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "lon": { + "shape": [ + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "standard_name": "longitude", + "bounds": "lon_bnds" + }, + "dimension_names": [ + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lat_bnds": { + "shape": [ + 5, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + }, + "dimension_names": [ + "lat", + "bounds2" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lat": { + "shape": [ + 5 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "standard_name": "latitude", + "bounds": "lat_bnds" + }, + "dimension_names": [ + "lat" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "standard_name": "time", + "units": "days since 2018-12-01" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lon_bnds": { + "shape": [ + 8, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 8, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + }, + "dimension_names": [ + "lon", + "bounds2" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "q": { + "shape": [ + 5, + 8 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 8 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0.0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "project": "research", + "standard_name": "specific_humidity", + "units": "1", + "cell_methods": "area: mean", + "coordinates": "time" + }, + "dimension_names": [ + "lat", + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} From b6dd8cbb4301d269bdaa62d2e48c76c17d57e140 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 16:43:32 +0100 Subject: [PATCH 34/37] fix test for new sample data path --- tests/integration/test_zarr_to_iris.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 128fb1a9..57889de8 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -39,6 +39,7 @@ def test_load_zarr2_local(): """Test loading a Zarr2 store from local FS.""" zarr_path = ( Path(importlib_files("tests")) + / "testdata" / "zarr-sample-data" / "example_field_0.zarr2" ) @@ -58,6 +59,7 @@ def test_load_zarr3_local(): """Test loading a Zarr3 store from local FS.""" zarr_path = ( Path(importlib_files("tests")) + / "testdata" / "zarr-sample-data" / "example_field_0.zarr3" ) From f8dd7bed1c8929708aff5bcf810c68600f869993 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 13 Aug 2025 16:52:56 +0100 Subject: [PATCH 35/37] run pre-commit --- tests/integration/test_zarr_to_iris.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 57889de8..ebc2db7a 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -1,4 +1,5 @@ """Test conversion of remote and local Zarr store to iris Cube.""" + from importlib.resources import files as importlib_files from pathlib import Path @@ -6,23 +7,19 @@ import iris import pytest import xarray as xr -import ncdata.iris_xarray - from ncdata.iris_xarray import cubes_from_xarray as conversion_func def _return_kwargs(): time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) - xr_kwargs = { - "consolidated": True, - "decode_times": time_coder, - "engine": "zarr", - "chunks": {}, - "backend_kwargs": {}, + return { + "consolidated": True, + "decode_times": time_coder, + "engine": "zarr", + "chunks": {}, + "backend_kwargs": {}, } - return xr_kwargs - def _run_checks(cube): """Run some standard checks.""" @@ -88,12 +85,14 @@ def _is_url_ok(url): return valid_zarr + S3_TEST_PATH = ( "https://uor-aces-o.s3-ext.jc.rl.ac.uk/" "esmvaltool-zarr/pr_Amon_CNRM-ESM2-1_02Kpd-11_r1i1p2f2_gr_200601-220112.zarr3" ) _S3_accessible = _is_url_ok(S3_TEST_PATH) + @pytest.mark.skipif(not _S3_accessible, reason="S3 url not accessible") def test_load_remote_zarr(): """Test loading a remote Zarr store. From e7df68dc98783371e803b9c391df3cd282ff3d0d Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 14 Aug 2025 12:57:57 +0100 Subject: [PATCH 36/37] remove unify chunks --- tests/integration/test_zarr_to_iris.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index ebc2db7a..5b8dc33b 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -43,7 +43,6 @@ def test_load_zarr2_local(): xr_kwargs = _return_kwargs() zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) - zarr_xr.unify_chunks() cubes = conversion_func(zarr_xr) @@ -63,7 +62,6 @@ def test_load_zarr3_local(): xr_kwargs = _return_kwargs() zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) - zarr_xr.unify_chunks() cubes = conversion_func(zarr_xr) @@ -105,7 +103,6 @@ def test_load_remote_zarr(): xr_kwargs = _return_kwargs() zarr_xr = xr.open_dataset(zarr_path, **xr_kwargs) - zarr_xr.unify_chunks() cubes = conversion_func(zarr_xr) From 2ee710bc932c8c95e63b8d2afa607ebcb5389d9a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:05:36 +0000 Subject: [PATCH 37/37] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- lib/ncdata/utils/_compare_nc_datasets.py | 1 - lib/ncdata/utils/_save_errors.py | 1 - tests/integration/equivalence_testing_utils.py | 3 +-- tests/integration/example_scripts/ex_dataset_print.py | 2 +- tests/integration/example_scripts/ex_iris_saveto_ncdata.py | 2 +- tests/integration/example_scripts/ex_iris_xarray_conversion.py | 2 +- .../integration/example_scripts/ex_ncdata_netcdf_conversion.py | 2 +- tests/integration/test_iris_load_and_save_equivalence.py | 2 +- tests/integration/test_iris_xarray_roundtrips.py | 2 +- tests/integration/test_netcdf_roundtrips.py | 1 + tests/integration/test_xarray_load_and_save_equivalence.py | 2 +- tests/unit/core/test_AttributeAccessMixin.py | 1 - tests/unit/core/test_AttrvalsDict.py | 1 - tests/unit/core/test_NameMap.py | 1 - tests/unit/core/test_NcAttribute.py | 1 - tests/unit/core/test_NcDimension.py | 1 - tests/unit/core/test_NcVariable.py | 1 - tests/unit/iris/test_from_iris.py | 2 +- tests/unit/iris/test_to_iris.py | 2 +- tests/unit/netcdf/test_from_nc4.py | 2 +- tests/unit/netcdf/test_to_nc4.py | 2 +- .../test_dataset_differences__additional.py | 2 +- .../test_dataset_differences__mainfunctions.py | 1 - .../utils/compare_nc_datasets/test_variable_differences.py | 1 - tests/unit/utils/test_ncdata_copy.py | 1 - tests/unit/utils/test_save_errors.py | 2 +- tests/unit/xarray/test_from_xarray.py | 2 +- tests/unit/xarray/test_to_xarray.py | 2 +- 28 files changed, 17 insertions(+), 28 deletions(-) diff --git a/lib/ncdata/utils/_compare_nc_datasets.py b/lib/ncdata/utils/_compare_nc_datasets.py index 8c7053db..b9892a39 100644 --- a/lib/ncdata/utils/_compare_nc_datasets.py +++ b/lib/ncdata/utils/_compare_nc_datasets.py @@ -14,7 +14,6 @@ import netCDF4 import netCDF4 as nc import numpy as np - from ncdata import NcData, NcVariable diff --git a/lib/ncdata/utils/_save_errors.py b/lib/ncdata/utils/_save_errors.py index aa2e7228..f8cfcaf8 100644 --- a/lib/ncdata/utils/_save_errors.py +++ b/lib/ncdata/utils/_save_errors.py @@ -3,7 +3,6 @@ import netCDF4 as nc import numpy as np - from ncdata import NcData, NcVariable diff --git a/tests/integration/equivalence_testing_utils.py b/tests/integration/equivalence_testing_utils.py index 8693bc6d..87a11dca 100644 --- a/tests/integration/equivalence_testing_utils.py +++ b/tests/integration/equivalence_testing_utils.py @@ -5,11 +5,10 @@ ncdata and other types of data preserve information. """ import dask.array as da +import iris.mesh import numpy as np import pytest -import iris.mesh - def cubes_equal__corrected(c1, c2): """ diff --git a/tests/integration/example_scripts/ex_dataset_print.py b/tests/integration/example_scripts/ex_dataset_print.py index 5f3251fb..83ea63eb 100644 --- a/tests/integration/example_scripts/ex_dataset_print.py +++ b/tests/integration/example_scripts/ex_dataset_print.py @@ -1,8 +1,8 @@ """Temporary integrational proof-of-concept example for dataset printout.""" import iris - import ncdata.iris as nci from ncdata import NcData, NcDimension, NcVariable + from tests import testdata_dir diff --git a/tests/integration/example_scripts/ex_iris_saveto_ncdata.py b/tests/integration/example_scripts/ex_iris_saveto_ncdata.py index c4a3c14b..ec24eeb2 100644 --- a/tests/integration/example_scripts/ex_iris_saveto_ncdata.py +++ b/tests/integration/example_scripts/ex_iris_saveto_ncdata.py @@ -4,8 +4,8 @@ Check that conversion succeeds and print the resulting dataset. """ import iris - from ncdata.iris import from_iris + from tests import testdata_dir diff --git a/tests/integration/example_scripts/ex_iris_xarray_conversion.py b/tests/integration/example_scripts/ex_iris_xarray_conversion.py index 018c1548..361a9c08 100644 --- a/tests/integration/example_scripts/ex_iris_xarray_conversion.py +++ b/tests/integration/example_scripts/ex_iris_xarray_conversion.py @@ -7,8 +7,8 @@ import iris import numpy as np import xarray as xr - from ncdata.iris_xarray import cubes_from_xarray, cubes_to_xarray + from tests import testdata_dir diff --git a/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py b/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py index 7f2e614e..becc646c 100644 --- a/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py +++ b/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py @@ -9,10 +9,10 @@ import netCDF4 as nc import numpy as np - from ncdata import NcData, NcDimension, NcVariable from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences + from tests import testdata_dir diff --git a/tests/integration/test_iris_load_and_save_equivalence.py b/tests/integration/test_iris_load_and_save_equivalence.py index 7ad3eb95..5117fca5 100644 --- a/tests/integration/test_iris_load_and_save_equivalence.py +++ b/tests/integration/test_iris_load_and_save_equivalence.py @@ -9,9 +9,9 @@ import iris import pytest - from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences + from tests.data_testcase_schemas import session_testdir, standard_testcase from tests.integration.equivalence_testing_utils import ( adjust_chunks, diff --git a/tests/integration/test_iris_xarray_roundtrips.py b/tests/integration/test_iris_xarray_roundtrips.py index 9ece022b..7cfeabac 100644 --- a/tests/integration/test_iris_xarray_roundtrips.py +++ b/tests/integration/test_iris_xarray_roundtrips.py @@ -13,13 +13,13 @@ import numpy as np import pytest import xarray - from ncdata.iris import from_iris from ncdata.iris_xarray import cubes_to_xarray from ncdata.netcdf4 import from_nc4 from ncdata.threadlock_sharing import lockshare_context from ncdata.utils import dataset_differences from ncdata.xarray import from_xarray + from tests.data_testcase_schemas import ( BAD_LOADSAVE_TESTCASES, session_testdir, diff --git a/tests/integration/test_netcdf_roundtrips.py b/tests/integration/test_netcdf_roundtrips.py index 6fe635d5..db9ea467 100644 --- a/tests/integration/test_netcdf_roundtrips.py +++ b/tests/integration/test_netcdf_roundtrips.py @@ -5,6 +5,7 @@ from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences + from tests.data_testcase_schemas import session_testdir, standard_testcase # Avoid complaints that the imported fixtures are "unused" diff --git a/tests/integration/test_xarray_load_and_save_equivalence.py b/tests/integration/test_xarray_load_and_save_equivalence.py index d7fb3164..f36eb06b 100644 --- a/tests/integration/test_xarray_load_and_save_equivalence.py +++ b/tests/integration/test_xarray_load_and_save_equivalence.py @@ -7,11 +7,11 @@ """ import pytest import xarray - from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.threadlock_sharing import lockshare_context from ncdata.utils import dataset_differences from ncdata.xarray import from_xarray, to_xarray + from tests.data_testcase_schemas import ( BAD_LOADSAVE_TESTCASES, session_testdir, diff --git a/tests/unit/core/test_AttributeAccessMixin.py b/tests/unit/core/test_AttributeAccessMixin.py index 803b6b15..615ea92e 100644 --- a/tests/unit/core/test_AttributeAccessMixin.py +++ b/tests/unit/core/test_AttributeAccessMixin.py @@ -6,7 +6,6 @@ """ import numpy as np import pytest - from ncdata import NcData, NcVariable diff --git a/tests/unit/core/test_AttrvalsDict.py b/tests/unit/core/test_AttrvalsDict.py index d15ad262..faca9325 100644 --- a/tests/unit/core/test_AttrvalsDict.py +++ b/tests/unit/core/test_AttrvalsDict.py @@ -3,7 +3,6 @@ import numpy as np import pytest - from ncdata import NcAttribute, NcVariable diff --git a/tests/unit/core/test_NameMap.py b/tests/unit/core/test_NameMap.py index 72650ce5..5538ecf0 100644 --- a/tests/unit/core/test_NameMap.py +++ b/tests/unit/core/test_NameMap.py @@ -4,7 +4,6 @@ from copy import deepcopy import pytest - from ncdata import NameMap, NcAttribute diff --git a/tests/unit/core/test_NcAttribute.py b/tests/unit/core/test_NcAttribute.py index 26ef2e10..45895764 100644 --- a/tests/unit/core/test_NcAttribute.py +++ b/tests/unit/core/test_NcAttribute.py @@ -5,7 +5,6 @@ """ import numpy as np import pytest - from ncdata import NcAttribute # Support for building testcases diff --git a/tests/unit/core/test_NcDimension.py b/tests/unit/core/test_NcDimension.py index 6af29e79..1ab98199 100644 --- a/tests/unit/core/test_NcDimension.py +++ b/tests/unit/core/test_NcDimension.py @@ -1,7 +1,6 @@ """Tests for class :class:`ncdata.NcDimension`.""" import numpy as np import pytest - from ncdata import NcDimension diff --git a/tests/unit/core/test_NcVariable.py b/tests/unit/core/test_NcVariable.py index 7376d183..798d7c64 100644 --- a/tests/unit/core/test_NcVariable.py +++ b/tests/unit/core/test_NcVariable.py @@ -6,7 +6,6 @@ import dask.array as da import numpy as np import pytest - from ncdata import NcVariable from ncdata.utils import variable_differences diff --git a/tests/unit/iris/test_from_iris.py b/tests/unit/iris/test_from_iris.py index 8ce3555a..0e8c7421 100644 --- a/tests/unit/iris/test_from_iris.py +++ b/tests/unit/iris/test_from_iris.py @@ -15,8 +15,8 @@ import pytest from iris.coords import DimCoord from iris.cube import Cube - from ncdata.iris import from_iris + from tests import MonitoredArray diff --git a/tests/unit/iris/test_to_iris.py b/tests/unit/iris/test_to_iris.py index 8170e804..e9067e1f 100644 --- a/tests/unit/iris/test_to_iris.py +++ b/tests/unit/iris/test_to_iris.py @@ -12,9 +12,9 @@ import numpy as np from iris._constraints import NameConstraint from iris.cube import CubeList - from ncdata import NcData, NcDimension, NcVariable from ncdata.iris import to_iris + from tests import MonitoredArray diff --git a/tests/unit/netcdf/test_from_nc4.py b/tests/unit/netcdf/test_from_nc4.py index ea612912..1f5b8a1d 100644 --- a/tests/unit/netcdf/test_from_nc4.py +++ b/tests/unit/netcdf/test_from_nc4.py @@ -13,10 +13,10 @@ import netCDF4 as nc import numpy as np import pytest - from ncdata import NcData, NcDimension, NcVariable from ncdata.netcdf4 import from_nc4 from ncdata.utils import dataset_differences + from tests.data_testcase_schemas import make_testcase_dataset diff --git a/tests/unit/netcdf/test_to_nc4.py b/tests/unit/netcdf/test_to_nc4.py index 8f2934a2..1f6574e3 100644 --- a/tests/unit/netcdf/test_to_nc4.py +++ b/tests/unit/netcdf/test_to_nc4.py @@ -14,10 +14,10 @@ import netCDF4 as nc import numpy as np import pytest - from ncdata import NcData from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences + from tests.data_testcase_schemas import make_testcase_dataset diff --git a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py index 373c62d8..9f738958 100644 --- a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py +++ b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py @@ -12,12 +12,12 @@ import netCDF4 as nc import numpy as np import pytest - from ncdata.utils._compare_nc_datasets import ( _attribute_differences, _namelist_differences, dataset_differences, ) + from tests.test_samplecode_cdlgen_comparablecdl import ncgen_from_cdl _simple_cdl = """ diff --git a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py index 4d3a3294..7c421b7f 100644 --- a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py +++ b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py @@ -8,7 +8,6 @@ """ import numpy as np import pytest - from ncdata import NcAttribute, NcData, NcDimension, NcVariable from ncdata.utils import dataset_differences diff --git a/tests/unit/utils/compare_nc_datasets/test_variable_differences.py b/tests/unit/utils/compare_nc_datasets/test_variable_differences.py index 840d8ef8..ddb60110 100644 --- a/tests/unit/utils/compare_nc_datasets/test_variable_differences.py +++ b/tests/unit/utils/compare_nc_datasets/test_variable_differences.py @@ -1,7 +1,6 @@ import dask.array as da import numpy as np import pytest - from ncdata import NcVariable from ncdata.utils import variable_differences diff --git a/tests/unit/utils/test_ncdata_copy.py b/tests/unit/utils/test_ncdata_copy.py index 17a0cc6e..9d369109 100644 --- a/tests/unit/utils/test_ncdata_copy.py +++ b/tests/unit/utils/test_ncdata_copy.py @@ -4,7 +4,6 @@ """ import numpy as np import pytest - from ncdata import NameMap, NcAttribute, NcData, NcDimension, NcVariable from ncdata.utils import dataset_differences, ncdata_copy diff --git a/tests/unit/utils/test_save_errors.py b/tests/unit/utils/test_save_errors.py index f9ad4c92..f5a508df 100644 --- a/tests/unit/utils/test_save_errors.py +++ b/tests/unit/utils/test_save_errors.py @@ -5,9 +5,9 @@ import numpy as np import pytest - from ncdata import NcData, NcDimension, NcVariable from ncdata.utils import save_errors + from tests.unit.core.test_NcAttribute import attrvalue, datatype, structuretype _ = datatype, structuretype diff --git a/tests/unit/xarray/test_from_xarray.py b/tests/unit/xarray/test_from_xarray.py index 587c1983..f64255bd 100644 --- a/tests/unit/xarray/test_from_xarray.py +++ b/tests/unit/xarray/test_from_xarray.py @@ -14,8 +14,8 @@ import numpy as np import pytest import xarray as xr - from ncdata.xarray import from_xarray + from tests import MonitoredArray from tests.data_testcase_schemas import make_testcase_dataset diff --git a/tests/unit/xarray/test_to_xarray.py b/tests/unit/xarray/test_to_xarray.py index f2e777f4..b91dd70d 100644 --- a/tests/unit/xarray/test_to_xarray.py +++ b/tests/unit/xarray/test_to_xarray.py @@ -11,9 +11,9 @@ import dask.array as da import numpy as np import pytest - from ncdata import NcData, NcDimension, NcVariable from ncdata.xarray import to_xarray + from tests import MonitoredArray