From 5b18898aa0e00b7e5b11c68e7487eb7d2548dd23 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 2 Jul 2025 00:30:00 +0100 Subject: [PATCH 01/14] WIP make all codeblocks into viable doctests. --- .../userdocs/getting_started/introduction.rst | 21 +++++++++---------- .../userdocs/user_guide/common_operations.rst | 7 +++++-- docs/userdocs/user_guide/data_objects.rst | 3 ++- docs/userdocs/user_guide/howtos.rst | 19 +++++++++++------ 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/docs/userdocs/getting_started/introduction.rst b/docs/userdocs/getting_started/introduction.rst index d89f743..a03803b 100644 --- a/docs/userdocs/getting_started/introduction.rst +++ b/docs/userdocs/getting_started/introduction.rst @@ -38,7 +38,7 @@ and :attr:`~ncdata.NcData.attributes`: >>> from ncdata import NcData, NcDimension, NcVariable >>> data = NcData("myname") >>> data - + >>> print(data) @@ -50,7 +50,6 @@ and :attr:`~ncdata.NcData.attributes`: dimensions: x = 3 > - >>> data.dimensions['x'] is dim True @@ -73,9 +72,9 @@ Simple example: >>> print(check_output('ncdump -h tmp.nc', shell=True).decode()) netcdf tmp { dimensions: - x = 3 ; + x = 3 ; } - + >>> data2 = from_nc4(filepath) >>> print(data2) >> data.variables.add(var) >>> data.variables - {'vx': } + {'vx': >> data.variables['vx'] is var True @@ -106,7 +105,7 @@ which behaves like a dictionary: variables: > @@ -137,7 +136,7 @@ or :class:`~ncdata.NcVariable`: variables: variables: >> from ndata.threadlock_sharing import enable_lockshare + >>> from ncdata.threadlock_sharing import enable_lockshare >>> enable_lockshare(iris=True, xarray=True) .. code-block:: python >>> from ncdata.netcdf import from_nc4 - >>> ncdata = from_nc4("datapath.nc") + >>> data = from_nc4("tmp.nc") .. code-block:: python >>> from ncdata.iris import to_iris, from_iris - >>> xx, yy = to_iris(ncdata, ['x_wind', 'y_wind']) + >>> xx, yy = to_iris(data, ['x_wind', 'y_wind']) >>> vv = (xx * xx + yy * yy) ** 0.5 >>> vv.units = xx.units diff --git a/docs/userdocs/user_guide/common_operations.rst b/docs/userdocs/user_guide/common_operations.rst index 9aa4cc8..46c9b4a 100644 --- a/docs/userdocs/user_guide/common_operations.rst +++ b/docs/userdocs/user_guide/common_operations.rst @@ -53,7 +53,9 @@ These however do *not* copy variable data arrays (either real or lazy), but prod .. code-block:: - >>> Construct a simple test dataset + >>> # Construct a simple test dataset + >>> import numpy as np + >>> from ncdata import NcData, NcDimension, NcVariable >>> ds = NcData( ... dimensions=[NcDimension('x', 12)], ... variables=[NcVariable('vx', ['x'], np.ones(12))] @@ -73,7 +75,8 @@ These however do *not* copy variable data arrays (either real or lazy), but prod >>> # So changing one actually CHANGES THE OTHER ... >>> ds.variables['vx'].data[6:] = 777 >>> ds_copy.variables['vx'].data - array([1., 1., 1., 1., 1., 1., 777., 777., 777., 777., 777., 777.]) + array([ 1., 1., 1., 1., 1., 1., 777., 777., 777., 777., 777., + 777.]) If needed you can of course replace variable data with copies yourself, since you can freely assign to ``.data``. diff --git a/docs/userdocs/user_guide/data_objects.rst b/docs/userdocs/user_guide/data_objects.rst index dc81ab4..560b8f7 100644 --- a/docs/userdocs/user_guide/data_objects.rst +++ b/docs/userdocs/user_guide/data_objects.rst @@ -214,6 +214,7 @@ either a pre-created container or a similar dictionary-like object : .. code-block:: python + >>> from ncdata import NcData, NcVariable >>> ds1 = NcData(groups={ ... 'x':NcData('x'), ... 'y':NcData('y') @@ -256,7 +257,7 @@ will be automatically converted to a NameMap of ``name: NcAttribute(name: value) >>> print(var) ): v3() v3:x = 'this' - v3:b = 1.4, + v3:b = 1.4 v3:arr = array([1, 2, 3]) > diff --git a/docs/userdocs/user_guide/howtos.rst b/docs/userdocs/user_guide/howtos.rst index 74caa5f..3cdd8f8 100644 --- a/docs/userdocs/user_guide/howtos.rst +++ b/docs/userdocs/user_guide/howtos.rst @@ -16,12 +16,19 @@ Index by component names to get the object which represents a particular element .. code-block:: python - >>> dataset.attributes["experiment"] - NcAttribute("'experiment', 'A301.7') - >>> dataset.dimensions["x"] + >>> from ncdata import NcData, NcAttribute, NcDimension, NcVariable + >>> data = NcData( + ... dimensions=[NcDimension("x", 3)], + ... variables=[NcVariable("vx", attributes={"units": "m.s-1"})], + ... attributes={"experiment": "A301.7"} + ... ) + ... + >>> data.attributes["experiment"] + NcAttribute('experiment', 'A301.7') + >>> data.dimensions["x"] NcDimension('x', 3) - >>> dataset.variables['vx'].attributes['units'] - NcAttribute("'unit', 'm s-1') + >>> data.variables['vx'].attributes['units'] + NcAttribute('units', 'm.s-1') Variable, attributes, dimensions and sub-groups are all stored by name like this, in a parent property which is a "component container" dictionary. @@ -42,7 +49,7 @@ a new item. >>> data.dimensions.add(NcDimension("y", 4)) >>> data.dimensions - {'x': NcDimension('x', 3) 'y': NcDimension('y', 3)} + {'x': NcDimension('x', 3), 'y': NcDimension('y', 4)} The item must be of the correct type, in this case a :class:`~ncdata.NcDimension`. If not, an error will be raised. From f739633c1f65660fe4aa967917b0d27c3cd97a49 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Sun, 27 Jul 2025 17:56:59 +0100 Subject: [PATCH 02/14] Odd fixes. --- .../userdocs/getting_started/introduction.rst | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/userdocs/getting_started/introduction.rst b/docs/userdocs/getting_started/introduction.rst index a03803b..3e82237 100644 --- a/docs/userdocs/getting_started/introduction.rst +++ b/docs/userdocs/getting_started/introduction.rst @@ -72,7 +72,7 @@ Simple example: >>> print(check_output('ncdump -h tmp.nc', shell=True).decode()) netcdf tmp { dimensions: - x = 3 ; + ...x = 3 ; } >>> data2 = from_nc4(filepath) @@ -234,21 +234,33 @@ Example code snippets : .. code-block:: python - >>> from ncdata.netcdf import from_nc4 + >>> from ncdata.netcdf4 import from_nc4 >>> data = from_nc4("tmp.nc") .. code-block:: python >>> from ncdata.iris import to_iris, from_iris - >>> xx, yy = to_iris(data, ['x_wind', 'y_wind']) + >>> data = NcData( + ... dimensions=[NcDimension("x", 3)], + ... variables=[ + ... NcVariable("vx0", ["x"], data=[1, 2, 1], + ... attributes={"long_name": "vx", "units": "m.s-1"}), + ... NcVariable("vx1", ["x"], data=[3, 4, 6], + ... attributes={"long_name": "vy", "units": "m.s-1"}) + ... ] + ... ) + >>> xx, yy = to_iris(data, constraints=['vx', 'vy']) + >>> print(xx) + unknown / (m.s-1) (-- : 3) >>> vv = (xx * xx + yy * yy) ** 0.5 - >>> vv.units = xx.units + >>> print(vv) + unknown / (m.s-1) (-- : 3) .. code-block:: python >>> from ncdata.xarray import to_xarray >>> xrds = to_xarray(from_iris(vv)) - >>> xrds.to_zarr(out_path) + >>> xrds.to_zarr("./zarr1") .. code-block:: python @@ -268,7 +280,7 @@ Thread safety .. code-block:: python - >>> from ndata.threadlock_sharing import enable_lockshare + >>> from ncdata.threadlock_sharing import enable_lockshare >>> enable_lockshare(iris=True, xarray=True) See details at :ref:`thread_safety`. From b1e079f7708d3514f9fad3f691f7f093eb21ab65 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 29 Jul 2025 17:36:59 +0100 Subject: [PATCH 03/14] Add support for sphinx-copybutton, and css for hiding codeblock sections. --- docs/_static/css/hiddencode.css | 1 + docs/conf.py | 10 ++++++++++ requirements/readthedocs.yml | 1 + 3 files changed, 12 insertions(+) create mode 100644 docs/_static/css/hiddencode.css diff --git a/docs/_static/css/hiddencode.css b/docs/_static/css/hiddencode.css new file mode 100644 index 0000000..bdc5b53 --- /dev/null +++ b/docs/_static/css/hiddencode.css @@ -0,0 +1 @@ +.hiddencode { display: none } diff --git a/docs/conf.py b/docs/conf.py index 05f704c..5f2f86f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -37,6 +37,7 @@ "sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.napoleon", + "sphinx_copybutton", ] intersphinx_mapping = { @@ -107,6 +108,15 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +html_css_files = ["css/hiddencode.css"] + + +# -- copybutton extension ----------------------------------------------------- +# See https://sphinx-copybutton.readthedocs.io/en/latest/ +copybutton_prompt_text = r">>> |\.\.\. " +copybutton_prompt_is_regexp = True +copybutton_line_continuation_character = "\\" + # Various scheme control settings. # See https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html diff --git a/requirements/readthedocs.yml b/requirements/readthedocs.yml index cc80ea0..8c6e512 100644 --- a/requirements/readthedocs.yml +++ b/requirements/readthedocs.yml @@ -15,5 +15,6 @@ dependencies: - python<3.13 - sphinx - sphinxcontrib-napoleon + - sphinx-copybutton - towncrier - xarray From 888e8e82cf7a50491c34b6f330f71853e1a42b1d Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 30 Jul 2025 13:59:50 +0100 Subject: [PATCH 04/14] All codeblocks converted to funcitoning doctests -- first working. --- .../userdocs/getting_started/introduction.rst | 96 +++- docs/userdocs/user_guide/data_objects.rst | 23 +- docs/userdocs/user_guide/howtos.rst | 473 ++++++++++++++---- 3 files changed, 470 insertions(+), 122 deletions(-) diff --git a/docs/userdocs/getting_started/introduction.rst b/docs/userdocs/getting_started/introduction.rst index 3e82237..8917fdf 100644 --- a/docs/userdocs/getting_started/introduction.rst +++ b/docs/userdocs/getting_started/introduction.rst @@ -45,13 +45,18 @@ and :attr:`~ncdata.NcData.attributes`: >>> dim = NcDimension("x", 3) >>> data.dimensions.add(dim) + >>> data.dimensions['x'] is dim + True + + >>> data.variables.add(NcVariable('vx', ["x"], dtype=float)) >>> print(data) + variables: + > - >>> data.dimensions['x'] is dim - True Getting data to+from files @@ -59,20 +64,37 @@ Getting data to+from files The :mod:`ncdata.netcdf4` module provides simple means of reading and writing NetCDF files via the `netcdf4-python package `_. +.. raw:: html + +
+ +.. code-block:: python + + >>> from subprocess import check_output + >>> def ncdump(path): + ... text = check_output(f'ncdump -h {path}', shell=True).decode() + ... text = text.replace("\t", " " * 3) + ... print(text) + +.. raw:: html + +
+ + Simple example: .. code-block:: python >>> from ncdata.netcdf4 import to_nc4, from_nc4 - >>> filepath = "./tmp.nc" >>> to_nc4(data, filepath) - >>> from subprocess import check_output - >>> print(check_output('ncdump -h tmp.nc', shell=True).decode()) + >>> ncdump("tmp.nc") # NOTE: function (not shown) calls command-line ncdump netcdf tmp { dimensions: - ...x = 3 ; + x = 3 ; + variables: + double vx(x) ; } >>> data2 = from_nc4(filepath) @@ -80,6 +102,9 @@ Simple example: + variables: + > Please see `Converting between data formats`_ for more details. @@ -92,14 +117,11 @@ which behaves like a dictionary: .. code-block:: python - >>> var = NcVariable("vx", dimensions=["x"], dtype=float) - >>> data.variables.add(var) - >>> data.variables - {'vx': } - >>> data.variables['vx'] is var - True + >>> var = NcVariable("newvar", dimensions=["x"], data=[1, 2, 3]) + >>> data.variables.add(var) >>> print(data) variables: + > + >>> # remove again, for simpler subsequent testing + >>> del data.variables["newvar"] + Attributes ^^^^^^^^^^ -Variables live in the ``attributes`` property of a :class:`~ncdata.NcData` +Attributes live in the ``attributes`` property of a :class:`~ncdata.NcData` or :class:`~ncdata.NcVariable`: .. code-block:: python + >>> var = data.variables["vx"] >>> var.set_attrval('a', 1) NcAttribute('a', 1) >>> var.set_attrval('b', 'this') @@ -229,6 +256,7 @@ Example code snippets : .. code-block:: python + >>> # (make sure that Iris and Ncdata won't conflict over netcdf access) >>> from ncdata.threadlock_sharing import enable_lockshare >>> enable_lockshare(iris=True, xarray=True) @@ -240,33 +268,53 @@ Example code snippets : .. code-block:: python >>> from ncdata.iris import to_iris, from_iris + >>> from iris import FUTURE + >>> # (avoid some irritating warnings) + >>> FUTURE.save_split_attrs = True + >>> data = NcData( ... dimensions=[NcDimension("x", 3)], ... variables=[ ... NcVariable("vx0", ["x"], data=[1, 2, 1], - ... attributes={"long_name": "vx", "units": "m.s-1"}), + ... attributes={"long_name": "speed_x", "units": "m.s-1"}), ... NcVariable("vx1", ["x"], data=[3, 4, 6], - ... attributes={"long_name": "vy", "units": "m.s-1"}) + ... attributes={"long_name": "speed_y", "units": "m.s-1"}) ... ] ... ) - >>> xx, yy = to_iris(data, constraints=['vx', 'vy']) - >>> print(xx) - unknown / (m.s-1) (-- : 3) - >>> vv = (xx * xx + yy * yy) ** 0.5 + >>> vx, vy = to_iris(data, constraints=['speed_x', 'speed_y']) + >>> print(vx) + speed_x / (m.s-1) (-- : 3) + >>> vv = (0.5 * (vx * vx + vy * vy)) ** 0.5 + >>> vv.rename("v_mag") >>> print(vv) - unknown / (m.s-1) (-- : 3) + v_mag / (m.s-1) (-- : 3) .. code-block:: python >>> from ncdata.xarray import to_xarray - >>> xrds = to_xarray(from_iris(vv)) - >>> xrds.to_zarr("./zarr1") + >>> xrds = to_xarray(from_iris([vx, vy, vv])) + >>> print(xrds) + Size: ... + Dimensions: (dim0: 3) + Dimensions without coordinates: dim0 + Data variables: + vx0 (dim0) int64 ... dask.array + vx1 (dim0) int64 ... dask.array + v_mag (dim0) float64 ... dask.array + Attributes: + Conventions: CF-1.7 .. code-block:: python >>> from ncdata.iris_xarray import cubes_from_xarray - >>> vv2 = cubes_from_xarray(xrds) - >>> assert vv2 == vv + >>> readback = cubes_from_xarray(xrds) + >>> # warning: order is indeterminate! + >>> from iris.cube import CubeList + >>> readback = CubeList(sorted(readback, key=lambda cube: cube.name())) + >>> print(readback) + 0: speed_x / (m.s-1) (-- : 3) + 1: speed_y / (m.s-1) (-- : 3) + 2: v_mag / (m.s-1) (-- : 3) Thread safety diff --git a/docs/userdocs/user_guide/data_objects.rst b/docs/userdocs/user_guide/data_objects.rst index 560b8f7..ff2f73f 100644 --- a/docs/userdocs/user_guide/data_objects.rst +++ b/docs/userdocs/user_guide/data_objects.rst @@ -161,10 +161,24 @@ not attribute values. Thus to fetch an attribute you might write, for example one of these : +.. raw:: html + +
+ +.. code-block:: + >>> from ncdata import NcData, NcVariable, NcAttribute + >>> dataset = NcData(variables=[NcVariable("var1", attributes={"units": "m"})]) + +.. raw:: html + +
+ + .. code-block:: - units1 = dataset.variables['var1'].get_attrval('units') - units1 = dataset.variables['var1'].attributes['units'].as_python_value() + >>> units1 = dataset.variables['var1'].get_attrval('units') + >>> units1 = dataset.variables['var1'].attributes['units'].as_python_value() + but **not** ``unit = dataset.variables['x'].attributes['attr1']`` @@ -172,8 +186,9 @@ Or, likewise, to **set** values, one of .. code-block:: - dataset.variables['var1'].set_attrval('units', "K") - dataset.variables['var1'].attributes['units'] = NcAttribute("units", K) + >>> dataset.variables['var1'].set_attrval('units', "K") + NcAttribute(...) + >>> dataset.variables['var1'].attributes['units'] = NcAttribute("units", "K") but **not** ``dataset.variables['x'].attributes['units'].value = "K"`` diff --git a/docs/userdocs/user_guide/howtos.rst b/docs/userdocs/user_guide/howtos.rst index 3cdd8f8..1e77857 100644 --- a/docs/userdocs/user_guide/howtos.rst +++ b/docs/userdocs/user_guide/howtos.rst @@ -8,6 +8,29 @@ documentation to describe concepts and technical details. i.e. wrong turns and gotchas, with brief descriptions of why. +.. raw:: html + +
+ +.. code-block:: + + >>> import xarray + >>> import iris + >>> iris.FUTURE.save_split_attrs = True + >>> import pathlib + >>> from pprint import pprint + >>> import numpy as np + >>> from subprocess import check_output + >>> def ncdump(path): + ... text = check_output(f'ncdump -h {path}', shell=True).decode() + ... text = text.replace("\t", " " * 3) + ... print(text) + +.. raw:: html + +
+ + .. _howto_access: Access a variable, dimension, attribute or group @@ -19,7 +42,7 @@ Index by component names to get the object which represents a particular element >>> from ncdata import NcData, NcAttribute, NcDimension, NcVariable >>> data = NcData( ... dimensions=[NcDimension("x", 3)], - ... variables=[NcVariable("vx", attributes={"units": "m.s-1"})], + ... variables=[NcVariable("vx", attributes={"units": "m.s-1", "q": 0})], ... attributes={"experiment": "A301.7"} ... ) ... @@ -47,16 +70,16 @@ Add a variable, dimension, attribute or group Use the :meth:`~ncdata.NameMap.add` method of a component-container property to insert a new item. - >>> data.dimensions.add(NcDimension("y", 4)) + >>> data.dimensions.add(NcDimension("y", 5)) >>> data.dimensions - {'x': NcDimension('x', 3), 'y': NcDimension('y', 4)} + {'x': NcDimension('x', 3), 'y': NcDimension('y', 5)} The item must be of the correct type, in this case a :class:`~ncdata.NcDimension`. If not, an error will be raised. .. Warning:: - **Why Not Just...** ``data.dimensions["y"] = NcDimension("y", 4)`` ? + **Why Not Just...** ``data.dimensions["y"] = NcDimension("y", 5)`` ? This does actually work, but the user must ensure that the dictionary key always matches the name of the component added. Using :meth:`~ncdata.NameMap.add` is thus @@ -71,10 +94,11 @@ The standard Python ``del`` operator can be applied to a component property to r something by its name. >>> data.dimensions - {'x': NcDimension('x', 3) 'y': NcDimension('y', 3)} - >>> del data.dimensions['x'] + {'x': NcDimension('x', 3), 'y': NcDimension('y', 5)} + + >>> del data.dimensions['y'] >>> data.dimensions - {'y': NcDimension('y', 3)} + {'x': NcDimension('x', 3)} .. _howto_rename_something: @@ -85,11 +109,12 @@ Use the :meth:`~ncdata.NameMap.rename` method to rename a component. .. code-block:: - >>> data.dimensions - {'x': NcDimension('x', 3) 'y': NcDimension('y', 3)} - >>> data.dimensions.rename['x', 'q'] - >>> data.dimensions - {'q': NcDimension('q', 3) 'y': NcDimension('y', 3)} + >>> data2 = NcData(variables=[NcVariable("xx")]) + >>> data2.variables + {'xx': } + >>> data2.variables.rename('xx', 'qqqq') + >>> data2.variables + {'qqqq': } Note that this affects both the element's container key *and* its ``.name``. @@ -131,19 +156,24 @@ method, which returns either a single (scalar) number, a numeric array, or a str .. code-block:: python - >>> variable.get_attrval("x") - 3.0 - >>> dataset.get_attrval("context") - "Results from experiment A301.7" - >>> dataset.variables["q"].get_attrval("level_settings") - [1.0, 2.5, 3.7] + >>> var = NcVariable("x", attributes={"a": [3.0], "levels": [1., 2, 3]}) + >>> var.get_attrval("a") + array(3.) + + >>> dataset = NcData(variables=[var], attributes={"a": "seven"}) + >>> print(dataset.get_attrval("a")) + seven + >>> print(dataset.get_attrval("context")) + None + >>> dataset.variables["x"].get_attrval("levels") + array([1., 2., 3.]) **Given an isolated** :class:`ncdata.NcAttribute` **instance** : -Its value is best read with the :meth:`ncdata.NcAttribute.get_python_value` method, +Its value is best read with the :meth:`ncdata.NcAttribute.as_python_value` method, which produces the same results as the above. - >>> variable.attributes[myname].get_python_value() + >>> print(var.attributes["a"].as_python_value()) 3.0 .. Warning:: @@ -154,13 +184,13 @@ which produces the same results as the above. .. code-block:: python - >>> data.variables["x"].attributes["q"].value - [1] + >>> print(var.attributes["a"].value) + [3.] - The ``.value`` is always stored as a :class:`~numpy.ndarray` array, but this is not - how it is stored in netCDF. The ``get_python_value()`` returns the attribute - as a straightforward value, compatible with what is seen in ``ncdump`` output, - and results from the ``netCDF4`` module. + The ``.value`` is always stored as a :class:`~numpy.ndarray` array (never a scalar), + but this is not how it is stored in netCDF. The ``get_python_value()`` returns the + attribute as a straightforward value, compatible with what is seen in ``ncdump`` + output, and results from the ``netCDF4`` module. .. _howto_write_attr: @@ -174,12 +204,15 @@ All attributes are writeable, and the type can be freely changed. .. code-block:: python - >>> variable.set_attr("x", 3.) - >>> variable.get_attrval("x") + >>> var.set_attrval("x", 3.) + NcAttribute('x', 3.0) + >>> print(var.get_attrval("x")) 3.0 - >>> variable.set_attr("x", "string-value") - >>> variable.get_attrval("x") - "string-value" + + >>> var.set_attrval("x", "string-value") + NcAttribute('x', 'string-value') + >>> var.get_attrval("x") + 'string-value' **Or** if you already have an attribute object in hand, you can simply set ``attribute.value`` directly : this a property with controlled access, so the @@ -189,10 +222,10 @@ For example .. code-block:: python - >>> attr = data.variables["x"].attributes["q"] + >>> attr = data.variables["vx"].attributes["q"] >>> attr.value = 4.2 >>> print(attr.value) - array(4.2) + 4.2 .. _howto_create_attr: @@ -206,7 +239,10 @@ attribute already exists or not. .. code-block:: python - >>> variable.set_attr("x", 3.) + >>> var.set_attrval("x", 3.) + NcAttribute('x', 3.0) + >>> print(var.attributes["x"]) + NcAttribute('x', 3.0) .. Note:: @@ -259,10 +295,10 @@ Read or write variable data The :attr:`~ncdata.NcVariable.data` property of a :class:`~ncdata.NcVariable` usually holds a data array. -.. code-block:: python +.. code-block:: >>> var.data = np.array([1, 2]) - >>> print(var.data) + >>> var.data array([1, 2]) This may be either a :class:`numpy.ndarray` (real) or a :class:`dask.array.Array` @@ -283,17 +319,36 @@ Read data from a NetCDF file ---------------------------- Use the :func:`ncdata.netcdf4.from_nc4` function to load a dataset from a netCDF file. +.. raw:: html + +
+ +.. code-block:: + + >>> _ds = NcData( + ... dimensions=[NcDimension("time", 10)], + ... variables=[NcVariable("time", ["time"], data=np.arange(10, dtype=int))], + ... ) + ... + >>> from ncdata.netcdf4 import to_nc4 + >>> filepath = "_t1.nc" + >>> to_nc4(_ds, filepath) + +.. raw:: html + +
+ .. code-block:: python - >>> from ncdata.netcdf4 from_nc4 + >>> from ncdata.netcdf4 import from_nc4 >>> ds = from_nc4(filepath) >>> print(ds) variables: - > @@ -303,9 +358,9 @@ Use the ``dim_chunks`` argument in the :func:`ncdata.netcdf4.from_nc4` function .. code-block:: python - >>> from ncdata.netcdf4 from_nc4 + >>> from ncdata.netcdf4 import from_nc4 >>> ds = from_nc4(filepath, dim_chunks={"time": 3}) - >>> print(ds.variables["x"].data.chunksize) + >>> print(ds.variables["time"].data.chunksize) (3,) @@ -317,7 +372,19 @@ Use the :func:`ncdata.netcdf4.to_nc4` function to write data to a file: >>> from ncdata.netcdf4 import to_nc4 >>> to_nc4(data, filepath) - + >>> ncdump(filepath) + netcdf ...{ + dimensions: + x = 3 ; + variables: + double vx ; + vx:units = "m.s-1" ; + vx:q = 4.2 ; + + // global attributes: + :experiment = "A301.7" ; + } + Read from or write to Iris cubes -------------------------------- @@ -326,10 +393,29 @@ Use :func:`ncdata.iris.to_iris` and :func:`ncdata.iris.from_iris`. .. code-block:: python >>> from ncdata.iris import from_iris, to_iris - >>> cubes = iris.load(file) + + >>> cubes = iris.load(filepath) + >>> print(cubes) + 0: vx / (m.s-1) (scalar cube) + >>> ncdata = from_iris(cubes) - >>> + >>> print(ncdata) + + variables: + + + global attributes: + :Conventions = 'CF-1.7' + :experiment = 'A301.7' + > + + >>> ncdata.variables.rename("vx", "vxxx") >>> cubes2 = to_iris(ncdata) + >>> print(cubes2) + 0: vxxx / (m.s-1) (scalar cube) Note that: @@ -404,18 +490,79 @@ file. Just be careful that any shared dimensions match. +.. raw:: html + +
+ +.. code-block:: python + + >>> d1 = NcData( + ... dimensions=[NcDimension("x", 3)], + ... variables=[NcVariable("DATA1_qqq", ["x"], data=[1, 2, 3])] + ... ) + >>> d2 = NcData( + ... dimensions=[NcDimension("x", 3)], + ... variables=[ + ... NcVariable("x1", ["x"], data=[111, 111, 111]), + ... NcVariable("x2", ["x"], data=[222, 222, 222]), + ... NcVariable("x3", ["x"], data=np.array([333, 333, 333], dtype=float)), + ... ] + ... ) + >>> to_nc4(d1, "input1.nc") + >>> to_nc4(d2, "input2.nc") + +.. raw:: html + +
+ .. code-block:: python >>> from ncdata.netcdf4 import from_nc4, to_nc4 - >>> data = from_nc4('input1.nc') + >>> data1 = from_nc4('input1.nc') + >>> print(data1) + + variables: + + > + >>> data2 = from_nc4('input2.nc') + >>> print(data2) + + variables: + + + + > + >>> # Add some known variables from file2 into file1 - >>> wanted = ('x1', 'x2', 'x3') + >>> wanted = ('x1', 'x3') >>> for name in wanted: - ... data.variables.add(data2.variables[name]) + ... data1.variables.add(data2.variables[name]) ... - >>> to_nc4(data, 'output.nc') + >>> # data1 has now been changed + >>> print(data1) + + variables: + + + + > + + >>> # just check that it also saves ok + >>> filepath = pathlib.Path('_temp_testdata.nc') + >>> to_nc4(data1, filepath) + >>> filepath.exists() + True Create a brand-new dataset -------------------------- @@ -428,8 +575,8 @@ Contents and components can be attached on creation ... >>> data = NcData( ... dimensions=[NcDimension("y", 2), NcDimension("x", 3)], ... variables=[ - ... NcVariable("y", ("y",), data=[0, 1]), - ... NcVariable("x", ("x",), data=[0, 1, 2]), + ... NcVariable("y", ("y",), data=list(range(2))), + ... NcVariable("x", ("x",), data=list(range(3))), ... NcVariable( ... "vyx", ("y", "x"), ... data=np.zeros((2, 3)), @@ -438,14 +585,14 @@ Contents and components can be attached on creation ... ... NcAttribute("units", "m s-1") ... ] ... )], - ... attributes=[NcAttribute("history", "imaginary")] + ... attributes={"history": "imaginary", "test_a1": 1, "test_a2": [2, 3]} ... ) >>> print(data) dimensions: y = 2 x = 3 - + variables: @@ -453,50 +600,77 @@ Contents and components can be attached on creation ... vyx:long_name = 'rate' vyx:units = 'm s-1' > - + global attributes: :history = 'imaginary' + :test_a1 = 1 + :test_a2 = array([2, 3]) > >>> + ... or added iteratively ... .. code-block:: python - >>> data = NcData() + >>> data2 = NcData() >>> ny, nx = 2, 3 - >>> data.dimensions.add(NcDimension("y", ny)) - >>> data.dimensions.add(NcDimension("x", nx)) - >>> data.variables.add(NcVariable("y", ("y",))) - >>> data.variables.add(NcVariable("x", ("x",))) - >>> data.variables.add(NcVariable("vyx", ("y", "x"))) - >>> vx, vy, vyx = [data.variables[k] for k in ("x", "y", "vyx")] + >>> data2.dimensions.add(NcDimension("y", ny)) + >>> data2.dimensions.add(NcDimension("x", nx)) + >>> data2.variables.add(NcVariable("y", ["y"], data=[0, 1])) + >>> data2.variables.add(NcVariable("x", ["x"], data=[0, 1, 2])) + >>> data2.variables.add(NcVariable("vyx", ("y", "x"), dtype=float)) + >>> vx, vy, vyx = [data2.variables[k] for k in ("x", "y", "vyx")] >>> vx.data = np.arange(nx) >>> vy.data = np.arange(ny) >>> vyx.data = np.zeros((ny, nx)) - >>> vyx.set_attrval("long_name", "rate"), + >>> vyx.set_attrval("long_name", "rate") + NcAttribute(... >>> vyx.set_attrval("units", "m s-1") - >>> data.set_attrval("history", "imaginary") + NcAttribute(... + >>> for k, v in [("history", "imaginary"), ("test_a1", 1), ("test_a2", [2, 3])]: + ... data2.set_attrval(k, v) + ... + NcAttribute(...)... + >>> # in fact, there should be NO difference between these two. + >>> from ncdata.utils import dataset_differences + >>> print(dataset_differences(data, data2) == []) + True Remove or rewrite specific attributes ------------------------------------- Load an input dataset with :func:`ncdata.netcdf4.from_nc4`. -Then you can modify, add or remove global and variable attributes at will. +Then you can modify, add or remove global and variable attributes at will, +and re-save as required. For example : +.. raw:: html + +
+ +.. code-block:: + + >>> # Save the above complex data-example + >>> to_nc4(data, "test_data.nc") + +.. raw:: html + +
+ .. code-block:: python >>> from ncdata.netcdf4 import from_nc4, to_nc4 - >>> ds = from_nc4('input.nc4') + >>> ds = from_nc4('test_data.nc') >>> history = ds.get_attrval("history") if "history" in ds.attributes else "" >>> ds.set_attrval("history", history + ": modified to SPEC-FIX.A") - >>> removes = ("grid_x", "review") + NcAttribute(...) + >>> removes = ("test_a1", "review") >>> for name in removes: ... if name in ds.attributes: - ... del ds.attributes.[name] + ... del ds.attributes[name] ... >>> for var in ds.variables.values(): ... if "coords" in var.attributes: @@ -505,7 +679,7 @@ For example : ... if units and units == "ppm": ... var.set_attrval("units", "1.e-6") # another common non-CF problem ... - >>> to_nc(ds, "output_fixed.nc") + >>> to_nc4(ds, "output_fixed.nc") Save selected variables to a new file @@ -517,26 +691,72 @@ save it with :func:`ncdata.netcdf4.to_nc4`. For a simple case with no groups, it could look something like this: +.. raw:: html + +
+ +.. code-block:: + + >>> ds = from_nc4("_temp_testdata.nc") + >>> ds.variables.add(NcVariable("z", data=[2.])) + >>> to_nc4(ds, "testfile.nc") + >>> input_filepath = "_testdata_plus.nc" + >>> to_nc4(ds, input_filepath) + >>> output_filepath = pathlib.Path("tmp.nc") + +.. raw:: html + +
+ .. code-block:: python >>> ds_in = from_nc4(input_filepath) >>> ds_out = NcData() - >>> for varname in ('data1', 'data2', 'dimx', 'dimy'): - >>> var = ds_in.variables[varname] - >>> ds_out.variables.add(var) - >>> for name in var.dimensions if name not in ds_out.dimensions: - >>> ds_out.dimensions.add(ds_in.dimensions[dimname]) + >>> wanted = ['DATA1_qqq', 'x3', 'z'] + >>> for varname in wanted: + ... var = ds_in.variables[varname] + ... ds_out.variables.add(var) + ... for dimname in var.dimensions: + ... if dimname not in ds_out.dimensions: + ... ds_out.dimensions.add(ds_in.dimensions[dimname]) ... + >>> assert "x" in ds_out.dimensions + >>> assert all(name in ds_out.variables for name in wanted) + + >>> # Also, just check that it saves OK >>> to_nc4(ds_out, output_filepath) + >>> output_filepath.exists() + True Sometimes it's simpler to load the input, delete content **not** wanted, then re-save. It's perfectly safe to do that, since the original file will be unaffected. +.. raw:: html + +
+ +.. code-block:: python + + >>> testds = NcData( + ... dimensions=[NcDimension("x", 2), NcDimension("pressure", 3)], + ... variables=[ + ... NcVariable("main1", ["x"], data=np.zeros(2)), + ... NcVariable("extra1", ["x", "pressure"], data=np.zeros((2, 3))), + ... NcVariable("extra2", ["pressure"], data=np.zeros(3)), + ... NcVariable("unwanted", data=7), + ... ], + ... ) + >>> to_nc4(testds, input_filepath) + +.. raw:: html + +
+ .. code-block:: python >>> data = from_nc4(input_filepath) - >>> for name in ('extra1', 'extra2', 'unwanted'): - >>> del data.variables[varname] + >>> for varname in ('extra1', 'extra2', 'unwanted'): + ... del data.variables[varname] ... >>> del data.dimensions['pressure'] >>> to_nc4(data, output_filepath) @@ -555,11 +775,11 @@ For example, to replace an invalid coordinate name in iris input : >>> from ncdata.netcdf4 import from_nc4 >>> from ncdata.iris import to_iris >>> ncdata = from_nc4(input_filepath) - >>> for var in ncdata.variables: - >>> coords = var.attributes.get('coordinates', "") - >>> if "old_varname" in coords: - >>> coords.replace("old_varname", "new_varname") - >>> var.set_attrval("coordinates", coords) + >>> for var in ncdata.variables.values(): + ... coords = var.attributes.get('coordinates', "") + ... if "old_varname" in coords: + ... coords.replace("old_varname", "new_varname") + ... var.set_attrval("coordinates", coords) ... >>> cubes = to_iris(ncdata) @@ -570,9 +790,9 @@ or, to replace a mis-used special attribute in xarray input : >>> from ncdata.netcdf4 import from_nc4 >>> from ncdata.xarray import to_xarray >>> ncdata = from_nc4(input_filepath) - >>> for var in ncdata.variables: - >>> if "_fillvalue" in var.attributes: - >>> var.attributes.rename("_fillvalue", "_FillValue") + >>> for var in ncdata.variables.values(): + ... if "_fillvalue" in var.attributes: + ... var.attributes.rename("_fillvalue", "_FillValue") ... >>> cubes = to_iris(ncdata) @@ -585,6 +805,31 @@ would be difficult to overcome if first written to an actual file. For example, to force an additional unlimited dimension in iris output : +.. raw:: html + +
+ +.. code-block:: python + + >>> from iris.cube import Cube + >>> from iris.coords import DimCoord + >>> co_x = DimCoord(np.arange(5.), long_name="x") + >>> co_t = DimCoord(np.arange(10.), long_name="timestep", units="days since 2010-05-01") + >>> cube = Cube(np.zeros((10, 5)), dim_coords_and_dims=[(co_t, 0), (co_x, 1)]) + >>> cubes = [cube] + + >>> # Also build a test xarray dataset. Cheat and use ncdata, to_xarray ? + >>> data = np.arange(10.) + >>> data[[2, 5]] = np.nan + >>> var = NcVariable("experiment", ["x"], data=data) + >>> ds = NcData(dimensions=[NcDimension("x", 10)], variables=[var]) + >>> to_nc4(ds, "__xr_tmp.nc") + >>> xr_dataset = xarray.open_dataset("__xr_tmp.nc", chunks=-1) + +.. raw:: html + +
+ .. code-block:: python >>> from ncdata.iris import from_iris @@ -598,15 +843,16 @@ or, to convert xarray data variable output to masked integers : .. code-block:: python >>> from numpy import ma - >>> from ncdata.iris import from_xarray + >>> from ncdata.xarray import from_xarray >>> from ncdata.netcdf4 import to_nc4 - >>> ncdata = from_xarray(dataset) + >>> ncdata = from_xarray(xr_dataset) >>> var = ncdata.variables['experiment'] - >>> mask = var.data.isnan() + >>> mask = np.isnan(var.data) >>> data = var.data.astype(np.int16) >>> data[mask] = -9999 >>> var.data = data >>> var.set_attrval("_FillValue", -9999) + NcAttribute(...) >>> to_nc4(ncdata, "output.nc") @@ -617,11 +863,38 @@ Load a file containing variable-width string variables You must supply a ``dim_chunks`` keyword to the :meth:`ncdata.netcdf4.from_nc4` method, specifying how to chunk all dimension(s) which the "string" type variable uses. +.. raw:: html + +
+ +.. code-block:: python + + >>> # manufacture a dataset with a "string" variable in it. + >>> cdl = """ + ... netcdf foo { + ... dimensions: + ... date = 6 ; + ... + ... variables: + ... string date_comments(date) ; + ... + ... data: + ... date_comments = "one", "two", "three", "four", "5", "sixteen" ; + ... } + ... """ + >>> from iris.tests.stock.netcdf import ncgen_from_cdl + >>> filepath = "_vlstring_data.nc" + >>> ncgen_from_cdl(cdl_str=cdl, cdl_path=None, nc_path=filepath) + +.. raw:: html + +
+ .. code-block:: python >>> from ncdata.netcdf4 import from_nc4 >>> # This file has a netcdf "string" type variable, with dimensions ('date',). - >>> # : don't chunk that dimension. + >>> # : **don't chunk that dimension**. >>> dataset = from_nc4(filepath, dim_chunks={"date": -1}) This is needed to avoid a Dask error like @@ -636,15 +909,27 @@ For example, something like this : .. code-block:: python - >>> var = dataset.variables['name'] - >>> data = var.data.compute() - >>> maxlen = max(len(s) for s in var.data) + >>> var = dataset.variables['date_comments'] + >>> string_objects = var.data.compute() + >>> bytes_objects = [string.encode() for string in string_objects] + >>> maxlen = max([len(bytes) for bytes in bytes_objects]) + >>> maxlen + 7 - >>> # convert to fixed-width character array - >>> data = np.array([[s.ljust(maxlen, "\0") for s in var.data]]) - >>> print(data.shape, data.dtype) - (1010, 12) >> # convert to fixed-width char array (a bit awkward because of how bytes index) + >>> newdata = np.array([[bytes[i:i+1] for i in range(maxlen)] for bytes in bytes_objects]) + >>> print(newdata.shape, newdata.dtype) + (6, 7) |S1 + >>> # NOTE: variable data dtype *must* be "S1" for intended behaviour >>> dataset.dimensions.add(NcDimension('name_strlen', maxlen)) >>> var.dimensions = var.dimensions + ("name_strlen",) - >>> var.data = data + >>> var.data = newdata + >>> # NOTE: at present it is also required to correct .dtype manually. See #88 + >>> var.dtype = newdata.dtype + + >>> # When re-saved, this data loads back OK without a chunk control + >>> to_nc4(dataset, "tmp.nc") + >>> readback = from_nc4("tmp.nc") + >>> print(readback.variables["date_comments"]) + From 8cc67f2965b128848e075dd130c4b99db8668ca8 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 30 Jul 2025 16:09:43 +0100 Subject: [PATCH 05/14] Use tests matrix for doctests(docs,api). --- .github/workflows/ci-tests.yml | 12 +++++++++++- lib/ncdata/xarray.py | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2bd80e9..d7863b3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -11,8 +11,11 @@ on: [pull_request, push] jobs: tests: - name: "Test Python ${{ matrix.version }}" + name: "Test Python ${{ matrix.version }} session= ${{ matrix.session }}" runs-on: ubuntu-latest + strategy: + matrix: + session: ["tests", "doctests-docs", "doctests-api"] defaults: run: shell: bash -l {0} @@ -61,6 +64,13 @@ jobs: mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/iris_test_data_download - name: "Run tests" + if: ${{ matrix.session }} == "tests" run: | ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests + + - name: "Run doctests: docs" + if: ${{ matrix.session }} == "doctests-docs" + run: | + cd docs + pytest --doctests-glob="*.rst" diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index cf92ce7..e106368 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -193,6 +193,12 @@ def to_xarray(ncdata: NcData, **xarray_load_kwargs) -> xr.Dataset: xrds : xarray.Dataset converted data in the form of an Xarray :class:`xarray.Dataset` + Example + ------- + >>> a = 3 + >>> a + 4 + """ return _XarrayNcDataStore(ncdata).to_xarray(**xarray_load_kwargs) From c51dac3dea5d8aaf7d73b8ef2ac4a487283d7e5f Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 30 Jul 2025 17:38:54 +0100 Subject: [PATCH 06/14] Fixes to tests gha. --- .github/workflows/ci-tests.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index d7863b3..64e7eaf 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -7,13 +7,24 @@ name: ci-tests # Triggers the workflow on pull-request or push events -on: [pull_request, push] +on: + push: + branches: + - "main" + - "v*x" + tags: + - "v*" + pull_request: + branches: + - "*" + workflow_dispatch: jobs: tests: name: "Test Python ${{ matrix.version }} session= ${{ matrix.session }}" runs-on: ubuntu-latest strategy: + fail-fast: false matrix: session: ["tests", "doctests-docs", "doctests-api"] defaults: @@ -73,4 +84,10 @@ jobs: if: ${{ matrix.session }} == "doctests-docs" run: | cd docs - pytest --doctests-glob="*.rst" + pytest --doctest-glob="*.rst" + + - name: "Run doctests: api" + if: ${{ matrix.session }} == "doctests-api" + run: | + cd lib + pytest --doctest-glob="*.py" From 6c83da3c7b3ac8776295cf594b5a3280a578d04d Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 30 Jul 2025 18:01:01 +0100 Subject: [PATCH 07/14] More gha fix. --- .github/workflows/ci-tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 64e7eaf..4e974f1 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -75,19 +75,19 @@ jobs: mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/iris_test_data_download - name: "Run tests" - if: ${{ matrix.session }} == "tests" + if: ${{ matrix.session }} == 'tests' run: | ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests - - name: "Run doctests: docs" - if: ${{ matrix.session }} == "doctests-docs" + - name: "Run doctests: Docs" + if: ${{ matrix.session }} == 'doctests-docs' run: | cd docs pytest --doctest-glob="*.rst" - - name: "Run doctests: api" - if: ${{ matrix.session }} == "doctests-api" + - name: "Run doctests: API" + if: ${{ matrix.session }} == 'doctests-api' run: | cd lib - pytest --doctest-glob="*.py" + pytest --doctest-modules From 8087a9be1dc8ffb7758073adfad8d1e6202b2c10 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 30 Jul 2025 18:10:17 +0100 Subject: [PATCH 08/14] More gha fix. --- .github/workflows/ci-tests.yml | 6 +++--- lib/ncdata/netcdf4.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4e974f1..b0eef1a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -75,19 +75,19 @@ jobs: mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/iris_test_data_download - name: "Run tests" - if: ${{ matrix.session }} == 'tests' + if: matrix.session == 'tests' run: | ls ${GITHUB_WORKSPACE}/iris_test_data_download/test_data OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/iris_test_data_download/test_data PYTHONPATH=./tests:$PYTHONPATH pytest -v ./tests - name: "Run doctests: Docs" - if: ${{ matrix.session }} == 'doctests-docs' + if: matrix.session == 'doctests-docs' run: | cd docs pytest --doctest-glob="*.rst" - name: "Run doctests: API" - if: ${{ matrix.session }} == 'doctests-api' + if: matrix.session == 'doctests-api' run: | cd lib pytest --doctest-modules diff --git a/lib/ncdata/netcdf4.py b/lib/ncdata/netcdf4.py index 5ddef22..b17c46a 100644 --- a/lib/ncdata/netcdf4.py +++ b/lib/ncdata/netcdf4.py @@ -310,12 +310,23 @@ def from_nc4( For example, to avoid cases where a simple dask ``from_array(chunks="auto")`` will fail + .. testsetup:: + + >>> from ncdata import NcData, NcDimension, NcVariable + >>> from ncdata.netcdf4 import to_nc4 + >>> testdata = NcData( + ... dimensions=[NcDimension("x", 100)], + ... variables=[NcVariable("vx", ["x"], data=np.ones(100))] + ... ) + >>> testfile_path = "tmp.nc" + >>> to_nc4(testdata, testfile_path) + + .. doctest:: + >>> from ncdata.netcdf4 import from_nc4 - >>> from tests import testdata_dir - >>> path = testdata_dir / "toa_brightness_temperature.nc" - >>> ds = from_nc4(path, dim_chunks={"x": 15}) + >>> ds = from_nc4(testfile_path, dim_chunks={"x": 15}) >>> ds.variables["data"].data.chunksize - (160, 15) + (15,) >>> See also : :ref:`howto_load_variablewidth_strings` : This illustrates a particular From 9c022cf80fb3f5487d5f8e85fb99165f0a3d9494 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 30 Jul 2025 18:28:17 +0100 Subject: [PATCH 09/14] Fix api doctests. --- lib/ncdata/netcdf4.py | 16 +++++++--------- lib/ncdata/xarray.py | 6 ------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/lib/ncdata/netcdf4.py b/lib/ncdata/netcdf4.py index b17c46a..35b6cc8 100644 --- a/lib/ncdata/netcdf4.py +++ b/lib/ncdata/netcdf4.py @@ -307,9 +307,6 @@ def from_nc4( Examples -------- - For example, to avoid cases where a simple dask ``from_array(chunks="auto")`` - will fail - .. testsetup:: >>> from ncdata import NcData, NcDimension, NcVariable @@ -321,13 +318,14 @@ def from_nc4( >>> testfile_path = "tmp.nc" >>> to_nc4(testdata, testfile_path) - .. doctest:: + For example, to avoid cases where a simple dask ``from_array(chunks="auto")`` + will fail - >>> from ncdata.netcdf4 import from_nc4 - >>> ds = from_nc4(testfile_path, dim_chunks={"x": 15}) - >>> ds.variables["data"].data.chunksize - (15,) - >>> + >>> from ncdata.netcdf4 import from_nc4 + >>> ds = from_nc4(testfile_path, dim_chunks={"x": 15}) + >>> ds.variables["vx"].data.chunksize + (15,) + >>> See also : :ref:`howto_load_variablewidth_strings` : This illustrates a particular case which **does** encounter an error with dask "auto" chunking, and therefore diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index e106368..cf92ce7 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -193,12 +193,6 @@ def to_xarray(ncdata: NcData, **xarray_load_kwargs) -> xr.Dataset: xrds : xarray.Dataset converted data in the form of an Xarray :class:`xarray.Dataset` - Example - ------- - >>> a = 3 - >>> a - 4 - """ return _XarrayNcDataStore(ncdata).to_xarray(**xarray_load_kwargs) From d1dc201d22d840e1faf448261c3f58e124ef1c0f Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 31 Jul 2025 10:27:07 +0100 Subject: [PATCH 10/14] Leave fragments when towncrier builds the changelog. --- docs/Makefile | 2 +- docs/_static/css/hiddencode.css | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 docs/_static/css/hiddencode.css diff --git a/docs/Makefile b/docs/Makefile index 94ad5c1..e03018d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -19,7 +19,7 @@ allapi: sphinx-apidoc -Mfe -o ./details/api ../lib/ncdata towncrier: - towncrier build --yes + towncrier build --keep # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/docs/_static/css/hiddencode.css b/docs/_static/css/hiddencode.css deleted file mode 100644 index bdc5b53..0000000 --- a/docs/_static/css/hiddencode.css +++ /dev/null @@ -1 +0,0 @@ -.hiddencode { display: none } From 88b5da659db30e07d59f766abbfe0d9f01250269 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 31 Jul 2025 10:30:26 +0100 Subject: [PATCH 11/14] Adopt 'sphinx.ext.doctest' and its testsetup/doctest syntax --but *don't* use it to run doctests. --- docs/conf.py | 3 +- .../userdocs/getting_started/introduction.rst | 10 +- docs/userdocs/user_guide/data_objects.rst | 11 +- docs/userdocs/user_guide/howtos.rst | 135 +++++------------- requirements/readthedocs.yml | 1 + 5 files changed, 40 insertions(+), 120 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5f2f86f..d9f62c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,6 +35,7 @@ # ones. extensions = [ "sphinx.ext.autodoc", + "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.napoleon", "sphinx_copybutton", @@ -108,8 +109,6 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -html_css_files = ["css/hiddencode.css"] - # -- copybutton extension ----------------------------------------------------- # See https://sphinx-copybutton.readthedocs.io/en/latest/ diff --git a/docs/userdocs/getting_started/introduction.rst b/docs/userdocs/getting_started/introduction.rst index 8917fdf..a2d2965 100644 --- a/docs/userdocs/getting_started/introduction.rst +++ b/docs/userdocs/getting_started/introduction.rst @@ -64,11 +64,7 @@ Getting data to+from files The :mod:`ncdata.netcdf4` module provides simple means of reading and writing NetCDF files via the `netcdf4-python package `_. -.. raw:: html - -
- -.. code-block:: python +.. testsetup:: >>> from subprocess import check_output >>> def ncdump(path): @@ -76,10 +72,6 @@ NetCDF files via the `netcdf4-python package - Simple example: diff --git a/docs/userdocs/user_guide/data_objects.rst b/docs/userdocs/user_guide/data_objects.rst index ff2f73f..9018b45 100644 --- a/docs/userdocs/user_guide/data_objects.rst +++ b/docs/userdocs/user_guide/data_objects.rst @@ -161,20 +161,13 @@ not attribute values. Thus to fetch an attribute you might write, for example one of these : -.. raw:: html +.. testsetup:: -
- -.. code-block:: >>> from ncdata import NcData, NcVariable, NcAttribute >>> dataset = NcData(variables=[NcVariable("var1", attributes={"units": "m"})]) -.. raw:: html - -
- -.. code-block:: +.. doctest:: >>> units1 = dataset.variables['var1'].get_attrval('units') >>> units1 = dataset.variables['var1'].attributes['units'].as_python_value() diff --git a/docs/userdocs/user_guide/howtos.rst b/docs/userdocs/user_guide/howtos.rst index 1e77857..c557d36 100644 --- a/docs/userdocs/user_guide/howtos.rst +++ b/docs/userdocs/user_guide/howtos.rst @@ -7,12 +7,7 @@ documentation to describe concepts and technical details. **"Why Not Just..."** sections highlight warnings for what *not* to do, i.e. wrong turns and gotchas, with brief descriptions of why. - -.. raw:: html - -
- -.. code-block:: +.. testsetup:: >>> import xarray >>> import iris @@ -26,18 +21,13 @@ i.e. wrong turns and gotchas, with brief descriptions of why. ... text = text.replace("\t", " " * 3) ... print(text) -.. raw:: html - -
- - .. _howto_access: Access a variable, dimension, attribute or group ------------------------------------------------ Index by component names to get the object which represents a particular element. -.. code-block:: python +.. doctest:: python >>> from ncdata import NcData, NcAttribute, NcDimension, NcVariable >>> data = NcData( @@ -154,7 +144,7 @@ To get an attribute of a dataset, group or variable, use the :meth:`ncdata.NcData.get_attrval` or :meth:`ncdata.NcVariable.get_attrval` method, which returns either a single (scalar) number, a numeric array, or a string. -.. code-block:: python +.. doctest:: python >>> var = NcVariable("x", attributes={"a": [3.0], "levels": [1., 2, 3]}) >>> var.get_attrval("a") @@ -182,7 +172,7 @@ which produces the same results as the above. For example - .. code-block:: python + .. doctest:: python >>> print(var.attributes["a"].value) [3.] @@ -202,7 +192,7 @@ To set an attribute of a dataset, group or variable, use the All attributes are writeable, and the type can be freely changed. -.. code-block:: python +.. doctest:: python >>> var.set_attrval("x", 3.) NcAttribute('x', 3.0) @@ -220,7 +210,7 @@ assigned value is cast with :func:`numpy.asarray`. For example -.. code-block:: python +.. doctest:: python >>> attr = data.variables["vx"].attributes["q"] >>> attr.value = 4.2 @@ -237,7 +227,7 @@ To create an attribute on a dataset, group or variable, just set its value with This works just like :ref:`howto_write_attr` : i.e. it makes no difference whether the attribute already exists or not. -.. code-block:: python +.. doctest:: python >>> var.set_attrval("x", 3.) NcAttribute('x', 3.0) @@ -259,7 +249,7 @@ variable with a name, dimensions, and optional data and attributes. A minimal example: -.. code-block:: python +.. doctest:: python >>> var = NcVariable("data", ("x_axis",)) >>> print(var) @@ -270,7 +260,7 @@ A minimal example: A more rounded example, including a data array: -.. code-block:: python +.. doctest:: python >>> var = NcVariable("vyx", ("y", "x"), ... data=[[1, 2, 3], [0, 1, 1]], @@ -319,11 +309,7 @@ Read data from a NetCDF file ---------------------------- Use the :func:`ncdata.netcdf4.from_nc4` function to load a dataset from a netCDF file. -.. raw:: html - -
- -.. code-block:: +.. testsetup:: >>> _ds = NcData( ... dimensions=[NcDimension("time", 10)], @@ -334,11 +320,8 @@ Use the :func:`ncdata.netcdf4.from_nc4` function to load a dataset from a netCDF >>> filepath = "_t1.nc" >>> to_nc4(_ds, filepath) -.. raw:: html -
- -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import from_nc4 >>> ds = from_nc4(filepath) @@ -356,7 +339,7 @@ Control chunking in a netCDF read --------------------------------- Use the ``dim_chunks`` argument in the :func:`ncdata.netcdf4.from_nc4` function -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import from_nc4 >>> ds = from_nc4(filepath, dim_chunks={"time": 3}) @@ -368,7 +351,7 @@ Save data to a new file ----------------------- Use the :func:`ncdata.netcdf4.to_nc4` function to write data to a file: -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import to_nc4 >>> to_nc4(data, filepath) @@ -390,7 +373,7 @@ Read from or write to Iris cubes -------------------------------- Use :func:`ncdata.iris.to_iris` and :func:`ncdata.iris.from_iris`. -.. code-block:: python +.. doctest:: python >>> from ncdata.iris import from_iris, to_iris @@ -432,7 +415,7 @@ Read from or write to Xarray datasets ------------------------------------- Use :func:`ncdata.xarray.to_xarray` and :func:`ncdata.xarray.from_xarray`. -.. code-block:: python +.. doctest:: python >>> from ncdata.xarray import from_xarray, to_xarray >>> dataset = xarray.open_dataset(filepath) @@ -457,7 +440,7 @@ Convert data directly from Iris to Xarray, or vice versa Use :func:`ncdata.iris_xarray.cubes_to_xarray` and :func:`ncdata.iris_xarray.cubes_from_xarray`. -.. code-block:: python +.. doctest:: python >>> from ncdata.iris_xarray import cubes_from_xarray, cubes_to_xarray >>> cubes = iris.load(filepath) @@ -472,7 +455,7 @@ or :func:`ncdata.iris.from_iris` then :func:`ncdata.xarray.to_xarray`. Extra keyword controls for the relevant iris and xarray load and save routines can be passed using specific dictionary keywords, e.g. -.. code-block:: python +.. doctest:: python >>> cubes = cubes_from_xarray( ... dataset, @@ -490,11 +473,7 @@ file. Just be careful that any shared dimensions match. -.. raw:: html - -
- -.. code-block:: python +.. testsetup:: python >>> d1 = NcData( ... dimensions=[NcDimension("x", 3)], @@ -511,11 +490,7 @@ Just be careful that any shared dimensions match. >>> to_nc4(d1, "input1.nc") >>> to_nc4(d2, "input2.nc") -.. raw:: html - -
- -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import from_nc4, to_nc4 >>> data1 = from_nc4('input1.nc') @@ -570,7 +545,7 @@ Use the :meth:`NcData() <~ncdata.NcData.__init__>` constructor to create a new d Contents and components can be attached on creation ... -.. code-block:: python +.. doctest:: python >>> data = NcData( ... dimensions=[NcDimension("y", 2), NcDimension("x", 3)], @@ -611,7 +586,7 @@ Contents and components can be attached on creation ... ... or added iteratively ... -.. code-block:: python +.. doctest:: python >>> data2 = NcData() >>> ny, nx = 2, 3 @@ -647,20 +622,12 @@ and re-save as required. For example : -.. raw:: html - -
- -.. code-block:: +.. testsetup:: python >>> # Save the above complex data-example >>> to_nc4(data, "test_data.nc") -.. raw:: html - -
- -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import from_nc4, to_nc4 >>> ds = from_nc4('test_data.nc') @@ -691,11 +658,7 @@ save it with :func:`ncdata.netcdf4.to_nc4`. For a simple case with no groups, it could look something like this: -.. raw:: html - -
- -.. code-block:: +.. testsetup:: python >>> ds = from_nc4("_temp_testdata.nc") >>> ds.variables.add(NcVariable("z", data=[2.])) @@ -704,11 +667,7 @@ For a simple case with no groups, it could look something like this: >>> to_nc4(ds, input_filepath) >>> output_filepath = pathlib.Path("tmp.nc") -.. raw:: html - -
- -.. code-block:: python +.. doctest:: python >>> ds_in = from_nc4(input_filepath) >>> ds_out = NcData() @@ -731,11 +690,7 @@ For a simple case with no groups, it could look something like this: Sometimes it's simpler to load the input, delete content **not** wanted, then re-save. It's perfectly safe to do that, since the original file will be unaffected. -.. raw:: html - -
- -.. code-block:: python +.. testsetup:: python >>> testds = NcData( ... dimensions=[NcDimension("x", 2), NcDimension("pressure", 3)], @@ -748,11 +703,7 @@ It's perfectly safe to do that, since the original file will be unaffected. ... ) >>> to_nc4(testds, input_filepath) -.. raw:: html - -
- -.. code-block:: python +.. doctest:: python >>> data = from_nc4(input_filepath) >>> for varname in ('extra1', 'extra2', 'unwanted'): @@ -770,7 +721,7 @@ to avoid loading problems. For example, to replace an invalid coordinate name in iris input : -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import from_nc4 >>> from ncdata.iris import to_iris @@ -785,7 +736,7 @@ For example, to replace an invalid coordinate name in iris input : or, to replace a mis-used special attribute in xarray input : -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import from_nc4 >>> from ncdata.xarray import to_xarray @@ -805,11 +756,7 @@ would be difficult to overcome if first written to an actual file. For example, to force an additional unlimited dimension in iris output : -.. raw:: html - -
- -.. code-block:: python +.. testsetup:: python >>> from iris.cube import Cube >>> from iris.coords import DimCoord @@ -826,11 +773,7 @@ For example, to force an additional unlimited dimension in iris output : >>> to_nc4(ds, "__xr_tmp.nc") >>> xr_dataset = xarray.open_dataset("__xr_tmp.nc", chunks=-1) -.. raw:: html - -
- -.. code-block:: python +.. doctest:: python >>> from ncdata.iris import from_iris >>> from ncdata.netcdf4 import to_nc4 @@ -840,7 +783,7 @@ For example, to force an additional unlimited dimension in iris output : or, to convert xarray data variable output to masked integers : -.. code-block:: python +.. doctest:: python >>> from numpy import ma >>> from ncdata.xarray import from_xarray @@ -863,11 +806,7 @@ Load a file containing variable-width string variables You must supply a ``dim_chunks`` keyword to the :meth:`ncdata.netcdf4.from_nc4` method, specifying how to chunk all dimension(s) which the "string" type variable uses. -.. raw:: html - -
- -.. code-block:: python +.. testsetup:: python >>> # manufacture a dataset with a "string" variable in it. >>> cdl = """ @@ -886,11 +825,7 @@ specifying how to chunk all dimension(s) which the "string" type variable uses. >>> filepath = "_vlstring_data.nc" >>> ncgen_from_cdl(cdl_str=cdl, cdl_path=None, nc_path=filepath) -.. raw:: html - -
- -.. code-block:: python +.. doctest:: python >>> from ncdata.netcdf4 import from_nc4 >>> # This file has a netcdf "string" type variable, with dimensions ('date',). @@ -907,7 +842,7 @@ more tractable, to work with it effectively. For example, something like this : -.. code-block:: python +.. doctest:: python >>> var = dataset.variables['date_comments'] >>> string_objects = var.data.compute() diff --git a/requirements/readthedocs.yml b/requirements/readthedocs.yml index 8c6e512..a049e87 100644 --- a/requirements/readthedocs.yml +++ b/requirements/readthedocs.yml @@ -16,5 +16,6 @@ dependencies: - sphinx - sphinxcontrib-napoleon - sphinx-copybutton + - sphinx-doctest - towncrier - xarray From 3a7f07c69e3de8eb8eef32d23cd26ac86caf46fa Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 31 Jul 2025 11:26:01 +0100 Subject: [PATCH 12/14] Fixup rtd requirements. --- .readthedocs.yml | 6 +++--- docs/change_log.rst | 36 ++++++++++++++++++++++++++++++++++++ requirements/readthedocs.yml | 1 - 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 8c77c7d..6b3674a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,7 +4,6 @@ build: os: ubuntu-20.04 tools: python: mambaforge-4.10 - jobs: # Content here largely copied from Iris # see : https://github.com/SciTools/iris/pull/4855 @@ -19,9 +18,10 @@ build: pre_install: - git stash post_install: - - sphinx-apidoc -Mfe -o ./docs/api ./lib/ncdata - - towncrier build --yes - git stash pop + pre_build: + - cd docs; make allapi + - cd docs; make towncrier conda: environment: requirements/readthedocs.yml diff --git a/docs/change_log.rst b/docs/change_log.rst index c7628e3..9f3811d 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -23,6 +23,42 @@ Summary of key features by release number: .. towncrier release notes start +Ncdata 0.3.0.dev4+dirty (2025-07-31) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Features +^^^^^^^^ + +- Added regular linkcheck gha. (`ISSUE#123 `_) +- Make :meth:`~ncdata.iris.to_iris` use the full iris load processing, + instead of :meth:`iris.fileformats.netcdf.loader.load_cubes`. + This means you can use load controls such as callbacks and constraints. (`ISSUE#131 `_) + + +Developer and Internal changes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Switch to towncrier for whats-new management. (`ISSUE#116 `_) + + +Ncdata 0.3.0.dev4+dirty (2025-07-31) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Features +^^^^^^^^ + +- Added regular linkcheck gha. (`ISSUE#123 `_) +- Make :meth:`~ncdata.iris.to_iris` use the full iris load processing, + instead of :meth:`iris.fileformats.netcdf.loader.load_cubes`. + This means you can use load controls such as callbacks and constraints. (`ISSUE#131 `_) + + +Developer and Internal changes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Switch to towncrier for whats-new management. (`ISSUE#116 `_) + + v0.2.0 ~~~~~~ Overhauled data manipulation APIs. Expanded and improved documentation. diff --git a/requirements/readthedocs.yml b/requirements/readthedocs.yml index a049e87..8c6e512 100644 --- a/requirements/readthedocs.yml +++ b/requirements/readthedocs.yml @@ -16,6 +16,5 @@ dependencies: - sphinx - sphinxcontrib-napoleon - sphinx-copybutton - - sphinx-doctest - towncrier - xarray From e05c123c8fcebd33fadca54d6e0aad05a2a78dcd Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 31 Jul 2025 12:06:55 +0100 Subject: [PATCH 13/14] Added whatsnew.. --- docs/changelog_fragments/136.dev.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changelog_fragments/136.dev.rst diff --git a/docs/changelog_fragments/136.dev.rst b/docs/changelog_fragments/136.dev.rst new file mode 100644 index 0000000..ed285ca --- /dev/null +++ b/docs/changelog_fragments/136.dev.rst @@ -0,0 +1 @@ +Made all docs examples into doctests; add doctest CI action. From 69ea5afdf31fd1eb0fb0247446bbbee49a67e18b Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 31 Jul 2025 12:21:04 +0100 Subject: [PATCH 14/14] Avoid ncdump tab output problem. --- docs/userdocs/getting_started/introduction.rst | 2 +- docs/userdocs/user_guide/howtos.rst | 6 +----- pyproject.toml | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/userdocs/getting_started/introduction.rst b/docs/userdocs/getting_started/introduction.rst index a2d2965..f3074ae 100644 --- a/docs/userdocs/getting_started/introduction.rst +++ b/docs/userdocs/getting_started/introduction.rst @@ -81,7 +81,7 @@ Simple example: >>> filepath = "./tmp.nc" >>> to_nc4(data, filepath) - >>> ncdump("tmp.nc") # NOTE: function (not shown) calls command-line ncdump + >>> print(check_output("ncdump -h tmp.nc", shell=True).decode()) # doctest: +NORMALIZE_WHITESPACE netcdf tmp { dimensions: x = 3 ; diff --git a/docs/userdocs/user_guide/howtos.rst b/docs/userdocs/user_guide/howtos.rst index c557d36..47bb9e1 100644 --- a/docs/userdocs/user_guide/howtos.rst +++ b/docs/userdocs/user_guide/howtos.rst @@ -16,10 +16,6 @@ i.e. wrong turns and gotchas, with brief descriptions of why. >>> from pprint import pprint >>> import numpy as np >>> from subprocess import check_output - >>> def ncdump(path): - ... text = check_output(f'ncdump -h {path}', shell=True).decode() - ... text = text.replace("\t", " " * 3) - ... print(text) .. _howto_access: @@ -355,7 +351,7 @@ Use the :func:`ncdata.netcdf4.to_nc4` function to write data to a file: >>> from ncdata.netcdf4 import to_nc4 >>> to_nc4(data, filepath) - >>> ncdump(filepath) + >>> print(check_output(f"ncdump -h {filepath}", shell=True).decode()) # doctest: +NORMALIZE_WHITESPACE netcdf ...{ dimensions: x = 3 ; diff --git a/pyproject.toml b/pyproject.toml index 0c7bb8f..a281d95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ Discussions = "https://github.com/pp-mo/ncdata/discussions" Documentation = "https://ncdata.readthedocs.io" Issues = "https://github.com/pp-mo/ncdata/issues" - [tool.setuptools] license-files = ["LICENSE"]