Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ minimum_pre_commit_version: 1.21.0

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v6.0.0
hooks:
# Prevent giant files from being committed.
- id: check-added-large-files
Expand All @@ -27,34 +27,34 @@ repos:
- id: no-commit-to-branch

- repo: https://github.com/codespell-project/codespell
rev: "v2.2.2"
rev: "v2.4.1"
hooks:
- id: codespell
types_or: [asciidoc, python, markdown, rst]
additional_dependencies: [tomli]

- repo: https://github.com/psf/black
rev: 23.3.0
rev: 25.1.0
hooks:
- id: black
pass_filenames: false
args: [--config=./pyproject.toml, .]

- repo: https://github.com/PyCQA/flake8
rev: 7.0.0
rev: 7.3.0
hooks:
- id: flake8
types: [file, python]

- repo: https://github.com/pycqa/isort
rev: 5.12.0
rev: 6.0.1
hooks:
- id: isort
types: [file, python]
args: [--filter-files, --skip=./lib/ncdata/__init__.py]

- repo: https://github.com/asottile/blacken-docs
rev: 1.13.0
rev: 1.19.1
hooks:
- id: blacken-docs
types: [file, rst]
Expand Down
1 change: 1 addition & 0 deletions lib/ncdata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
thus acting as an efficient exchange channel between any of those forms.

"""

# N.B. this file excluded from isort, as we want a specific class order for the docs

from ._core import NameMap, NcAttribute, NcData, NcDimension, NcVariable
Expand Down
17 changes: 9 additions & 8 deletions lib/ncdata/_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
of dimensions referenced by variables.

"""

from functools import wraps
from typing import (
Any,
Expand Down Expand Up @@ -377,7 +378,7 @@ def _addlines_indent(text, indent=""):


# common indent spacing
_indent = " " * 4
_STANDARD_INDENT = " " * 4


class NcData(_AttributeAccessMixin):
Expand Down Expand Up @@ -420,7 +421,7 @@ def _print_content(self) -> str:
class, so it isn't technically an abstract method).
This "NcData._print_content()" is called recursively for groups.
"""
global _indent
global _STANDARD_INDENT # noqa: F824
# Define a header line (always a separate line).
noname = "<'no-name'>"
lines = [f"<NcData: {self.name or noname}"]
Expand All @@ -431,20 +432,20 @@ def _print_content(self) -> str:
if len(els):
if eltype == "attributes":
# Attributes a bit different: #1 add 'globol' to section title.
lines += [f"{_indent}global attributes:"]
lines += [f"{_STANDARD_INDENT}global attributes:"]
# NOTE: #2 show like variable attributes, but *no parent name*.
attrs_lines = [
f":{attr._print_content()}"
for attr in self.attributes.values()
]
lines += _addlines_indent(
"\n".join(attrs_lines), _indent * 2
"\n".join(attrs_lines), _STANDARD_INDENT * 2
)
else:
lines += [f"{_indent}{eltype}:"]
lines += [f"{_STANDARD_INDENT}{eltype}:"]
for el in els.values():
lines += _addlines_indent(
el._print_content(), _indent * 2
el._print_content(), _STANDARD_INDENT * 2
)
lines.append("")

Expand Down Expand Up @@ -589,7 +590,7 @@ def __init__(
# return self.data.shape

def _print_content(self):
global _indent
global _STANDARD_INDENT # noqa: F824
dimstr = ", ".join(self.dimensions)
typestr = str(self.dtype) if self.dtype else "<no-dtype>"
hdr = f"<NcVariable({typestr}): {self.name}({dimstr})"
Expand All @@ -602,7 +603,7 @@ def _print_content(self):
f"{self.name}:{attr._print_content()}"
for attr in self.attributes.values()
]
lines += _addlines_indent("\n".join(attrs_lines), _indent)
lines += _addlines_indent("\n".join(attrs_lines), _STANDARD_INDENT)
lines += [">"]
return "\n".join(lines)

Expand Down
3 changes: 2 additions & 1 deletion lib/ncdata/dataset_like.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
:class:`~ncdata.NcVariable` objects, but emulate the access APIs of
:class:`netCDF4.Dataset` :class:`netCDF4.Dimension` and :class:`netCDF4.Variable`.

This is provided primarily to support a re-use of the :mod:`iris.fileformats.netcdf`
This is provided primarily to support the reuse of the :mod:`iris.fileformats.netcdf`
file format load + save, to convert cubes to + from ncdata objects (and hence,
especially, to convert Iris :class:`~iris.cube.Cube`\s to + from an Xarray
:class:`~xarray.Dataset`
Expand All @@ -29,6 +29,7 @@
complete, so this module may need to be extended, in future, to support other such uses.

"""

from typing import Any, Dict, List

import dask.array as da
Expand Down
1 change: 1 addition & 0 deletions lib/ncdata/iris.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
Convert :class:`~ncdata.NcData`\s to and from Iris :class:`~iris.cube.Cube`\s.

"""

from typing import Any, AnyStr, Dict, Iterable, List, Union

#
Expand Down
1 change: 1 addition & 0 deletions lib/ncdata/iris_xarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
the relevant load/save routines.

"""

import xarray
from iris.cube import CubeList

Expand Down
1 change: 1 addition & 0 deletions lib/ncdata/netcdf4.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
Converts :class:`ncdata.NcData` to and from :class:`netCDF4.Dataset` objects.

"""

from pathlib import Path
from threading import Lock
from typing import Dict, Optional, Union
Expand Down
1 change: 1 addition & 0 deletions lib/ncdata/threadlock_sharing.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
thread-safe. So probably can only be applied at the outer level of an operation.

"""

from contextlib import contextmanager
from unittest import mock

Expand Down
1 change: 1 addition & 0 deletions lib/ncdata/utils/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""General user utility functions."""

from ._compare_nc_datasets import dataset_differences, variable_differences
from ._copy import ncdata_copy
from ._save_errors import save_errors
Expand Down
1 change: 0 additions & 1 deletion lib/ncdata/utils/_compare_nc_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import netCDF4
import netCDF4 as nc
import numpy as np

from ncdata import NcData, NcVariable


Expand Down
2 changes: 1 addition & 1 deletion lib/ncdata/utils/_save_errors.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
"""User utility routines for ncdata."""

from typing import Dict, List, Union

import netCDF4 as nc
import numpy as np

from ncdata import NcData, NcVariable


Expand Down
1 change: 1 addition & 0 deletions tests/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Tests for :mod:`ncdata`."""

from pathlib import Path

testdata_dir = Path(__file__).parent / "testdata"
Expand Down
10 changes: 5 additions & 5 deletions tests/data_testcase_schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
specs. This enables us to perform various translation tests on standard testfiles from
the Iris and Xarray test suites.
"""

from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Union
Expand Down Expand Up @@ -298,7 +299,7 @@ def make_testcase_dataset(filepath, spec):
}

# Define a sequence of standard testfile specs, with suitable param-names.
_Standard_Testcases: Dict[str, Union[Path, dict]] = {}
_STANDARD_TESTCASES: Dict[str, Union[Path, dict]] = {}


# A decorator for spec-generating routines.
Expand All @@ -311,8 +312,7 @@ def standard_testcases_func(func):
A decorator for spec-generating routines. It automatically **calls** the wrapped
function, and adds the results into the global "_Standard_Testcases" dictionary.
"""
global _Standard_Testcases
_Standard_Testcases.update(func())
_STANDARD_TESTCASES.update(func())
return func


Expand Down Expand Up @@ -473,7 +473,7 @@ class TestcaseSchema:
filepath: Path = None


@pytest.fixture(params=list(_Standard_Testcases.keys()))
@pytest.fixture(params=list(_STANDARD_TESTCASES.keys()))
def standard_testcase(request, session_testdir):
"""
Provide a set of "standard" dataset testcases.
Expand All @@ -489,7 +489,7 @@ def standard_testcase(request, session_testdir):
For those not based on a spec, 'spec' is None.
"""
name = request.param
spec = _Standard_Testcases[name]
spec = _STANDARD_TESTCASES[name]
if isinstance(spec, dict):
# Build a temporary testfile from the spec, and pass that out.
filepath = session_testdir / f"sampledata_{name}.nc"
Expand Down
5 changes: 2 additions & 3 deletions tests/integration/equivalence_testing_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
Used by routines in tests/integration which attempt to show that conversions between
ncdata and other types of data preserve information.
"""

import dask.array as da
import iris.mesh
import numpy as np
import pytest

import iris.mesh


def cubes_equal__corrected(c1, c2):
"""
Expand Down Expand Up @@ -127,7 +127,6 @@ def adjust_chunks():
"""
import dask.config as dcfg

global _USE_TINY_CHUNKS, _CHUNKSIZE_SPEC
if _USE_TINY_CHUNKS:
with dcfg.set({"array.chunk-size": _CHUNKSIZE_SPEC}):
yield
Expand Down
3 changes: 2 additions & 1 deletion tests/integration/example_scripts/ex_dataset_print.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
"""Temporary integrational proof-of-concept example for dataset printout."""
import iris

import iris
import ncdata.iris as nci
from ncdata import NcData, NcDimension, NcVariable

from tests import testdata_dir


Expand Down
3 changes: 2 additions & 1 deletion tests/integration/example_scripts/ex_iris_saveto_ncdata.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@

Check that conversion succeeds and print the resulting dataset.
"""
import iris

import iris
from ncdata.iris import from_iris

from tests import testdata_dir


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@

Showing conversion from Xarray to Iris, and back again.
"""

import dask.array as da
import iris
import numpy as np
import xarray as xr

from ncdata.iris_xarray import cubes_from_xarray, cubes_to_xarray

from tests import testdata_dir


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@

Showing loading and saving ncdata to/from netcdf files.
"""

import tempfile
from pathlib import Path
from shutil import rmtree

import netCDF4 as nc
import numpy as np

from ncdata import NcData, NcDimension, NcVariable
from ncdata.netcdf4 import from_nc4, to_nc4
from ncdata.utils import dataset_differences

from tests import testdata_dir


Expand Down
3 changes: 2 additions & 1 deletion tests/integration/test_iris_load_and_save_equivalence.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
(1) check equivalence of cubes : iris.load(file) VS iris.load(ncdata(file))
(2) check equivalence of files : iris -> file VS iris->ncdata->file
"""

from subprocess import check_output

import iris
import pytest

from ncdata.netcdf4 import from_nc4, to_nc4
from ncdata.utils import dataset_differences

from tests.data_testcase_schemas import session_testdir, standard_testcase
from tests.integration.equivalence_testing_utils import (
adjust_chunks,
Expand Down
3 changes: 2 additions & 1 deletion tests/integration/test_iris_xarray_roundtrips.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
(1) check equivalence of cubes : iris.load(file) VS iris.load(ncdata(file))
(2) check equivalence of files : iris -> file VS iris->ncdata->file
"""

from subprocess import check_output

import dask.array as da
Expand All @@ -13,13 +14,13 @@
import numpy as np
import pytest
import xarray

from ncdata.iris import from_iris
from ncdata.iris_xarray import cubes_to_xarray
from ncdata.netcdf4 import from_nc4
from ncdata.threadlock_sharing import lockshare_context
from ncdata.utils import dataset_differences
from ncdata.xarray import from_xarray

from tests.data_testcase_schemas import (
BAD_LOADSAVE_TESTCASES,
session_testdir,
Expand Down
2 changes: 2 additions & 0 deletions tests/integration/test_netcdf_roundtrips.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
"""
Test ncdata.netcdf by checking load-save roundtrips for standard testcases.
"""

from subprocess import check_output

from ncdata.netcdf4 import from_nc4, to_nc4
from ncdata.utils import dataset_differences

from tests.data_testcase_schemas import session_testdir, standard_testcase

# Avoid complaints that the imported fixtures are "unused"
Expand Down
3 changes: 2 additions & 1 deletion tests/integration/test_xarray_load_and_save_equivalence.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
(1) check equivalence of cubes : xarray.load(file) VS xarray.load(ncdata(file))
(2) check equivalence of files : xarray -> file VS xarray->ncdata->file
"""

import pytest
import xarray

from ncdata.netcdf4 import from_nc4, to_nc4
from ncdata.threadlock_sharing import lockshare_context
from ncdata.utils import dataset_differences
from ncdata.xarray import from_xarray, to_xarray

from tests.data_testcase_schemas import (
BAD_LOADSAVE_TESTCASES,
session_testdir,
Expand Down
Loading
Loading