From 907a587941863138d6f7d8b523bb590a7e9af75c Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 31 Oct 2022 12:32:09 +0000 Subject: [PATCH 1/8] Implement split cube attributes. --- lib/iris/cube.py | 207 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index abe37c35fb..6c692f7e65 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -14,6 +14,7 @@ import copy from copy import deepcopy from functools import partial, reduce +import itertools import operator import warnings from xml.dom.minidom import Document @@ -33,6 +34,7 @@ import iris.aux_factory from iris.common import CFVariableMixin, CubeMetadata, metadata_manager_factory from iris.common.metadata import metadata_filter +from iris.common.mixin import LimitedAttributeDict import iris.coord_systems import iris.coords import iris.exceptions @@ -764,6 +766,195 @@ def _is_single_item(testee): return isinstance(testee, str) or not isinstance(testee, Iterable) +class CubeAttrsDict(MutableMapping): + """ + A dict-like object for "Cube.attributes", which provides unified user access to + the combined cube 'local' and 'global' attributes, mimicking the behaviour of a + simple dict. + + This supports all the regular methods of a 'dict', + However, a few things such as the detailed print (repr) are different. + + In addition, the 'locals' and 'globals' properties provide specific access to local + and global attributes, as regular `~iris.common.mixin.LimitedAttributeDict`s. + + Notes + ----- + For type testing, "issubclass(CubeAttrsDict, Mapping)" is True, but + "issubclass(CubeAttrsDict, dict)" is False. + + Properties 'locals' and 'globals' are the two sets of cube attributes. These are + both :class:`~iris.common.mixin.LimitedAttributeDict`. The CubeAttrsDict object + contains *no* additional state of its own, but simply acts as a view on these two. + + When reading (__getitem__, pop. popitem, keys, values etc), it contains all the + keys + values of both 'locals' and 'globals'. When a key occurs in *both* 'locals' + and 'globals', the result is the local value. + + When writing (__setitem__, setdefault, update, etc) to a key already present, the + existing entry in either 'locals' or 'globals' is updated. If both are present, + 'locals' is updated. + + When writing to a new key, this generally updates 'locals'. However, certain + specific names would never normally be 'data' attributes, and these are written as + 'globals' instead. These are determined by Appendix A of the + `_CF Conventions: https://cfconventions.org/` . + At present, these are : 'conventions', 'featureType', 'history', 'title'. + + Examples + -------- + .. doctest:: + :hide: + >>> cube = Cube([0]) + + >>> cube.attributes.update({'history':'fresh', 'x':3}) + >>> print(cube.attributes) + {'history': 'fresh', 'x': 3} + >>> print(repr(cube.attributes)) + CubeAttrsDict(globals={'history': 'fresh'}, locals={'x': 3}) + + >>> cube.attributes['history'] += '+added' + >>> print(repr(cube.attributes)) + CubeAttrsDict(globals={'history': 'fresh+added'}, locals={'x': 3}) + + >>> cube.attributes.locals['history'] = 'per-var' + >>> print(cube.attributes) + {'history': 'per-var', 'x': 3} + >>> print(repr(cube.attributes)) + CubeAttrsDict(globals={'history': 'fresh+added'}, locals={'x': 3, 'history': 'per-var'}) + + """ + + def __init__(self, combined=None, locals=None, globals=None): + # Allow initialisation from a generic dictionary, or local/global specific ones. + # Initialise local + global, defaulting to empty. + # N.B. this is also the way to achieve copy-less construction. + self.locals = locals + self.globals = globals + # Update with combined, if present. + if combined: + # Treat a single input as a 'copy' operation. + # N.B. enforce deep copying, consistent with general Iris usage. + if hasattr(combined, "globals") and hasattr(combined, "locals"): + globals = deepcopy(combined.globals) + locals = deepcopy(combined.locals) + self.globals.update(globals) + self.locals.update(locals) + else: + # Convert an arbitrary single input value to a dict, and update. + combined = deepcopy(dict(combined)) + for key, value in combined.items(): + self[key] = value + + # + # Provide a serialisation interface + # + def __getstate__(self): + return (self.locals, self.globals) + + def __setstate__(self, state): + self.locals, self.globals = state + + # + # Ensure that the stored local/global dictionaries are "LimitedAttributeDicts". + # + @property + def locals(self): + return self._locals + + @locals.setter + def locals(self, attributes): + self._locals = LimitedAttributeDict(attributes or {}) + + @property + def globals(self): + return self._globals + + @globals.setter + def globals(self, attributes): + self._globals = LimitedAttributeDict(attributes or {}) + + # + # The remaining methods are sufficient to generate a complete standard Mapping + # API -- see docs for :class:`collections.abc.MutableMapping`. + # + + def __iter__(self): + # Ordering: all global keys, then all local ones, but omitting duplicates. + # NOTE: this means that in the "summary" view, attributes present in both + # locals+globals are listed first, amongst the globals, even though they appear + # with the *value* from locals. + # Otherwise follows order of insertion, as is normal for dicts. + return itertools.chain( + self.globals.keys(), + (x for x in self.locals.keys() if x not in self.globals), + ) + + def __len__(self): + # Return the number of keys in the 'combined' view. + return len(list(iter(self))) + + def __getitem__(self, key): + # Fetch an item from the "combined attributes". + # If both present, the local setting takes priority. + if key in self.locals: + store = self.locals + else: + store = self.globals + return store[key] + + def __setitem__(self, key, value): + # Assign an attribute value. + # Make local or global according to the existing content, or the known set of + # "normally global" attributes given by CF. + + # If an attribute of this name is already present, update that + # (the local one having priority). + if key in self.locals: + store = self.locals + elif key in self.globals: + store = self.globals + else: + # If NO existing attribute, create local unless it is a "known global" one. + from iris.fileformats.netcdf.saver import _CF_GLOBAL_ATTRS + + # Not in existing + if key in _CF_GLOBAL_ATTRS: + store = self.globals + else: + store = self.locals + + store[key] = value + + def __delitem__(self, key): + """ + Remove an attribute + + Delete from both local + global. + + """ + if key in self.locals: + del self.locals[key] + if key in self.globals: + del self.globals[key] + + def __str__(self): + # Print it just like a "normal" dictionary. + # Convert to a normal dict to do that. + return str(dict(self)) + + def __repr__(self): + # Special repr form, showing "real" contents. + return f"CubeAttrsDict(globals={self.globals}, locals={self.locals})" + + # A copy method is wanted, which is *not* provided by MutableMapping + def copy(self): + # Implement as a deepcopy, consistent with general Iris usage. + globals = deepcopy(self.globals) + locals = deepcopy(self.locals) + return CubeAttrsDict(globals=globals, locals=locals) + + class Cube(CFVariableMixin): """ A single Iris cube of data and metadata. @@ -1019,6 +1210,22 @@ def _names(self): """ return self._metadata_manager._names + # + # Ensure that .attributes is always a :class:`CubeAttrsDict`. + # + @property + def attributes(self): + return self._metadata_manager.attributes + + @attributes.setter + def attributes(self, attributes): + """ + An override to CfVariableMixin.attributes.setter, which ensures that Cube + attributes are stored in a way which distinguishes global + local ones. + + """ + self._metadata_manager.attributes = CubeAttrsDict(attributes) + def _dimensional_metadata(self, name_or_dimensional_metadata): """ Return a single _DimensionalMetadata instance that matches the given From e580fea95f0467307d853f3af7135d263d595b3f Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 31 Oct 2022 15:15:31 +0000 Subject: [PATCH 2/8] Test fixes. --- lib/iris/common/metadata.py | 6 +++- lib/iris/cube.py | 59 +++++++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/lib/iris/common/metadata.py b/lib/iris/common/metadata.py index cb3149fe58..ba9507aa96 100644 --- a/lib/iris/common/metadata.py +++ b/lib/iris/common/metadata.py @@ -242,7 +242,11 @@ def __str__(self): field_strings = [] for field in self._fields: value = getattr(self, field) - if value is None or isinstance(value, (str, dict)) and not value: + if ( + value is None + or isinstance(value, (str, Mapping)) + and not value + ): continue field_strings.append(f"{field}={value}") diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 6c692f7e65..42eb9d3c56 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -846,15 +846,6 @@ def __init__(self, combined=None, locals=None, globals=None): for key, value in combined.items(): self[key] = value - # - # Provide a serialisation interface - # - def __getstate__(self): - return (self.locals, self.globals) - - def __setstate__(self, state): - self.locals, self.globals = state - # # Ensure that the stored local/global dictionaries are "LimitedAttributeDicts". # @@ -874,6 +865,49 @@ def globals(self): def globals(self, attributes): self._globals = LimitedAttributeDict(attributes or {}) + # + # Provide a serialisation interface + # + def __getstate__(self): + return (self.locals, self.globals) + + def __setstate__(self, state): + self.locals, self.globals = state + + # + # Support simple comparison, even when contents are arrays. + # + def __eq__(self, other): + # Copied from :class:`~iris.common.mixin.LimitedAttributeDict` + match = set(self.keys()) == set(other.keys()) + if match: + for key, value in self.items(): + match = value == other[key] + try: + match = bool(match) + except ValueError: + match = match.all() + if not match: + break + return match + + def __ne__(self, other): + return not self == other + + # + # Provide a copy method, as for 'dict', but *not* provided by MutableMapping + # + def copy(self): + """ + Return a copy. + + Implemented as a deepcopy, consistent with general Iris usage. + + """ + globals = deepcopy(self.globals) + locals = deepcopy(self.locals) + return CubeAttrsDict(globals=globals, locals=locals) + # # The remaining methods are sufficient to generate a complete standard Mapping # API -- see docs for :class:`collections.abc.MutableMapping`. @@ -947,13 +981,6 @@ def __repr__(self): # Special repr form, showing "real" contents. return f"CubeAttrsDict(globals={self.globals}, locals={self.locals})" - # A copy method is wanted, which is *not* provided by MutableMapping - def copy(self): - # Implement as a deepcopy, consistent with general Iris usage. - globals = deepcopy(self.globals) - locals = deepcopy(self.locals) - return CubeAttrsDict(globals=globals, locals=locals) - class Cube(CFVariableMixin): """ From fdd42401cf52a4ce5d3dc09061bdf86d194e639b Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 1 Nov 2022 12:01:16 +0000 Subject: [PATCH 3/8] Modify examples for simpler metadata printouts. --- docs/src/further_topics/metadata.rst | 15 ++++++++------- lib/iris/cube.py | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/src/further_topics/metadata.rst b/docs/src/further_topics/metadata.rst index 4c55047d4c..079e95cc99 100644 --- a/docs/src/further_topics/metadata.rst +++ b/docs/src/further_topics/metadata.rst @@ -810,16 +810,17 @@ the ``from_metadata`` class method. For example, given the following .. doctest:: metadata-convert - >>> cube.metadata # doctest: +SKIP - CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) + >>> cube.metadata + CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes=CubeAttrsDict(globals={}, locals={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}), cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) We can easily convert it to a :class:`~iris.common.metadata.DimCoordMetadata` instance using ``from_metadata``, .. doctest:: metadata-convert - >>> DimCoordMetadata.from_metadata(cube.metadata) # doctest: +SKIP - DimCoordMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}, coord_system=None, climatological=None, circular=None) + >>> newmeta = DimCoordMetadata.from_metadata(cube.metadata) + >>> print(newmeta) + DimCoordMetadata(standard_name=air_temperature, var_name=air_temperature, units=K, attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}) By examining :numref:`metadata members table`, we can see that the :class:`~iris.cube.Cube` and :class:`~iris.coords.DimCoord` container @@ -849,9 +850,9 @@ class instance, .. doctest:: metadata-convert - >>> longitude.metadata.from_metadata(cube.metadata) - DimCoordMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}, coord_system=None, climatological=None, circular=None) - + >>> newmeta = longitude.metadata.from_metadata(cube.metadata) + >>> print(newmeta) + DimCoordMetadata(standard_name=air_temperature, var_name=air_temperature, units=K, attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}) .. _metadata assignment: diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 42eb9d3c56..e7916f8db4 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -776,7 +776,7 @@ class CubeAttrsDict(MutableMapping): However, a few things such as the detailed print (repr) are different. In addition, the 'locals' and 'globals' properties provide specific access to local - and global attributes, as regular `~iris.common.mixin.LimitedAttributeDict`s. + and global attributes, as regular :class:`~iris.common.mixin.LimitedAttributeDict`s. Notes ----- @@ -796,7 +796,7 @@ class CubeAttrsDict(MutableMapping): 'locals' is updated. When writing to a new key, this generally updates 'locals'. However, certain - specific names would never normally be 'data' attributes, and these are written as + specific names would never normally be 'data' attributes, and these are created as 'globals' instead. These are determined by Appendix A of the `_CF Conventions: https://cfconventions.org/` . At present, these are : 'conventions', 'featureType', 'history', 'title'. @@ -836,6 +836,7 @@ def __init__(self, combined=None, locals=None, globals=None): # Treat a single input as a 'copy' operation. # N.B. enforce deep copying, consistent with general Iris usage. if hasattr(combined, "globals") and hasattr(combined, "locals"): + # Copy a mapping with globals/locals, like another 'CubeAttrsDict' globals = deepcopy(combined.globals) locals = deepcopy(combined.locals) self.globals.update(globals) From b70b97ffda808e0e32037a6be7fd3ad6cdf01246 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 3 Nov 2022 00:32:26 +0000 Subject: [PATCH 4/8] Added tests, small behaviour fixes. --- lib/iris/cube.py | 81 +++-- lib/iris/tests/unit/cube/test_Cube.py | 28 +- .../tests/unit/cube/test_CubeAttrsDict.py | 284 ++++++++++++++++++ 3 files changed, 359 insertions(+), 34 deletions(-) create mode 100644 lib/iris/tests/unit/cube/test_CubeAttrsDict.py diff --git a/lib/iris/cube.py b/lib/iris/cube.py index e7916f8db4..51e2979a7d 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -10,12 +10,19 @@ """ from collections import OrderedDict -from collections.abc import Container, Iterable, Iterator, MutableMapping import copy from copy import deepcopy from functools import partial, reduce import itertools import operator +from typing import ( + Container, + Iterable, + Iterator, + Mapping, + MutableMapping, + Optional, +) import warnings from xml.dom.minidom import Document import zlib @@ -825,38 +832,56 @@ class CubeAttrsDict(MutableMapping): """ - def __init__(self, combined=None, locals=None, globals=None): + def __init__( + self, + combined: Optional[Mapping] = "__unset", + locals: Optional[Mapping] = None, + globals: Optional[Mapping] = None, + ): # Allow initialisation from a generic dictionary, or local/global specific ones. - # Initialise local + global, defaulting to empty. - # N.B. this is also the way to achieve copy-less construction. + # First initialise locals + globals, defaulting to empty. self.locals = locals self.globals = globals # Update with combined, if present. - if combined: - # Treat a single input as a 'copy' operation. + if not isinstance(combined, str) or combined != "__unset": + # Treat a single input with 'locals' and 'globals' properties as an + # existing CubeAttrsDict, and update from its content. # N.B. enforce deep copying, consistent with general Iris usage. if hasattr(combined, "globals") and hasattr(combined, "locals"): # Copy a mapping with globals/locals, like another 'CubeAttrsDict' - globals = deepcopy(combined.globals) - locals = deepcopy(combined.locals) - self.globals.update(globals) - self.locals.update(locals) + self.globals.update(deepcopy(combined.globals)) + self.locals.update(deepcopy(combined.locals)) else: - # Convert an arbitrary single input value to a dict, and update. - combined = deepcopy(dict(combined)) - for key, value in combined.items(): - self[key] = value + # Treat any arbitrary single input value as a mapping (dict), and + # update from it. + self.update(dict(deepcopy(combined))) # # Ensure that the stored local/global dictionaries are "LimitedAttributeDicts". # + @staticmethod + def _normalise_attrs( + attributes: Optional[Mapping], + ) -> LimitedAttributeDict: + # Convert an input attributes arg into a standard form. + # N.B. content is always a LimitedAttributeDict, and a deep copy of input. + # Allow arg of None, etc. + if not attributes: + attributes = {} + else: + attributes = deepcopy(attributes) + + # Ensure the expected mapping type. + attributes = LimitedAttributeDict(attributes) + return attributes + @property def locals(self): return self._locals @locals.setter def locals(self, attributes): - self._locals = LimitedAttributeDict(attributes or {}) + self._locals = self._normalise_attrs(attributes) @property def globals(self): @@ -864,7 +889,7 @@ def globals(self): @globals.setter def globals(self, attributes): - self._globals = LimitedAttributeDict(attributes or {}) + self._globals = self._normalise_attrs(attributes) # # Provide a serialisation interface @@ -879,18 +904,10 @@ def __setstate__(self, state): # Support simple comparison, even when contents are arrays. # def __eq__(self, other): - # Copied from :class:`~iris.common.mixin.LimitedAttributeDict` - match = set(self.keys()) == set(other.keys()) - if match: - for key, value in self.items(): - match = value == other[key] - try: - match = bool(match) - except ValueError: - match = match.all() - if not match: - break - return match + # For equality, require both globals + locals to match + other = CubeAttrsDict(other) + result = self.locals == other.locals and self.globals == other.globals + return result def __ne__(self, other): return not self == other @@ -902,12 +919,10 @@ def copy(self): """ Return a copy. - Implemented as a deepcopy, consistent with general Iris usage. + Implemented with deep copying, consistent with general Iris usage. """ - globals = deepcopy(self.globals) - locals = deepcopy(self.locals) - return CubeAttrsDict(globals=globals, locals=locals) + return CubeAttrsDict(globals=self.globals, locals=self.locals) # # The remaining methods are sufficient to generate a complete standard Mapping @@ -1252,7 +1267,7 @@ def attributes(self, attributes): attributes are stored in a way which distinguishes global + local ones. """ - self._metadata_manager.attributes = CubeAttrsDict(attributes) + self._metadata_manager.attributes = CubeAttrsDict(attributes or {}) def _dimensional_metadata(self, name_or_dimensional_metadata): """ diff --git a/lib/iris/tests/unit/cube/test_Cube.py b/lib/iris/tests/unit/cube/test_Cube.py index 8e9e00dce8..e0b468bb34 100644 --- a/lib/iris/tests/unit/cube/test_Cube.py +++ b/lib/iris/tests/unit/cube/test_Cube.py @@ -32,7 +32,7 @@ CellMethod, DimCoord, ) -from iris.cube import Cube +from iris.cube import Cube, CubeAttrsDict import iris.exceptions from iris.exceptions import ( AncillaryVariableNotFoundError, @@ -3213,5 +3213,31 @@ def test_fail_assign_duckcellmethod(self): self.cube.cell_methods = (test_object,) +class TestAttributesProperty: + def test_attrs_type(self): + # Cube attributes are always of a special dictionary type. + cube = Cube([0], attributes={"a": 1}) + assert type(cube.attributes) == CubeAttrsDict + assert cube.attributes == {"a": 1} + + def test_attrs_remove(self): + # Wiping attributes replaces the stored object + cube = Cube([0], attributes={"a": 1}) + attrs = cube.attributes + cube.attributes = None + assert cube.attributes is not attrs + assert type(cube.attributes) == CubeAttrsDict + assert cube.attributes == {} + + def test_attrs_clear(self): + # Clearing attributes leaves the same object + cube = Cube([0], attributes={"a": 1}) + attrs = cube.attributes + cube.attributes.clear() + assert cube.attributes is attrs + assert type(cube.attributes) == CubeAttrsDict + assert cube.attributes == {} + + if __name__ == "__main__": tests.main() diff --git a/lib/iris/tests/unit/cube/test_CubeAttrsDict.py b/lib/iris/tests/unit/cube/test_CubeAttrsDict.py new file mode 100644 index 0000000000..87112b8618 --- /dev/null +++ b/lib/iris/tests/unit/cube/test_CubeAttrsDict.py @@ -0,0 +1,284 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +"""Unit tests for the `iris.cube.CubeAttrsDict` class.""" + +import pickle + +import numpy as np +import pytest + +from iris.common.mixin import LimitedAttributeDict +from iris.cube import CubeAttrsDict + + +@pytest.fixture +def sample_attrs() -> CubeAttrsDict: + return CubeAttrsDict( + locals={"a": 1, "z": "this"}, globals={"b": 2, "z": "that"} + ) + + +def check_content(attrs, locals=None, globals=None, matches=None): + # Check a CubeAttrsDict for expected properties. + # If locals/globals are set (including =None), test for equality and not identity. + assert isinstance(attrs, CubeAttrsDict) + attr_locals, attr_globals = attrs.locals, attrs.globals + assert type(attr_locals) == LimitedAttributeDict + assert type(attr_globals) == LimitedAttributeDict + if matches: + locals, globals = matches.locals, matches.globals + + def check(arg, content): + if not arg: + arg = {} + if not isinstance(arg, LimitedAttributeDict): + arg = LimitedAttributeDict(arg) + # N.B. if 'arg' is an actual given LimitedAttributeDict, it is not changed.. + # .. we proceed to ensure that the stored content is equal but NOT the same + assert content == arg + assert content is not arg + + check(locals, attr_locals) + check(globals, attr_globals) + + +class Test___init__: + def test_empty(self): + attrs = CubeAttrsDict() + check_content(attrs, None, None) + + def test_from_combined_dict(self): + attrs = CubeAttrsDict({"q": 3, "history": "something"}) + check_content(attrs, locals={"q": 3}, globals={"history": "something"}) + + def test_from_separate_dicts(self): + locals = {"q": 3} + globals = {"history": "something"} + attrs = CubeAttrsDict(locals=locals, globals=globals) + check_content(attrs, locals=locals, globals=globals) + + def test_from_cubeattrsdict(self, sample_attrs): + result = CubeAttrsDict(sample_attrs) + check_content( + result, locals=sample_attrs.locals, globals=sample_attrs.globals + ) + + def test_from_cubeattrsdict_like(self): + class MyDict: + pass + + mydict = MyDict() + locals, globals = {"a": 1}, {"b": 2} + mydict.locals = locals + mydict.globals = globals + attrs = CubeAttrsDict(mydict) + check_content(attrs, locals=locals, globals=globals) + + +class Test_OddMethods: + def test_pickle(self, sample_attrs): + bytes = pickle.dumps(sample_attrs) + result = pickle.loads(bytes) + check_content(result, matches=sample_attrs) + + def test_clear(self, sample_attrs): + sample_attrs.clear() + check_content(sample_attrs, {}, {}) + + def test_del(self, sample_attrs): + # 'z' is in boht locals+globals. Delete removes both. + assert "z" in sample_attrs.keys() + del sample_attrs["z"] + assert "z" not in sample_attrs.keys() + + def test_copy(self, sample_attrs): + copy = sample_attrs.copy() + assert copy is not sample_attrs + check_content(copy, matches=sample_attrs) + + def test_update(self, sample_attrs): + updated = sample_attrs.copy() + updated.update({"q": 77}) + expected_locals = sample_attrs.locals.copy() + expected_locals["q"] = 77 + check_content( + updated, globals=sample_attrs.globals, locals=expected_locals + ) + + def test_to_dict(self, sample_attrs): + result = dict(sample_attrs) + expected = sample_attrs.globals.copy() + expected.update(sample_attrs.locals) + assert result == expected + + def test_array_copies(self): + array = np.array([3, 2, 1, 4]) + map = {"array": array} + attrs = CubeAttrsDict(map) + check_content(attrs, globals=None, locals=map) + attrs_array = attrs["array"] + assert np.all(attrs_array == array) + assert attrs_array is not array + + def test__str__(self, sample_attrs): + result = str(sample_attrs) + assert result == "{'b': 2, 'z': 'this', 'a': 1}" + + def test__repr__(self, sample_attrs): + result = repr(sample_attrs) + expected = ( + "CubeAttrsDict(" + "globals={'b': 2, 'z': 'that'}, " + "locals={'a': 1, 'z': 'this'})" + ) + assert result == expected + + +class TestEq: + def test_eq_empty(self): + attrs_1 = CubeAttrsDict() + attrs_2 = CubeAttrsDict() + assert attrs_1 == attrs_2 + + def test_eq_nonempty(self, sample_attrs): + attrs_1 = sample_attrs + attrs_2 = sample_attrs.copy() + assert attrs_1 == attrs_2 + + @pytest.mark.parametrize("aspect", ["locals", "globals"]) + def test_ne_missing(self, sample_attrs, aspect): + attrs_1 = sample_attrs + attrs_2 = sample_attrs.copy() + del getattr(attrs_2, aspect)["z"] + assert attrs_1 != attrs_2 + assert attrs_2 != attrs_1 + + @pytest.mark.parametrize("aspect", ["locals", "globals"]) + def test_ne_different(self, sample_attrs, aspect): + attrs_1 = sample_attrs + attrs_2 = sample_attrs.copy() + getattr(attrs_2, aspect)["z"] = 99 + assert attrs_1 != attrs_2 + assert attrs_2 != attrs_1 + + def test_ne_locals_vs_globals(self): + attrs_1 = CubeAttrsDict(locals={"a": 1}) + attrs_2 = CubeAttrsDict(globals={"a": 1}) + assert attrs_1 != attrs_2 + assert attrs_2 != attrs_1 + + def test_eq_dict(self): + # A CubeAttrsDict can be equal to a plain dictionary (which would create it) + vals_dict = {"a": 1, "b": 2, "history": "this"} + attrs = CubeAttrsDict(vals_dict) + assert attrs == vals_dict + assert vals_dict == attrs + + def test_ne_dict_local_global(self): + # Dictionary equivalence fails if the local/global assignments are wrong. + # sample dictionary + vals_dict = {"title": "b"} + # these attrs are *not* the same, because 'title' is global by default + attrs = CubeAttrsDict(locals={"title": "b"}) + assert attrs != vals_dict + assert vals_dict != attrs + + def test_empty_not_none(self): + # An empty CubeAttrsDict is not None, and does not compare to 'None' + # N.B. this for compatibility with the LimitedAttributeDict + attrs = CubeAttrsDict() + assert attrs is not None + with pytest.raises(TypeError, match="iterable"): + # Cannot *compare* to None (or anything non-iterable) + # N.B. not actually testing against None, as it upsets black (!) + attrs == 0 + + def test_empty_eq_iterables(self): + # An empty CubeAttrsDict is "equal" to various empty containers + attrs = CubeAttrsDict() + assert attrs == {} + assert attrs == [] + assert attrs == () + + +class TestDictOrderBehaviour: + def test_ordering(self): + attrs = CubeAttrsDict({"a": 1, "b": 2}) + assert list(attrs.keys()) == ["a", "b"] + # Remove, then reinstate 'a' : it will go to the back + del attrs["a"] + attrs["a"] = 1 + assert list(attrs.keys()) == ["b", "a"] + + def test_globals_locals_ordering(self): + # create attrs with a global attribute set *before* a local one .. + attrs = CubeAttrsDict() + attrs.globals.update(dict(a=1, m=3)) + attrs.locals.update(dict(f=7, z=4)) + # .. and check key order of combined attrs + assert list(attrs.keys()) == ["a", "m", "f", "z"] + # create the "same" thing with locals before globals, *and* different key order + attrs = CubeAttrsDict() + attrs.locals.update(dict(z=4, f=7)) + attrs.globals.update(dict(m=3, a=1)) + # .. this shows that the result is not affected either by alphabetical key + # order, or the order of adding locals/globals + # I.E. result is globals-in-create-order, then locals-in-create-order + assert list(attrs.keys()) == ["m", "a", "z", "f"] + + +class TestSettingBehaviours: + def test_add_localtype(self): + attrs = CubeAttrsDict() + attrs["z"] = 3 + check_content(attrs, locals={"z": 3}) + + def test_add_globaltype(self): + attrs = CubeAttrsDict() + attrs["history"] = "this" + check_content(attrs, globals={"history": "this"}) + + def test_overwrite_local(self): + attrs = CubeAttrsDict(locals={"a": 1}) + attrs["a"] = "this" + check_content(attrs, locals={"a": "this"}) + + def test_overwrite_global(self): + attrs = CubeAttrsDict(globals={"a": 1}) + # Since a global attr exists, it will write that instead of creating a local + attrs["a"] = "this" + check_content(attrs, globals={"a": "this"}) + + def test_overwrite_forced_local(self): + attrs = CubeAttrsDict(locals={"history": 1}) + # The attr remains local, even though it would be created global by default + attrs["history"] = 2 + check_content(attrs, locals={"history": 2}) + + def test_overwrite_forced_global(self): + attrs = CubeAttrsDict(globals={"data": 1}) + # The attr remains global, even though it would be created global by default + attrs["data"] = 2 + check_content(attrs, globals={"data": 2}) + + def test_overwrite_both(self): + attrs = CubeAttrsDict(locals={"z": 1}, globals={"z": 1}) + # Where both exits, it will update the local one + attrs["z"] = 2 + check_content(attrs, locals={"z": 2}, globals={"z": 1}) + + def test_local_global_masking(self, sample_attrs): + # initially, local 'z' masks the global one + assert sample_attrs["z"] == sample_attrs.locals["z"] + # remove local, global will show + del sample_attrs.locals["z"] + assert sample_attrs["z"] == sample_attrs.globals["z"] + # re-set local + sample_attrs.locals["z"] = "new" + assert sample_attrs["z"] == "new" + # change the global, makes no difference + sample_attrs.globals["z"] == "other" + assert sample_attrs["z"] == "new" From 3e6a67454368e00a94d1cf9ac520f741b5319931 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 3 Nov 2022 21:29:59 +0000 Subject: [PATCH 5/8] Simplify copy. --- lib/iris/cube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 51e2979a7d..7fed62a554 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -922,7 +922,7 @@ def copy(self): Implemented with deep copying, consistent with general Iris usage. """ - return CubeAttrsDict(globals=self.globals, locals=self.locals) + return CubeAttrsDict(self) # # The remaining methods are sufficient to generate a complete standard Mapping From 3f946a9e2a57a1c89f4eb61a9c92b9d5de381492 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 3 Nov 2022 21:32:41 +0000 Subject: [PATCH 6/8] Fix doctests. --- docs/src/further_topics/metadata.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/src/further_topics/metadata.rst b/docs/src/further_topics/metadata.rst index 079e95cc99..3b2248b8ca 100644 --- a/docs/src/further_topics/metadata.rst +++ b/docs/src/further_topics/metadata.rst @@ -131,7 +131,7 @@ We can easily get all of the associated metadata of the :class:`~iris.cube.Cube` using the ``metadata`` property: >>> cube.metadata - CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) + CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes=CubeAttrsDict(globals={}, locals={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}), cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) We can also inspect the ``metadata`` of the ``longitude`` :class:`~iris.coords.DimCoord` attached to the :class:`~iris.cube.Cube` in the same way: @@ -601,7 +601,7 @@ Now, let's compare the two above instances and see what ``attributes`` member di .. doctest:: richer-metadata - >>> longitude.metadata.difference(metadata) # doctest: +SKIP + >>> longitude.metadata.difference(metadata) DimCoordMetadata(standard_name=None, long_name=None, var_name=None, units=None, attributes=({'neutral face': '😐'}, {'neutral face': '😜', 'upside-down face': '🙃'}), coord_system=None, climatological=None, circular=None) @@ -675,8 +675,8 @@ For example, consider the following :class:`~iris.common.metadata.CubeMetadata`, .. doctest:: metadata-combine - >>> cube.metadata # doctest: +SKIP - CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) + >>> cube.metadata + CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes=CubeAttrsDict(globals={}, locals={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}), cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) We can perform the **identity function** by comparing the metadata with itself, @@ -700,8 +700,8 @@ which is replaced with a **different value**, >>> metadata = cube.metadata._replace(standard_name="air_pressure_at_sea_level") >>> metadata != cube.metadata True - >>> metadata.combine(cube.metadata) # doctest: +SKIP - CubeMetadata(standard_name=None, long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'STASH': STASH(model=1, section=3, item=236), 'source': 'Data from Met Office Unified Model 6.05', 'Model scenario': 'A1B', 'Conventions': 'CF-1.5'}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) + >>> metadata.combine(cube.metadata) + CubeMetadata(standard_name=None, long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Model scenario': 'A1B', 'STASH': STASH(model=1, section=3, item=236), 'Conventions': 'CF-1.5', 'source': 'Data from Met Office Unified Model 6.05'}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) The ``combine`` method combines metadata by performing a **strict** comparison between each of the associated metadata member values, @@ -979,7 +979,7 @@ Indeed, it's also possible to assign to the ``metadata`` property with a >>> longitude.metadata DimCoordMetadata(standard_name='longitude', long_name=None, var_name='longitude', units=Unit('degrees'), attributes={}, coord_system=GeogCS(6371229.0), climatological=False, circular=False) >>> longitude.metadata = cube.metadata - >>> longitude.metadata # doctest: +SKIP + >>> longitude.metadata DimCoordMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Conventions': 'CF-1.5', 'STASH': STASH(model=1, section=3, item=236), 'Model scenario': 'A1B', 'source': 'Data from Met Office Unified Model 6.05'}, coord_system=GeogCS(6371229.0), climatological=False, circular=False) Note that, only **common** metadata members will be assigned new associated From adce99f29ff5c36d1c7a56526d082ae45d7a1e8f Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 3 Nov 2022 23:31:49 +0000 Subject: [PATCH 7/8] Skip doctests with non-replicable outputs (from use of sets). --- docs/src/further_topics/metadata.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/further_topics/metadata.rst b/docs/src/further_topics/metadata.rst index 3b2248b8ca..24b1a01057 100644 --- a/docs/src/further_topics/metadata.rst +++ b/docs/src/further_topics/metadata.rst @@ -601,7 +601,7 @@ Now, let's compare the two above instances and see what ``attributes`` member di .. doctest:: richer-metadata - >>> longitude.metadata.difference(metadata) + >>> longitude.metadata.difference(metadata) # doctest: +SKIP DimCoordMetadata(standard_name=None, long_name=None, var_name=None, units=None, attributes=({'neutral face': '😐'}, {'neutral face': '😜', 'upside-down face': '🙃'}), coord_system=None, climatological=None, circular=None) @@ -700,8 +700,8 @@ which is replaced with a **different value**, >>> metadata = cube.metadata._replace(standard_name="air_pressure_at_sea_level") >>> metadata != cube.metadata True - >>> metadata.combine(cube.metadata) - CubeMetadata(standard_name=None, long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Model scenario': 'A1B', 'STASH': STASH(model=1, section=3, item=236), 'Conventions': 'CF-1.5', 'source': 'Data from Met Office Unified Model 6.05'}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) + >>> metadata.combine(cube.metadata) # doctest: +SKIP + CubeMetadata(standard_name=None, long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'Conventions': 'CF-1.5', 'Model scenario': 'A1B', 'STASH': STASH(model=1, section=3, item=236), 'source': 'Data from Met Office Unified Model 6.05'}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),)) The ``combine`` method combines metadata by performing a **strict** comparison between each of the associated metadata member values, From e9b565fc2c11e9c0573e813578b792f85624a2c6 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Fri, 4 Nov 2022 00:09:42 +0000 Subject: [PATCH 8/8] Tidy test comments, and add extra test. --- .../tests/unit/cube/test_CubeAttrsDict.py | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/lib/iris/tests/unit/cube/test_CubeAttrsDict.py b/lib/iris/tests/unit/cube/test_CubeAttrsDict.py index 87112b8618..29d24fb218 100644 --- a/lib/iris/tests/unit/cube/test_CubeAttrsDict.py +++ b/lib/iris/tests/unit/cube/test_CubeAttrsDict.py @@ -12,6 +12,7 @@ from iris.common.mixin import LimitedAttributeDict from iris.cube import CubeAttrsDict +from iris.fileformats.netcdf.saver import _CF_GLOBAL_ATTRS @pytest.fixture @@ -23,7 +24,7 @@ def sample_attrs() -> CubeAttrsDict: def check_content(attrs, locals=None, globals=None, matches=None): # Check a CubeAttrsDict for expected properties. - # If locals/globals are set (including =None), test for equality and not identity. + # If locals/globals are set, test for equality and non-identity. assert isinstance(attrs, CubeAttrsDict) attr_locals, attr_globals = attrs.locals, attrs.globals assert type(attr_locals) == LimitedAttributeDict @@ -233,13 +234,16 @@ def test_globals_locals_ordering(self): class TestSettingBehaviours: def test_add_localtype(self): attrs = CubeAttrsDict() + # Any attribute not recognised as global should go into 'locals' attrs["z"] = 3 check_content(attrs, locals={"z": 3}) - def test_add_globaltype(self): + @pytest.mark.parametrize("attrname", _CF_GLOBAL_ATTRS) + def test_add_globaltype(self, attrname): + # These specific attributes are recognised as belonging in 'globals' attrs = CubeAttrsDict() - attrs["history"] = "this" - check_content(attrs, globals={"history": "this"}) + attrs[attrname] = "this" + check_content(attrs, globals={attrname: "this"}) def test_overwrite_local(self): attrs = CubeAttrsDict(locals={"a": 1}) @@ -252,21 +256,22 @@ def test_overwrite_global(self): attrs["a"] = "this" check_content(attrs, globals={"a": "this"}) - def test_overwrite_forced_local(self): - attrs = CubeAttrsDict(locals={"history": 1}) - # The attr remains local, even though it would be created global by default - attrs["history"] = 2 - check_content(attrs, locals={"history": 2}) + @pytest.mark.parametrize("global_attrname", _CF_GLOBAL_ATTRS) + def test_overwrite_forced_local(self, global_attrname): + attrs = CubeAttrsDict(locals={global_attrname: 1}) + # The attr *remains* local, even though it would be created global by default + attrs[global_attrname] = 2 + check_content(attrs, locals={global_attrname: 2}) def test_overwrite_forced_global(self): attrs = CubeAttrsDict(globals={"data": 1}) - # The attr remains global, even though it would be created global by default + # The attr remains local, even though it would be created local by default attrs["data"] = 2 check_content(attrs, globals={"data": 2}) def test_overwrite_both(self): attrs = CubeAttrsDict(locals={"z": 1}, globals={"z": 1}) - # Where both exits, it will update the local one + # Where both exist, it will always update the local one attrs["z"] = 2 check_content(attrs, locals={"z": 2}, globals={"z": 1}) @@ -282,3 +287,37 @@ def test_local_global_masking(self, sample_attrs): # change the global, makes no difference sample_attrs.globals["z"] == "other" assert sample_attrs["z"] == "new" + + @pytest.mark.parametrize("globals_or_locals", ("globals", "locals")) + @pytest.mark.parametrize( + "value_type", + ("replace", "emptylist", "emptytuple", "none", "zero", "false"), + ) + def test_replace_subdict(self, globals_or_locals, value_type): + # Writing to attrs.xx always replaces content with a *new* LimitedAttributeDict + locals, globals = {"a": 1}, {"b": 2} + attrs = CubeAttrsDict(locals=locals, globals=globals) + # Snapshot old + write new value, of either locals or globals + old_content = getattr(attrs, globals_or_locals) + value = { + "replace": {"qq": 77}, + "emptytuple": (), + "emptylist": [], + "none": None, + "zero": 0, + "false": False, + }[value_type] + setattr(attrs, globals_or_locals, value) + # check new content is expected type and value + new_content = getattr(attrs, globals_or_locals) + assert isinstance(new_content, LimitedAttributeDict) + assert new_content is not old_content + if value_type != "replace": + value = {} + assert new_content == value + # Check expected whole: i.e. either globals or locals was replaced with value + if globals_or_locals == "globals": + globals = value + else: + locals = value + check_content(attrs, locals=locals, globals=globals)