From f9e1b1e44dfdf994a3275ff0ca3ffcb4da9942df Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 19 Dec 2019 15:44:31 +0000 Subject: [PATCH 1/7] ensure flags load/save without units --- lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb | 4 ++++ lib/iris/fileformats/netcdf.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 815d71a5f4..9a468331db 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -1689,6 +1689,10 @@ fc_extras if np.issubdtype(cf_var.dtype, np.str_): attr_units = cf_units._NO_UNIT_STRING + if "flag_values" in cf_var._nc_attrs or "flag_masks" in cf_var._nc_attrs \ + or "flag_meanings" in cf_var._nc_attrs: + attr_units = cf_units._NO_UNIT_STRING + # Get any assoicated calendar for a time reference coordinate. if cf_units.as_unit(attr_units).is_time_reference(): attr_calendar = getattr(cf_var, CF_ATTR_CALENDAR, None) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 08b079c3ed..84c0fa2af3 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -1776,7 +1776,7 @@ def _inner_create_cf_cellmeasure_or_ancil_variable( # Add the data to the CF-netCDF variable. cf_var[:] = data - if dimensional_metadata.units != "unknown": + if dimensional_metadata.units not in("no_unit", "unknown"): _setncattr(cf_var, "units", str(dimensional_metadata.units)) if dimensional_metadata.standard_name is not None: @@ -1942,7 +1942,7 @@ def _create_cf_coord_variable(self, cube, dimension_names, coord): # Deal with CF-netCDF units and standard name. standard_name, long_name, units = self._cf_coord_identity(coord) - if units != "unknown": + if units not in("no_unit", "unknown"): _setncattr(cf_var, "units", units) if standard_name is not None: @@ -2387,7 +2387,7 @@ def store(data, cf_var, fill_value): if cube.long_name: _setncattr(cf_var, "long_name", cube.long_name) - if cube.units != "unknown": + if cube.units not in("no_unit", "unknown"): _setncattr(cf_var, "units", str(cube.units)) # Add the CF-netCDF calendar attribute. From 6c740502e192316d26c387fc5f46121d22a605ac Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 19 Dec 2019 16:13:53 +0000 Subject: [PATCH 2/7] add load test --- lib/iris/tests/test_netcdf.py | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index 91e37dd3a8..35907d07d2 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -296,6 +296,53 @@ def test_ancillary_variables(self): ) self.assertEqual(avs[0], expected) + def test_status_flags(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + variables: + int64 qqv(axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:ancillary_variables = "my_av" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + byte my_av(axv) ; + my_av:long_name = "qq status_flag" ; + my_av:flag_values = 1b, 2b ; + my_av:flag_meanings = "a b" ; + data: + axv = 11, 21, 31; + my_av = 1b, 1b, 2b; + } + """ + self.tmpdir = tempfile.mkdtemp() + cdl_path = os.path.join(self.tmpdir, "tst.cdl") + nc_path = os.path.join(self.tmpdir, "tst.nc") + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(ref_cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(nc_load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + avs = cubes[0].ancillary_variables() + self.assertEqual(len(avs), 1) + expected = AncillaryVariable( + np.ma.array([1, 1, 2], dtype=np.int8), + long_name="qq status_flag", + var_name="my_av", + units="no_unit", + attributes={"flag_values": np.array([1, 2], dtype=np.int8), + "flag_meanings": "a b"}, + ) + self.assertEqual(avs[0], expected) + def test_cell_measures(self): # Note: using a CDL string as a test data reference, rather than a binary file. ref_cdl = """ From 76d28bd9e0625ecf123963706fd65052dc1cdd5d Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 19 Dec 2019 16:28:58 +0000 Subject: [PATCH 3/7] minor fixes --- lib/iris/fileformats/netcdf.py | 6 +++--- lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl | 1 - lib/iris/tests/test_netcdf.py | 6 ++++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 84c0fa2af3..2ba80c33a2 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -1776,7 +1776,7 @@ def _inner_create_cf_cellmeasure_or_ancil_variable( # Add the data to the CF-netCDF variable. cf_var[:] = data - if dimensional_metadata.units not in("no_unit", "unknown"): + if dimensional_metadata.units not in ("no_unit", "unknown"): _setncattr(cf_var, "units", str(dimensional_metadata.units)) if dimensional_metadata.standard_name is not None: @@ -1942,7 +1942,7 @@ def _create_cf_coord_variable(self, cube, dimension_names, coord): # Deal with CF-netCDF units and standard name. standard_name, long_name, units = self._cf_coord_identity(coord) - if units not in("no_unit", "unknown"): + if units not in ("no_unit", "unknown"): _setncattr(cf_var, "units", units) if standard_name is not None: @@ -2387,7 +2387,7 @@ def store(data, cf_var, fill_value): if cube.long_name: _setncattr(cf_var, "long_name", cube.long_name) - if cube.units not in("no_unit", "unknown"): + if cube.units not in ("no_unit", "unknown"): _setncattr(cf_var, "units", str(cube.units)) # Add the CF-netCDF calendar attribute. diff --git a/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl b/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl index e67316b2f7..e01eb1b31a 100644 --- a/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl +++ b/lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl @@ -10,7 +10,6 @@ variables: double dim1(dim1) ; dim1:units = "m" ; char unknown_scalar(string6) ; - unknown_scalar:units = "no_unit" ; // global attributes: :Conventions = "CF-1.7" ; diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index 35907d07d2..96e432714b 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -338,8 +338,10 @@ def test_status_flags(self): long_name="qq status_flag", var_name="my_av", units="no_unit", - attributes={"flag_values": np.array([1, 2], dtype=np.int8), - "flag_meanings": "a b"}, + attributes={ + "flag_values": np.array([1, 2], dtype=np.int8), + "flag_meanings": "a b", + }, ) self.assertEqual(avs[0], expected) From 6274f2a00fca05536eb318ceeba77a9cabafef2d Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Wed, 3 Jun 2020 15:09:33 +0100 Subject: [PATCH 4/7] change method for checking cf_var attributes --- lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 9a468331db..cb1c9f392d 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -1689,8 +1689,7 @@ fc_extras if np.issubdtype(cf_var.dtype, np.str_): attr_units = cf_units._NO_UNIT_STRING - if "flag_values" in cf_var._nc_attrs or "flag_masks" in cf_var._nc_attrs \ - or "flag_meanings" in cf_var._nc_attrs: + if any(hasattr(cf_var.cf_data, name) for name in ("flag_values", "flag_masks", "flag_meanings")): attr_units = cf_units._NO_UNIT_STRING # Get any assoicated calendar for a time reference coordinate. From 1267b8fcba41a72802fec2a05d774f20f03586d0 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Wed, 3 Jun 2020 17:10:38 +0100 Subject: [PATCH 5/7] Fix tests --- lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb | 2 +- .../fc_rules_cf_fc/test_build_auxiliary_coordinate.py | 4 ++-- .../compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py | 2 +- .../fc_rules_cf_fc/test_build_dimension_coordinate.py | 6 +++--- .../compiled_krb/fc_rules_cf_fc/test_get_attr_units.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index cb1c9f392d..7598374a90 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -1689,7 +1689,7 @@ fc_extras if np.issubdtype(cf_var.dtype, np.str_): attr_units = cf_units._NO_UNIT_STRING - if any(hasattr(cf_var.cf_data, name) for name in ("flag_values", "flag_masks", "flag_meanings")): + if any(name in cf_var.cf_data for name in ("flag_values", "flag_masks", "flag_meanings")): attr_units = cf_units._NO_UNIT_STRING # Get any assoicated calendar for a time reference coordinate. diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py index 70d72fb133..1ddf0b8c0c 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py @@ -92,7 +92,7 @@ def _get_per_test_bounds_var(_coord_unused): def _make_array_and_cf_data(cls, dimension_names): shape = tuple(cls.dim_names_lens[name] for name in dimension_names) - cf_data = mock.Mock(_FillValue=None) + cf_data = mock.MagicMock(_FillValue=None) cf_data.chunking = mock.MagicMock(return_value=shape) return np.zeros(shape), cf_data @@ -145,7 +145,7 @@ class TestDtype(tests.IrisTest): def setUp(self): # Create coordinate cf variables and pyke engine. points = np.arange(6).reshape(2, 3) - cf_data = mock.Mock(_FillValue=None) + cf_data = mock.MagicMock(_FillValue=None) cf_data.chunking = mock.MagicMock(return_value=points.shape) self.cf_coord_var = mock.Mock( diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py index 7f6ecb27c2..fc31092b58 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py @@ -28,7 +28,7 @@ def _make_engine(global_attributes=None, standard_name=None, long_name=None): cf_group = mock.Mock(global_attributes=global_attributes) - cf_var = mock.Mock( + cf_var = mock.MagicMock( cf_name='wibble', standard_name=standard_name, long_name=long_name, diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py index 1917034a6b..a67e3baab8 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py @@ -221,7 +221,7 @@ def setUp(self): RulesTestMixin.setUp(self) # Create test coordinate cf variable. points = np.arange(6) - self.cf_coord_var = mock.Mock( + self.cf_coord_var = mock.MagicMock( dimensions=('foo',), cf_name='wibble', standard_name=None, @@ -332,7 +332,7 @@ def setUp(self): def _make_vars(self, points, bounds=None, units='degrees'): points = np.array(points) - self.cf_coord_var = mock.Mock( + self.cf_coord_var = mock.MagicMock( dimensions=('foo',), cf_name='wibble', standard_name=None, @@ -429,7 +429,7 @@ def _make_vars(self, bounds): # Note that for a scalar the shape of the array from # the cf var is (), rather than (1,). points = np.array([0.]) - self.cf_coord_var = mock.Mock( + self.cf_coord_var = mock.MagicMock( dimensions=(), cf_name='wibble', standard_name=None, diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py index c5e36e8d8e..ff90cefa91 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py @@ -29,7 +29,7 @@ def _make_cf_var(global_attributes=None): cf_group = mock.Mock(global_attributes=global_attributes) - cf_var = mock.Mock( + cf_var = mock.MagicMock( cf_name='sound_frequency', standard_name=None, long_name=None, From f9157039ce8b2f2d6a9259de4e7d55f0fe59cb2f Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 4 Jun 2020 16:32:27 +0100 Subject: [PATCH 6/7] change cf_var attribute retrieval from in to hasattr --- lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb | 2 +- .../fc_rules_cf_fc/test_build_auxiliary_coordinate.py | 4 ++-- .../fc_rules_cf_fc/test_build_dimension_coordinate.py | 8 ++++++-- .../compiled_krb/fc_rules_cf_fc/test_get_attr_units.py | 1 + 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 7598374a90..cb1c9f392d 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -1689,7 +1689,7 @@ fc_extras if np.issubdtype(cf_var.dtype, np.str_): attr_units = cf_units._NO_UNIT_STRING - if any(name in cf_var.cf_data for name in ("flag_values", "flag_masks", "flag_meanings")): + if any(hasattr(cf_var.cf_data, name) for name in ("flag_values", "flag_masks", "flag_meanings")): attr_units = cf_units._NO_UNIT_STRING # Get any assoicated calendar for a time reference coordinate. diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py index 1ddf0b8c0c..0811e75b52 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py @@ -92,7 +92,7 @@ def _get_per_test_bounds_var(_coord_unused): def _make_array_and_cf_data(cls, dimension_names): shape = tuple(cls.dim_names_lens[name] for name in dimension_names) - cf_data = mock.MagicMock(_FillValue=None) + cf_data = mock.MagicMock(_FillValue=None, spec=[]) cf_data.chunking = mock.MagicMock(return_value=shape) return np.zeros(shape), cf_data @@ -219,7 +219,7 @@ def setUp(self): scale_factor=1, add_offset=0, cf_name='wibble', - cf_data=mock.MagicMock(chunking=mock.Mock(return_value=None)), + cf_data=mock.MagicMock(chunking=mock.Mock(return_value=None), spec=[]), standard_name=None, long_name='wibble', units='days since 1970-01-01', diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py index a67e3baab8..f330fc2c45 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py @@ -75,6 +75,7 @@ def _set_cf_coord_var(self, points): self.cf_coord_var = mock.Mock( dimensions=('foo',), cf_name='wibble', + cf_data=mock.Mock(spec=[]), standard_name=None, long_name='wibble', units='days since 1970-01-01', @@ -221,11 +222,12 @@ def setUp(self): RulesTestMixin.setUp(self) # Create test coordinate cf variable. points = np.arange(6) - self.cf_coord_var = mock.MagicMock( + self.cf_coord_var = mock.Mock( dimensions=('foo',), cf_name='wibble', standard_name=None, long_name='wibble', + cf_data=mock.Mock(spec=[]), units='m', shape=points.shape, dtype=points.dtype, @@ -337,6 +339,7 @@ def _make_vars(self, points, bounds=None, units='degrees'): cf_name='wibble', standard_name=None, long_name='wibble', + cf_data=mock.Mock(spec=[]), units=units, shape=points.shape, dtype=points.dtype, @@ -429,12 +432,13 @@ def _make_vars(self, bounds): # Note that for a scalar the shape of the array from # the cf var is (), rather than (1,). points = np.array([0.]) - self.cf_coord_var = mock.MagicMock( + self.cf_coord_var = mock.Mock( dimensions=(), cf_name='wibble', standard_name=None, long_name='wibble', units='degrees', + cf_data=mock.Mock(spec=[]), shape=(), dtype=points.dtype, __getitem__=lambda self, key: points[key]) diff --git a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py index ff90cefa91..4df00fe209 100644 --- a/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py +++ b/lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py @@ -31,6 +31,7 @@ def _make_cf_var(global_attributes=None): cf_var = mock.MagicMock( cf_name='sound_frequency', + cf_data=mock.Mock(spec=[]), standard_name=None, long_name=None, units=u'\u266b', From 5665d11d220e64c0ce9a2a2b4a515000417e3765 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 5 Jun 2020 10:50:11 +0100 Subject: [PATCH 7/7] change dependencies --- requirements/core.txt | 6 +++--- requirements/test.txt | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements/core.txt b/requirements/core.txt index c3f5775d7e..c7adb469cc 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -6,9 +6,9 @@ cartopy>=0.12 #conda: proj4<6 cf-units>=2 -cftime +cftime<1.1 dask[array]>=2 #conda: dask>=2 -matplotlib +matplotlib<=3.1 netcdf4 -numpy>=1.14 +numpy<=1.17 scipy diff --git a/requirements/test.txt b/requirements/test.txt index 89358f7f76..89cd043fa1 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -3,7 +3,8 @@ black==19.10b0 #conda: black=19.10b0 filelock -imagehash>=4.0 +pillow<7 +imagehash nose pre-commit requests