From f22de6cc2e483f2047add078d9e136e19dea15f6 Mon Sep 17 00:00:00 2001 From: Peter Killick Date: Fri, 13 Oct 2017 14:18:06 +0100 Subject: [PATCH 1/2] Move lbproc_map to avoid circular import --- lib/iris/fileformats/pp.py | 26 ------------------------- lib/iris/fileformats/pp_rules.py | 4 ++-- lib/iris/fileformats/pp_save_rules.py | 2 +- lib/iris/fileformats/rules.py | 28 ++++++++++++++++++++++++++- lib/iris/tests/test_cube_to_pp.py | 7 ++++--- lib/iris/tests/test_netcdf.py | 5 +++-- lib/iris/tests/test_pp_to_cube.py | 8 ++++---- 7 files changed, 41 insertions(+), 39 deletions(-) diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index ad5fab1d66..b00e46989b 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -26,7 +26,6 @@ import abc import collections from copy import deepcopy -import itertools import operator import os import re @@ -222,31 +221,6 @@ 'default': np.dtype('>f4'), } -# LBPROC codes and their English equivalents -LBPROC_PAIRS = ((1, "Difference from another experiment"), - (2, "Difference from zonal (or other spatial) mean"), - (4, "Difference from time mean"), - (8, "X-derivative (d/dx)"), - (16, "Y-derivative (d/dy)"), - (32, "Time derivative (d/dt)"), - (64, "Zonal mean field"), - (128, "Time mean field"), - (256, "Product of two fields"), - (512, "Square root of a field"), - (1024, "Difference between fields at levels BLEV and BRLEV"), - (2048, "Mean over layer between levels BLEV and BRLEV"), - (4096, "Minimum value of field during time period"), - (8192, "Maximum value of field during time period"), - (16384, "Magnitude of a vector, not specifically wind speed"), - (32768, "Log10 of a field"), - (65536, "Variance of a field"), - (131072, "Mean over an ensemble of parallel runs")) - -# lbproc_map is dict mapping lbproc->English and English->lbproc -# essentially a one to one mapping -lbproc_map = {x: y for x, y in - itertools.chain(LBPROC_PAIRS, ((y, x) for x, y in LBPROC_PAIRS))} - class STASH(collections.namedtuple('STASH', 'model section item')): """ diff --git a/lib/iris/fileformats/pp_rules.py b/lib/iris/fileformats/pp_rules.py index a86c5f358e..38404dcff7 100644 --- a/lib/iris/fileformats/pp_rules.py +++ b/lib/iris/fileformats/pp_rules.py @@ -29,7 +29,7 @@ from iris.aux_factory import HybridHeightFactory, HybridPressureFactory from iris.coords import AuxCoord, CellMethod, DimCoord from iris.fileformats.rules import (ConversionMetadata, Factory, Reference, - ReferenceTarget) + ReferenceTarget, lbproc_map) from iris.fileformats.um_cf_map import (LBFC_TO_CF, STASH_TO_CF, STASHCODE_IMPLIED_HEIGHTS) import iris.fileformats.pp @@ -1027,7 +1027,7 @@ def _all_other_rules(f): if unhandled_lbproc: attributes["ukmo__process_flags"] = tuple(sorted( [name - for value, name in six.iteritems(iris.fileformats.pp.lbproc_map) + for value, name in six.iteritems(lbproc_map) if isinstance(value, int) and f.lbproc & value])) if (f.lbsrce % 10000) == 1111: diff --git a/lib/iris/fileformats/pp_save_rules.py b/lib/iris/fileformats/pp_save_rules.py index 8457cf0191..e5f20b70e5 100644 --- a/lib/iris/fileformats/pp_save_rules.py +++ b/lib/iris/fileformats/pp_save_rules.py @@ -24,10 +24,10 @@ import iris from iris.fileformats._ff_cross_references import STASH_TRANS from iris.aux_factory import HybridHeightFactory, HybridPressureFactory -from iris.fileformats.pp import lbproc_map from iris.fileformats.um_cf_map import CF_TO_LBFC from iris.fileformats.rules import (aux_factory, has_aux_factory, + lbproc_map, scalar_cell_method, scalar_coord, vector_coord) diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index 5b2dbc7603..d7521d42ee 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -27,6 +27,7 @@ import collections from contextlib import contextmanager import getpass +import itertools import logging import logging.handlers as handlers import os @@ -50,6 +51,32 @@ ('name', 'transform')) +# LBPROC codes and their English equivalents +LBPROC_PAIRS = ((1, "Difference from another experiment"), + (2, "Difference from zonal (or other spatial) mean"), + (4, "Difference from time mean"), + (8, "X-derivative (d/dx)"), + (16, "Y-derivative (d/dy)"), + (32, "Time derivative (d/dt)"), + (64, "Zonal mean field"), + (128, "Time mean field"), + (256, "Product of two fields"), + (512, "Square root of a field"), + (1024, "Difference between fields at levels BLEV and BRLEV"), + (2048, "Mean over layer between levels BLEV and BRLEV"), + (4096, "Minimum value of field during time period"), + (8192, "Maximum value of field during time period"), + (16384, "Magnitude of a vector, not specifically wind speed"), + (32768, "Log10 of a field"), + (65536, "Variance of a field"), + (131072, "Mean over an ensemble of parallel runs")) + +# lbproc_map is dict mapping lbproc->English and English->lbproc +# essentially a one to one mapping +lbproc_map = {x: y for x, y in + itertools.chain(LBPROC_PAIRS, ((y, x) for x, y in LBPROC_PAIRS))} + + class ConcreteReferenceTarget(object): """Everything you need to make a real Cube for a named reference.""" @@ -911,4 +938,3 @@ def loadcubes_user_callback_wrapper(cube, field, filename): converter=loader.converter, user_callback_wrapper=loadcubes_user_callback_wrapper): yield cube - diff --git a/lib/iris/tests/test_cube_to_pp.py b/lib/iris/tests/test_cube_to_pp.py index fa78f95166..db5f5dd169 100644 --- a/lib/iris/tests/test_cube_to_pp.py +++ b/lib/iris/tests/test_cube_to_pp.py @@ -32,6 +32,7 @@ import iris.coord_systems import iris.fileformats.pp from iris.fileformats.pp import PPField3 +import iris.fileformats.rules from iris.tests import mock import iris.tests.pp as pp import iris.util @@ -239,7 +240,7 @@ def lbproc_from_pp(self, filename): def test_pp_save_rules(self): # Test single process flags - for _, process_desc in iris.fileformats.pp.LBPROC_PAIRS[1:]: + for _, process_desc in iris.fileformats.rules.LBPROC_PAIRS[1:]: # Get basic cube and set process flag manually ll_cube = stock.lat_lon_cube() ll_cube.attributes["ukmo__process_flags"] = (process_desc,) @@ -250,7 +251,7 @@ def test_pp_save_rules(self): # Check the lbproc is what we expect self.assertEqual(self.lbproc_from_pp(temp_filename), - iris.fileformats.pp.lbproc_map[process_desc]) + iris.fileformats.rules.lbproc_map[process_desc]) os.remove(temp_filename) @@ -258,7 +259,7 @@ def test_pp_save_rules(self): multiple_bit_values = ((128, 64), (4096, 1024), (8192, 1024)) # Maps lbproc value to the process flags that should be created - multiple_map = {sum(bits) : [iris.fileformats.pp.lbproc_map[bit] for bit in bits] for bits in multiple_bit_values} + multiple_map = {sum(bits) : [iris.fileformats.rules.lbproc_map[bit] for bit in bits] for bits in multiple_bit_values} for lbproc, descriptions in six.iteritems(multiple_map): ll_cube = stock.lat_lon_cube() diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index 0ae900c3b3..99b4914809 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -41,6 +41,7 @@ import iris.analysis.trajectory import iris.fileformats._pyke_rules.compiled_krb.fc_rules_cf_fc as pyke_rules import iris.fileformats.netcdf +import iris.fileformats.rules import iris.std_names import iris.util import iris.coord_systems as icoord_systems @@ -910,7 +911,7 @@ def test_std_name_lookup_fail(self): class TestNetCDFUKmoProcessFlags(tests.IrisTest): def test_process_flags(self): # Test single process flags - for _, process_desc in iris.fileformats.pp.LBPROC_PAIRS[1:]: + for _, process_desc in iris.fileformats.rules.LBPROC_PAIRS[1:]: # Get basic cube and set process flag manually ll_cube = stock.lat_lon_cube() ll_cube.attributes["ukmo__process_flags"] = (process_desc,) @@ -933,7 +934,7 @@ def test_process_flags(self): multiple_bit_values = ((128, 64), (4096, 1024), (8192, 1024)) # Maps lbproc value to the process flags that should be created - multiple_map = {bits: [iris.fileformats.pp.lbproc_map[bit] for + multiple_map = {bits: [iris.fileformats.rules.lbproc_map[bit] for bit in bits] for bits in multiple_bit_values} for bits, descriptions in six.iteritems(multiple_map): diff --git a/lib/iris/tests/test_pp_to_cube.py b/lib/iris/tests/test_pp_to_cube.py index e4a74da282..687cb74a8a 100644 --- a/lib/iris/tests/test_pp_to_cube.py +++ b/lib/iris/tests/test_pp_to_cube.py @@ -158,7 +158,7 @@ def test_cell_methods(self): 8192: "maximum"} # Make test values as list of single bit values and some multiple bit values - single_bit_values = list(iris.fileformats.pp.LBPROC_PAIRS) + single_bit_values = list(iris.fileformats.rules.LBPROC_PAIRS) multiple_bit_values = [(128 + 32, ""), (4096 + 2096, ""), (8192 + 1024, "")] test_values = list(single_bit_values) + multiple_bit_values @@ -192,7 +192,7 @@ def test_process_flags(self): omit_process_flags_values = (64, 128, 4096, 8192) # Test single flag values - for value, _ in iris.fileformats.pp.LBPROC_PAIRS: + for value, _ in iris.fileformats.rules.LBPROC_PAIRS: f = next(iris.fileformats.pp.load(orig_file)) f.lbproc = value # set value @@ -209,7 +209,7 @@ def test_process_flags(self): self.assertEqual(cube.attributes.get("ukmo__process_flags", None), None) else: # Check ukmo__process_flags attribute contains correct values - self.assertIn(iris.fileformats.pp.lbproc_map[value], cube.attributes["ukmo__process_flags"]) + self.assertIn(iris.fileformats.rules.lbproc_map[value], cube.attributes["ukmo__process_flags"]) os.remove(temp_filename) @@ -217,7 +217,7 @@ def test_process_flags(self): multiple_bit_values = ((128, 32), (4096, 1024), (8192, 1024)) # Maps lbproc value to the process flags that should be created - multiple_map = {sum(x) : [iris.fileformats.pp.lbproc_map[y] for y in x] for x in multiple_bit_values} + multiple_map = {sum(x) : [iris.fileformats.rules.lbproc_map[y] for y in x] for x in multiple_bit_values} for bit_values in multiple_bit_values: f = next(iris.fileformats.pp.load(orig_file)) From 46f6b25f1a71953653c4f5ef1c482e60d463d860 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 16 Oct 2017 13:49:56 +0100 Subject: [PATCH 2/2] Reorganisation of pp lbproc_pairs feature. --- lib/iris/fileformats/_pp_lbproc_pairs.py | 48 ++++++++++++++++++++++++ lib/iris/fileformats/pp.py | 5 +++ lib/iris/fileformats/pp_rules.py | 7 ++-- lib/iris/fileformats/pp_save_rules.py | 4 +- lib/iris/fileformats/rules.py | 27 ------------- lib/iris/tests/test_cube_to_pp.py | 7 ++-- lib/iris/tests/test_netcdf.py | 5 +-- lib/iris/tests/test_pp_to_cube.py | 8 ++-- 8 files changed, 68 insertions(+), 43 deletions(-) create mode 100644 lib/iris/fileformats/_pp_lbproc_pairs.py diff --git a/lib/iris/fileformats/_pp_lbproc_pairs.py b/lib/iris/fileformats/_pp_lbproc_pairs.py new file mode 100644 index 0000000000..28e35a8e86 --- /dev/null +++ b/lib/iris/fileformats/_pp_lbproc_pairs.py @@ -0,0 +1,48 @@ +# (C) British Crown Copyright 2017, Met Office +# +# This file is part of Iris. +# +# Iris is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Iris is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Iris. If not, see . + +from __future__ import (absolute_import, division, print_function) +from six.moves import (filter, input, map, range, zip) # noqa +import six + +import itertools + + +# LBPROC codes and their English equivalents +LBPROC_PAIRS = ((1, "Difference from another experiment"), + (2, "Difference from zonal (or other spatial) mean"), + (4, "Difference from time mean"), + (8, "X-derivative (d/dx)"), + (16, "Y-derivative (d/dy)"), + (32, "Time derivative (d/dt)"), + (64, "Zonal mean field"), + (128, "Time mean field"), + (256, "Product of two fields"), + (512, "Square root of a field"), + (1024, "Difference between fields at levels BLEV and BRLEV"), + (2048, "Mean over layer between levels BLEV and BRLEV"), + (4096, "Minimum value of field during time period"), + (8192, "Maximum value of field during time period"), + (16384, "Magnitude of a vector, not specifically wind speed"), + (32768, "Log10 of a field"), + (65536, "Variance of a field"), + (131072, "Mean over an ensemble of parallel runs")) + +# lbproc_map is dict mapping lbproc->English and English->lbproc +# essentially a one to one mapping +LBPROC_MAP = {x: y for x, y in + itertools.chain(LBPROC_PAIRS, ((y, x) for x, y in LBPROC_PAIRS))} diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index b00e46989b..f9f8069a32 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -42,6 +42,11 @@ import iris.config import iris.fileformats.pp_rules from iris.fileformats.pp_save_rules import verify + +# NOTE: this is for backwards-compatitibility *ONLY* +# We could simply remove it for v2.0 ? +from iris.fileformats._pp_lbproc_pairs import (LBPROC_PAIRS, + LBPROC_MAP as lbproc_map) import iris.fileformats.rules import iris.coord_systems diff --git a/lib/iris/fileformats/pp_rules.py b/lib/iris/fileformats/pp_rules.py index 38404dcff7..f709ac30c9 100644 --- a/lib/iris/fileformats/pp_rules.py +++ b/lib/iris/fileformats/pp_rules.py @@ -29,10 +29,11 @@ from iris.aux_factory import HybridHeightFactory, HybridPressureFactory from iris.coords import AuxCoord, CellMethod, DimCoord from iris.fileformats.rules import (ConversionMetadata, Factory, Reference, - ReferenceTarget, lbproc_map) + ReferenceTarget) +import iris.fileformats.pp +from iris.fileformats._pp_lbproc_pairs import LBPROC_MAP from iris.fileformats.um_cf_map import (LBFC_TO_CF, STASH_TO_CF, STASHCODE_IMPLIED_HEIGHTS) -import iris.fileformats.pp ############################################################################### @@ -1027,7 +1028,7 @@ def _all_other_rules(f): if unhandled_lbproc: attributes["ukmo__process_flags"] = tuple(sorted( [name - for value, name in six.iteritems(lbproc_map) + for value, name in six.iteritems(LBPROC_MAP) if isinstance(value, int) and f.lbproc & value])) if (f.lbsrce % 10000) == 1111: diff --git a/lib/iris/fileformats/pp_save_rules.py b/lib/iris/fileformats/pp_save_rules.py index e5f20b70e5..2d11b757d8 100644 --- a/lib/iris/fileformats/pp_save_rules.py +++ b/lib/iris/fileformats/pp_save_rules.py @@ -25,9 +25,9 @@ from iris.fileformats._ff_cross_references import STASH_TRANS from iris.aux_factory import HybridHeightFactory, HybridPressureFactory from iris.fileformats.um_cf_map import CF_TO_LBFC +from iris.fileformats._pp_lbproc_pairs import LBPROC_MAP from iris.fileformats.rules import (aux_factory, has_aux_factory, - lbproc_map, scalar_cell_method, scalar_coord, vector_coord) @@ -568,7 +568,7 @@ def _lbproc_rules(cube, pp): pp.lbproc = 0 if cube.attributes.get("ukmo__process_flags", None): - pp.lbproc += sum([lbproc_map[name] + pp.lbproc += sum([LBPROC_MAP[name] for name in cube.attributes["ukmo__process_flags"]]) # Zonal-mean: look for a CellMethod which is a "mean" over "longitude" or diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index d7521d42ee..d156fa8b9e 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -27,7 +27,6 @@ import collections from contextlib import contextmanager import getpass -import itertools import logging import logging.handlers as handlers import os @@ -51,32 +50,6 @@ ('name', 'transform')) -# LBPROC codes and their English equivalents -LBPROC_PAIRS = ((1, "Difference from another experiment"), - (2, "Difference from zonal (or other spatial) mean"), - (4, "Difference from time mean"), - (8, "X-derivative (d/dx)"), - (16, "Y-derivative (d/dy)"), - (32, "Time derivative (d/dt)"), - (64, "Zonal mean field"), - (128, "Time mean field"), - (256, "Product of two fields"), - (512, "Square root of a field"), - (1024, "Difference between fields at levels BLEV and BRLEV"), - (2048, "Mean over layer between levels BLEV and BRLEV"), - (4096, "Minimum value of field during time period"), - (8192, "Maximum value of field during time period"), - (16384, "Magnitude of a vector, not specifically wind speed"), - (32768, "Log10 of a field"), - (65536, "Variance of a field"), - (131072, "Mean over an ensemble of parallel runs")) - -# lbproc_map is dict mapping lbproc->English and English->lbproc -# essentially a one to one mapping -lbproc_map = {x: y for x, y in - itertools.chain(LBPROC_PAIRS, ((y, x) for x, y in LBPROC_PAIRS))} - - class ConcreteReferenceTarget(object): """Everything you need to make a real Cube for a named reference.""" diff --git a/lib/iris/tests/test_cube_to_pp.py b/lib/iris/tests/test_cube_to_pp.py index db5f5dd169..fa78f95166 100644 --- a/lib/iris/tests/test_cube_to_pp.py +++ b/lib/iris/tests/test_cube_to_pp.py @@ -32,7 +32,6 @@ import iris.coord_systems import iris.fileformats.pp from iris.fileformats.pp import PPField3 -import iris.fileformats.rules from iris.tests import mock import iris.tests.pp as pp import iris.util @@ -240,7 +239,7 @@ def lbproc_from_pp(self, filename): def test_pp_save_rules(self): # Test single process flags - for _, process_desc in iris.fileformats.rules.LBPROC_PAIRS[1:]: + for _, process_desc in iris.fileformats.pp.LBPROC_PAIRS[1:]: # Get basic cube and set process flag manually ll_cube = stock.lat_lon_cube() ll_cube.attributes["ukmo__process_flags"] = (process_desc,) @@ -251,7 +250,7 @@ def test_pp_save_rules(self): # Check the lbproc is what we expect self.assertEqual(self.lbproc_from_pp(temp_filename), - iris.fileformats.rules.lbproc_map[process_desc]) + iris.fileformats.pp.lbproc_map[process_desc]) os.remove(temp_filename) @@ -259,7 +258,7 @@ def test_pp_save_rules(self): multiple_bit_values = ((128, 64), (4096, 1024), (8192, 1024)) # Maps lbproc value to the process flags that should be created - multiple_map = {sum(bits) : [iris.fileformats.rules.lbproc_map[bit] for bit in bits] for bits in multiple_bit_values} + multiple_map = {sum(bits) : [iris.fileformats.pp.lbproc_map[bit] for bit in bits] for bits in multiple_bit_values} for lbproc, descriptions in six.iteritems(multiple_map): ll_cube = stock.lat_lon_cube() diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index 99b4914809..0ae900c3b3 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -41,7 +41,6 @@ import iris.analysis.trajectory import iris.fileformats._pyke_rules.compiled_krb.fc_rules_cf_fc as pyke_rules import iris.fileformats.netcdf -import iris.fileformats.rules import iris.std_names import iris.util import iris.coord_systems as icoord_systems @@ -911,7 +910,7 @@ def test_std_name_lookup_fail(self): class TestNetCDFUKmoProcessFlags(tests.IrisTest): def test_process_flags(self): # Test single process flags - for _, process_desc in iris.fileformats.rules.LBPROC_PAIRS[1:]: + for _, process_desc in iris.fileformats.pp.LBPROC_PAIRS[1:]: # Get basic cube and set process flag manually ll_cube = stock.lat_lon_cube() ll_cube.attributes["ukmo__process_flags"] = (process_desc,) @@ -934,7 +933,7 @@ def test_process_flags(self): multiple_bit_values = ((128, 64), (4096, 1024), (8192, 1024)) # Maps lbproc value to the process flags that should be created - multiple_map = {bits: [iris.fileformats.rules.lbproc_map[bit] for + multiple_map = {bits: [iris.fileformats.pp.lbproc_map[bit] for bit in bits] for bits in multiple_bit_values} for bits, descriptions in six.iteritems(multiple_map): diff --git a/lib/iris/tests/test_pp_to_cube.py b/lib/iris/tests/test_pp_to_cube.py index 687cb74a8a..e4a74da282 100644 --- a/lib/iris/tests/test_pp_to_cube.py +++ b/lib/iris/tests/test_pp_to_cube.py @@ -158,7 +158,7 @@ def test_cell_methods(self): 8192: "maximum"} # Make test values as list of single bit values and some multiple bit values - single_bit_values = list(iris.fileformats.rules.LBPROC_PAIRS) + single_bit_values = list(iris.fileformats.pp.LBPROC_PAIRS) multiple_bit_values = [(128 + 32, ""), (4096 + 2096, ""), (8192 + 1024, "")] test_values = list(single_bit_values) + multiple_bit_values @@ -192,7 +192,7 @@ def test_process_flags(self): omit_process_flags_values = (64, 128, 4096, 8192) # Test single flag values - for value, _ in iris.fileformats.rules.LBPROC_PAIRS: + for value, _ in iris.fileformats.pp.LBPROC_PAIRS: f = next(iris.fileformats.pp.load(orig_file)) f.lbproc = value # set value @@ -209,7 +209,7 @@ def test_process_flags(self): self.assertEqual(cube.attributes.get("ukmo__process_flags", None), None) else: # Check ukmo__process_flags attribute contains correct values - self.assertIn(iris.fileformats.rules.lbproc_map[value], cube.attributes["ukmo__process_flags"]) + self.assertIn(iris.fileformats.pp.lbproc_map[value], cube.attributes["ukmo__process_flags"]) os.remove(temp_filename) @@ -217,7 +217,7 @@ def test_process_flags(self): multiple_bit_values = ((128, 32), (4096, 1024), (8192, 1024)) # Maps lbproc value to the process flags that should be created - multiple_map = {sum(x) : [iris.fileformats.rules.lbproc_map[y] for y in x] for x in multiple_bit_values} + multiple_map = {sum(x) : [iris.fileformats.pp.lbproc_map[y] for y in x] for x in multiple_bit_values} for bit_values in multiple_bit_values: f = next(iris.fileformats.pp.load(orig_file))