Skip to content

Cube merge string coords. #475

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
esc24 merged 1 commit into from
May 1, 2013
Merged

Cube merge string coords. #475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 53 additions & 31 deletions lib/iris/_merge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# (C) British Crown Copyright 2010 - 2012, Met Office
# (C) British Crown Copyright 2010 - 2013, Met Office
#
# This file is part of Iris.
#
Expand Down Expand Up @@ -582,7 +582,7 @@ def _is_dependent(dependent, independent, positions, function_mapping=None):
return valid


def _derive_separable_consistent_groups(relation_matrix, separable_group):
def _derive_consistent_groups(relation_matrix, separable_group):
"""
Determine the largest combinations of candidate dimensions within the
separable group that are self consistently separable from one another.
Expand Down Expand Up @@ -656,7 +656,7 @@ def _build_separable_group(space, group, separable_consistent_groups, positions,
participates in a functional relationship.

Returns:
None.
Boolean.

"""
valid = False
Expand All @@ -683,8 +683,8 @@ def _build_separable_group(space, group, separable_consistent_groups, positions,
if valid:
space.update({name: None for name in independent})
space.update({name: tuple(independent) for name in dependent})
else:
raise iris.exceptions.NotYetImplementedError('No functional relationship between separable and inseparable candidate dimensions.')

return valid


def _build_inseparable_group(space, group, positions, function_matrix):
Expand Down Expand Up @@ -820,7 +820,8 @@ def derive_space(groups, relation_matrix, positions, function_matrix=None):
participates in a functional relationship.

Returns:
A space dictionary describing the relationship between each candidate dimension.
A space dictionary describing the relationship between each
candidate dimension.

"""
space = {}
Expand All @@ -829,22 +830,33 @@ def derive_space(groups, relation_matrix, positions, function_matrix=None):
separable_group = _derive_separable_group(relation_matrix, group)

if len(group) == 1 and not separable_group:
# This single candidate dimension is separable from all other candidate dimensions
# in the group, therefore it is a genuine dimension of the space.
# This single candidate dimension is separable from all other
# candidate dimensions in the group, therefore it is a genuine
# dimension of the space.
space.update({name: None for name in group})
elif separable_group:
# Determine the largest combination of the candidate dimensions
# Determine the largest combination of the candidate dimensions
# in the separable group that are consistently separable.
separable_consistent_groups = _derive_separable_consistent_groups(relation_matrix, separable_group)
_build_separable_group(space, group, separable_consistent_groups, positions, function_matrix)
consistent_groups = _derive_consistent_groups(relation_matrix,
separable_group)
if not _build_separable_group(space, group, consistent_groups,
positions, function_matrix):
# There is no relationship between any of the candidate
# dimensions in the separable group, so merge them together
# into a new combined dimesion of the space.
_build_combination_group(space, group,
positions, function_matrix)
else:
# Determine whether there is a scalar relationship between one of
# the candidate dimensions and each of the other candidate dimensions
# in this inseparable group.
if not _build_inseparable_group(space, group, positions, function_matrix):
# There is no relationship between any of the candidate dimensions in this
# inseparable group, so merge them together into a new combined dimension of the space.
_build_combination_group(space, group, positions, function_matrix)
# the candidate dimensions and each of the other candidate
# dimensions in this inseparable group.
if not _build_inseparable_group(space, group,
positions, function_matrix):
# There is no relationship between any of the candidate
# dimensions in this inseparable group, so merge them together
# into a new combined dimension of the space.
_build_combination_group(space, group,
positions, function_matrix)

return space

Expand Down Expand Up @@ -1064,7 +1076,8 @@ def _define_space(self, space, positions, indexes, function_matrix):
dim_by_name[name] = len(self._shape)
self._nd_names.append(name)
self._shape.append(len(cells))
self._cache_by_name[name] = {cell:index for index, cell in enumerate(cells)}
self._cache_by_name[name] = {cell: index for index, cell \
in enumerate(cells)}
else:
# TODO: Consider appropriate sort order (ascending, decending) i.e. use CF positive attribute.
cells = sorted(indexes[name])
Expand All @@ -1086,7 +1099,8 @@ def _define_space(self, space, positions, indexes, function_matrix):
else:
self._dim_templates.append(_Template(dim, points, bounds, kwargs))
self._shape.append(len(cells))
self._cache_by_name[name] = {cell:index for index, cell in enumerate(cells)}
self._cache_by_name[name] = {cell: index for index, cell \
in enumerate(cells)}

# Second pass - Build the auxiliary coordinate templates for the space.
for name in names:
Expand Down Expand Up @@ -1202,10 +1216,10 @@ def _build_coordinates(self):

# Build the dimension coordinates.
for template in self._dim_templates:
# sometimes its not possible to build a dim coord (for example,
# if your bounds are not monotonic, so try building the coordinate,
# and if it fails make the coordinate into a aux coord. This will
# ultimately make an anonymous dimension.
# Sometimes it's not possible to build a dim coordinate e.g.
# the bounds are not monotonic, so try building the coordinate,
# and if it fails make the coordinate into an auxiliary coordinate.
# This will ultimately make an anonymous dimension.
try:
coord = iris.coords.DimCoord(template.points,
bounds=template.bounds,
Expand All @@ -1214,10 +1228,10 @@ def _build_coordinates(self):
except ValueError:
self._aux_templates.append(template)

# there is the potential that there are still anonymous dimensions
# get a list of the dimensions which are not anonymous at this stage
covered_dims = [dim_coord_and_dim.dims
for dim_coord_and_dim in dim_coords_and_dims]
# There is the potential that there are still anonymous dimensions.
# Get a list of the dimensions which are not anonymous at this stage.
covered_dims = [dim_coord_and_dim.dims \
for dim_coord_and_dim in dim_coords_and_dims]

# Build the auxiliary coordinates.
for template in self._aux_templates:
Expand Down Expand Up @@ -1271,6 +1285,10 @@ def _add_cube(self, cube, coord_payload):
"""Create and add the source-cube skeleton to the ProtoCube."""

skeleton = _Skeleton(coord_payload.scalar.values, cube._data)
# Attempt to do something sensible with mixed scalar dtypes.
Copy link
Member Author

@bjlittle bjlittle Apr 30, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, this is pretty interesting ... I seem to have uncovered a grey area in coordinate signature comparison between cubes for scalar coordinates.

When it comes to comparing the coordinate signature for scalar coordinates, cube merge is really pretty relaxed. It only compares scalar coordinate definitions and that's all. It does not consider differences in scalar coordinate meta-data i.e. cell point dtype, bounds dtype, whether both cells are bounded, and the circular flag for DimCoord scalar coordinates.

This change here, is a pretty small step towards firming up this loose scalar coordinate comparison between different cubes in the light of string scalar coordinates (which are not bounded).

I don't think this PR is the place to nail down the contract for scalar coordinate comparison. I'm happy to let this sleeping dog lie at the moment, what do you suggest @rhattersley ?

Copy link
Member

@rhattersley rhattersley Apr 30, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, let's just record the issue (preferably with a code snippet to produce a problem) and move on.

Copy link
Member Author

@bjlittle bjlittle May 1, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Raised issue #484 to cover this.

for i, metadata in enumerate(coord_payload.scalar.metadata):
if metadata.points_dtype > self._coord_metadata[i].points_dtype:
self._coord_metadata[i] = metadata
self._skeletons.append(skeleton)

def _extract_coord_payload(self, cube):
Expand Down Expand Up @@ -1301,10 +1319,14 @@ def _extract_coord_payload(self, cube):
hint_dict = {name: i for i, name in zip(range(len(self._hints), 0, -1), self._hints[::-1])}
# Coordinate axis ordering dictionary.
axis_dict = {'T': 0, 'Z': 1, 'Y': 2, 'X': 3}
# Coordinate sort function - by coordinate hint, then by guessed coordinate axis, then
# by coordinate definition, in ascending order.
key_func = lambda coord: (hint_dict.get(coord.name(), len(hint_dict) + 1),
axis_dict.get(iris.util.guess_coord_axis(coord), len(axis_dict) + 1),
# Coordinate sort function.
key_func = lambda coord: (not np.issubdtype(coord.points.dtype,
np.number),
not isinstance(coord, iris.coords.DimCoord),
hint_dict.get(coord.name(),
len(hint_dict) + 1),
axis_dict.get(iris.util.guess_coord_axis(coord),
len(axis_dict) + 1),
coord._as_defn())

# Order the coordinates by hints, axis, and definition.
Expand Down
4 changes: 2 additions & 2 deletions lib/iris/etc/pp_rules.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# (C) British Crown Copyright 2010 - 2012, Met Office
# (C) British Crown Copyright 2010 - 2013, Met Office
#
# This file is part of Iris.
#
Expand Down Expand Up @@ -356,7 +356,7 @@ f.lbvc == 65
THEN
#need orography field to calculate 3D array height coord
###, coord_system=HybridHeightCS(Reference('orography')))
CoordAndDims(AuxCoord(f.lblev, standard_name='model_level_number', attributes={'positive': 'up'}))
CoordAndDims(DimCoord(f.lblev, standard_name='model_level_number', attributes={'positive': 'up'}))
CoordAndDims(DimCoord(f.blev, long_name='level_height', units='m', bounds=[f.brlev, f.brsvd[0]], attributes={'positive': 'up'}))
CoordAndDims(AuxCoord(f.bhlev, long_name='sigma', bounds=[f.bhrlev, f.brsvd[1]]))
Factory(HybridHeightFactory, [{'long_name': 'level_height'}, {'long_name': 'sigma'}, Reference('orography')])
Expand Down
Loading