From c3bd8ce42f603c51cc78632f6841f85fa2f8c083 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 21 Jan 2014 09:24:26 -0800 Subject: [PATCH] Remove deepcopies when slicing or manipulating cubes This is a first attempt at removing unnecessary (and very slow) deepcopy operations with slicing or otherwise manipulating cubes and coordinates. See #914. Note: A few of the unit tests are failing, because they insist on checking the order (Fortran or C) of numpy arrays. I think these checks should be removed, because it is a waste of computational effort to always ensure arrays are contiguous. If some code needs to interface with external modules code that require continguous arrays, it should use np.ascontiguousarray or np.asfortranarray at the immediate level of the wrapper. --- lib/iris/analysis/maths.py | 6 ++++-- lib/iris/coords.py | 4 +++- lib/iris/cube.py | 18 ++++++------------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/iris/analysis/maths.py b/lib/iris/analysis/maths.py index 324c76fec8..8ddfa5b344 100644 --- a/lib/iris/analysis/maths.py +++ b/lib/iris/analysis/maths.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2013, Met Office +# (C) British Crown Copyright 2010 - 2014, Met Office # # This file is part of Iris. # @@ -536,7 +536,9 @@ def _math_op_common(cube, operation_function, new_unit, in_place=False): new_cube = cube operation_function(new_cube.data, out=new_cube.data) else: - new_cube = cube.copy(data=operation_function(cube.data)) + # use a slice to shallow copy the cube + new_cube = cube[:] + new_cube.data = operation_function(cube.data) iris.analysis.clear_phenomenon_identity(new_cube) new_cube.units = new_unit return new_cube diff --git a/lib/iris/coords.py b/lib/iris/coords.py index e9bcb1d999..66af022af1 100644 --- a/lib/iris/coords.py +++ b/lib/iris/coords.py @@ -431,7 +431,9 @@ def copy(self, points=None, bounds=None): raise ValueError('If bounds are specified, points must also be ' 'specified') - new_coord = copy.deepcopy(self) + new_coord = copy.copy(self) + new_coord.attributes = copy.deepcopy(self.attributes) + new_coord.coord_system = copy.deepcopy(self.coord_system) if points is not None: # Explicitly not using the points property as we don't want the # shape the new points to be constrained by the shape of diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 5c3bd2d293..f540e4afed 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -551,7 +551,7 @@ def __init__(self, data, standard_name=None, long_name=None, if data_manager is not None: self._data = data - self._data_manager = data_manager + self._data_manager = copy.deepcopy(data_manager) else: if isinstance(data, np.ndarray): self._data = data @@ -1760,10 +1760,10 @@ def __getitem__(self, keys): data = self.data[first_slice] else: if use_data_proxy: - data = copy.deepcopy(self._data) + data = self._data data_manager = copy.deepcopy(self._data_manager) else: - data = copy.deepcopy(self.data) + data = self.data for other_slice in slice_gen: if use_data_proxy: @@ -1771,11 +1771,6 @@ def __getitem__(self, keys): else: data = data[other_slice] - # We don't want a view of the numpy array, so take a copy of it if - # it's not our own (this applies to proxy "empty data" arrays too) - if not data.flags['OWNDATA']: - data = data.copy() - # We can turn a masked array into a normal array if it's full. if isinstance(data, ma.core.MaskedArray): if ma.count_masked(data) == 0: @@ -2012,10 +2007,9 @@ def transpose(self, new_order=None): elif len(new_order) != self.data.ndim: raise ValueError('Incorrect number of dimensions.') - # The data needs to be copied, otherwise this view of the transposed - # data will not be contiguous. Ensure not to assign via the cube.data - # setter property since we are reshaping the cube payload in-place. - self._data = np.transpose(self.data, new_order).copy() + # Ensure not to assign via the cube.data setter property since we are + # reshaping the cube payload in-place. + self._data = np.transpose(self.data, new_order) dim_mapping = {src: dest for dest, src in enumerate(new_order)}