From 8e914e7b505970f21fa8b0125fd929487f82ddf7 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 11:51:00 +0200 Subject: [PATCH 01/34] Fix calendar units to 'days since 1850-01-01' on a standard calendar --- esmvalcore/preprocessor/_multimodel.py | 44 ++++++++---------- .../_multimodel/test_multimodel.py | 46 ++++++------------- 2 files changed, 33 insertions(+), 57 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index a7e68143c4..5676a1c688 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -25,14 +25,6 @@ logger = logging.getLogger(__name__) -def _get_time_offset(time_unit): - """Return a datetime object equivalent to tunit.""" - # tunit e.g. 'day since 1950-01-01 00:00:00.0000000 UTC' - cfunit = cf_units.Unit(time_unit, calendar=cf_units.CALENDAR_STANDARD) - time_offset = cfunit.num2date(0) - return time_offset - - def _plev_fix(dataset, pl_idx): """Extract valid plev data. @@ -112,11 +104,10 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): times = template_cube.coord('time') else: unit_name = template_cube.coord('time').units.name - tunits = cf_units.Unit(unit_name, calendar="standard") - times = iris.coords.DimCoord( - t_axis, - standard_name='time', - units=tunits) + tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") + times = iris.coords.DimCoord(t_axis, + standard_name='time', + units=tunits) coord_names = [c.long_name for c in template_cube.coords()] coord_names.extend([c.standard_name for c in template_cube.coords()]) @@ -152,8 +143,9 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): # correct dspec if necessary fixed_dspec = np.ma.fix_invalid(cube_data, copy=False, fill_value=1e+20) # put in cube - stats_cube = iris.cube.Cube( - fixed_dspec, dim_coords_and_dims=cspec, long_name=statistic) + stats_cube = iris.cube.Cube(fixed_dspec, + dim_coords_and_dims=cspec, + long_name=statistic) coord_names = [coord.name() for coord in template_cube.coords()] if 'air_pressure' in coord_names: if len(template_cube.shape) == 3: @@ -181,11 +173,9 @@ def _datetime_to_int_days(cube): real_dates.append(real_date) # get the number of days starting from the reference unit - time_unit = cube.coord('time').units.name - time_offset = _get_time_offset(time_unit) - days = [(date_obj - time_offset).days for date_obj in real_dates] - - return days + reference_date = datetime(1850, 1, 1) + integer_days = [(date - reference_date).days for date in real_dates] + return integer_days def _align_yearly_axes(cube): @@ -275,8 +265,10 @@ def _assemble_overlap_data(cubes, interval, statistic): for cube, indx in zip(cubes, indices) ] stats_dats[i] = _compute_statistic(time_data, statistic) - stats_cube = _put_in_cube( - cubes[0][sl_1:sl_2 + 1], stats_dats, statistic, t_axis=None) + stats_cube = _put_in_cube(cubes[0][sl_1:sl_2 + 1], + stats_dats, + statistic, + t_axis=None) return stats_cube @@ -384,8 +376,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): logger.debug("Using full time spans to compute statistics.") else: raise ValueError( - "Unexpected value for span {}, choose from 'overlap', 'full'" - .format(span)) + "Unexpected value for span {}, choose from 'overlap', 'full'". + format(span)) for statistic in statistics: # Compute statistic @@ -393,8 +385,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): statistic_cube = _assemble_overlap_data(cubes, interval, statistic) elif span == 'full': statistic_cube = _assemble_full_data(cubes, statistic) - statistic_cube.data = np.ma.array( - statistic_cube.data, dtype=np.dtype('float32')) + statistic_cube.data = np.ma.array(statistic_cube.data, + dtype=np.dtype('float32')) if output_products: # Add to output product and log provenance diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 4cb7533d49..999969c848 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -2,27 +2,19 @@ import unittest -import cftime import iris import numpy as np from cf_units import Unit import tests from esmvalcore.preprocessor import multi_model_statistics -from esmvalcore.preprocessor._multimodel import (_assemble_full_data, - _assemble_overlap_data, - _compute_statistic, - _datetime_to_int_days, - _get_overlap, - _get_time_offset, - _plev_fix, - _put_in_cube, - _slice_cube) +from esmvalcore.preprocessor._multimodel import ( + _assemble_full_data, _assemble_overlap_data, _compute_statistic, + _datetime_to_int_days, _get_overlap, _plev_fix, _put_in_cube, _slice_cube) class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" - def setUp(self): """Prepare tests.""" coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) @@ -35,35 +27,33 @@ def setUp(self): time = iris.coords.DimCoord([15, 45], standard_name='time', bounds=[[1., 30.], [30., 60.]], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + units=Unit('days since 1850-01-01', + calendar='gregorian')) time2 = iris.coords.DimCoord([1., 2., 3., 4.], standard_name='time', bounds=[ [0.5, 1.5], [1.5, 2.5], [2.5, 3.5], - [3.5, 4.5], ], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + [3.5, 4.5], + ], + units=Unit('days since 1850-01-01', + calendar='gregorian')) yr_time = iris.coords.DimCoord([15, 410], standard_name='time', bounds=[[1., 30.], [395., 425.]], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + units=Unit('days since 1850-01-01', + calendar='gregorian')) yr_time2 = iris.coords.DimCoord([1., 367., 733., 1099.], standard_name='time', bounds=[ [0.5, 1.5], [366, 368], [732, 734], - [1098, 1100], ], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + [1098, 1100], + ], + units=Unit('days since 1850-01-01', + calendar='gregorian')) zcoord = iris.coords.DimCoord([0.5, 5., 50.], standard_name='air_pressure', long_name='air_pressure', @@ -98,12 +88,6 @@ def setUp(self): self.cube2_yr = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5_yr) - def test_get_time_offset(self): - """Test time unit.""" - result = _get_time_offset("days since 1950-01-01") - expected = cftime.real_datetime(1950, 1, 1, 0, 0) - np.testing.assert_equal(result, expected) - def test_compute_statistic(self): """Test statistic.""" data = [self.cube1.data[0], self.cube2.data[0]] From d6b9947073920a393d14dc3767e7cd48308b5363 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 12:18:37 +0200 Subject: [PATCH 02/34] More thorough check on source time frequency; align behaviour with regrid time; raise for daily data --- esmvalcore/preprocessor/_multimodel.py | 50 +++++++++-------- .../_multimodel/test_multimodel.py | 54 ++++++++++++------- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 5676a1c688..347091080d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -159,32 +159,36 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): def _datetime_to_int_days(cube): - """Return list of int(days) converted from cube datetime cells.""" - cube = _align_yearly_axes(cube) - time_cells = [cell.point for cell in cube.coord('time').cells()] - - # extract date info - real_dates = [] - for date_obj in time_cells: - # real_date resets the actual data point day - # to the 1st of the month so that there are no - # wrong overlap indices - real_date = datetime(date_obj.year, date_obj.month, 1, 0, 0, 0) - real_dates.append(real_date) - - # get the number of days starting from the reference unit - reference_date = datetime(1850, 1, 1) - integer_days = [(date - reference_date).days for date in real_dates] - return integer_days + """Return list of int(days) with respect to a common reference. + Cubes may have different calendars. This function extracts the date + information from the cube and re-constructs a default calendar, + resetting the actual dates to the 15th of the month or 1st of july for + yearly data (consistent with `regrid_time`), so that there are no + mismatches in the time arrays. -def _align_yearly_axes(cube): - """Perform a time-regridding operation to align time axes for yr data.""" + Doesn't work for (sub)daily data, because different calendars may have + different number of days in the year. + """ + # Extract date info from cube years = [cell.point.year for cell in cube.coord('time').cells()] - # be extra sure that the first point is not in the previous year - if 0 not in np.diff(years): - return regrid_time(cube, 'yr') - return cube + months = [cell.point.month for cell in cube.coord('time').cells()] + + # Reconstruct default calendar + if not 0 in np.diff(years): + # yearly data + standard_dates = [datetime(year, 7, 1) for year in years] + elif not 0 in np.diff(months): + # monthly data + standard_dates = [datetime(year, month, 15) + for year, month in zip(years, months)] + else: + # (sub)daily data + raise ValueError("Multimodel only supports yearly or monthly data") + + # Get the number of days starting from the reference + reference_date = datetime(1850, 1, 1) + return [(date - reference_date).days for date in standard_dates] def _get_overlap(cubes): diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 999969c848..23552617f6 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -29,16 +29,22 @@ def setUp(self): bounds=[[1., 30.], [30., 60.]], units=Unit('days since 1850-01-01', calendar='gregorian')) - time2 = iris.coords.DimCoord([1., 2., 3., 4.], + time2 = iris.coords.DimCoord([45, 75, 105, 135], standard_name='time', bounds=[ - [0.5, 1.5], - [1.5, 2.5], - [2.5, 3.5], - [3.5, 4.5], - ], - units=Unit('days since 1850-01-01', - calendar='gregorian')) + [30., 60.], + [60., 90.], + [90., 120.], + [120., 150.]], + units=Unit( + 'days since 1850-01-01', + calendar='gregorian')) + day_time = iris.coords.DimCoord([1., 2.], + standard_name='time', + bounds=[[0.5, 1.5], [1.5, 2.5]], + units=Unit( + 'days since 1850-01-01', + calendar='gregorian')) yr_time = iris.coords.DimCoord([15, 410], standard_name='time', bounds=[[1., 30.], [395., 425.]], @@ -87,6 +93,10 @@ def setUp(self): coords_spec5_yr = [(yr_time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] self.cube2_yr = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5_yr) + coords_spec_day = [(day_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube1_day = iris.cube.Cube(data2, + dim_coords_and_dims=coords_spec_day) + def test_compute_statistic(self): """Test statistic.""" @@ -101,9 +111,9 @@ def test_compute_statistic(self): def test_compute_full_statistic_mon_cube(self): data = [self.cube1, self.cube2] stats = multi_model_statistics(data, 'full', ['mean']) - expected_full_mean = np.ma.ones((2, 3, 2, 2)) - expected_full_mean.mask = np.zeros((2, 3, 2, 2)) - expected_full_mean.mask[1] = True + expected_full_mean = np.ma.ones((5, 3, 2, 2)) + expected_full_mean.mask = np.ones((5, 3, 2, 2)) + expected_full_mean.mask[1] = False self.assert_array_equal(stats['mean'].data, expected_full_mean) def test_compute_full_statistic_yr_cube(self): @@ -155,36 +165,36 @@ def test_put_in_cube(self): stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) self.assert_array_equal(stat_cube.data, self.cube1.data) - def test_datetime_to_int_days_no_overlap(self): + def test_datetime_to_int_days(self): """Test _datetime_to_int_days.""" computed_dats = _datetime_to_int_days(self.cube1) - expected_dats = [0, 31] + expected_dats = [14, 45] self.assert_array_equal(computed_dats, expected_dats) def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], - [0, 31], "mean") + [14, 45], "mean") expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) def test_assemble_full_data(self): """Test full data.""" comp_full_mean = _assemble_full_data([self.cube1, self.cube2], "mean") - expected_full_mean = np.ma.ones((2, 3, 2, 2)) - expected_full_mean.mask = np.zeros((2, 3, 2, 2)) - expected_full_mean.mask[1] = True + expected_full_mean = np.ma.ones((5, 3, 2, 2)) + expected_full_mean.mask = np.ones((5, 3, 2, 2)) + expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) def test_slice_cube(self): """Test slice cube.""" - comp_slice = _slice_cube(self.cube1, 0, 31) + comp_slice = _slice_cube(self.cube1, 14, 45) self.assert_array_equal([0, 1], comp_slice) def test_get_overlap(self): """Test get overlap.""" full_ovlp = _get_overlap([self.cube1, self.cube1]) - self.assert_array_equal([0, 31], full_ovlp) + self.assert_array_equal([14, 45], full_ovlp) no_ovlp = _get_overlap([self.cube1, self.cube2]) np.testing.assert_equal(None, no_ovlp) @@ -194,6 +204,12 @@ def test_plev_fix(self): expected_data = np.ma.ones((3, 2, 2)) self.assert_array_equal(expected_data, fixed_data) + def test_raise_daily(self): + """Test raise for daily input data.""" + with self.assertRaises(ValueError): + _datetime_to_int_days(self.cube1_day) + + if __name__ == '__main__': unittest.main() From 12f9aa4c6b5265396c27f109db8622d151f23592 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 12:57:17 +0200 Subject: [PATCH 03/34] Simplify _get_overlap --- esmvalcore/preprocessor/_multimodel.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 347091080d..5caa858c18 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -192,26 +192,12 @@ def _datetime_to_int_days(cube): def _get_overlap(cubes): - """ - Get discrete time overlaps. - - This method gets the bounds of coord time - from the cube and assembles a continuous time - axis with smallest unit 1; then it finds the - overlaps by doing a 1-dim intersect; - takes the floor of first date and - ceil of last date. - """ - all_times = [] - for cube in cubes: - span = _datetime_to_int_days(cube) - start, stop = span[0], span[-1] - all_times.append([start, stop]) - bounds = [range(b[0], b[-1] + 1) for b in all_times] - time_pts = reduce(np.intersect1d, bounds) - if len(time_pts) > 1: - time_bounds_list = [time_pts[0], time_pts[-1]] - return time_bounds_list + """Return the intersection of all cubes' time arrays.""" + time_spans = [_datetime_to_int_days(cube) for cube in cubes] + overlap = reduce(np.intersect1d, time_spans) + if len(overlap) > 1: + return [overlap[0], overlap[-1]] + return def _slice_cube(cube, t_1, t_2): From bf82085f24030545b1c5770def1d8396c6d567fb Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 13:27:57 +0200 Subject: [PATCH 04/34] Align behaviour for union (full) and intersection (overlap) of time arrays --- esmvalcore/preprocessor/_multimodel.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 5caa858c18..2b5e73901a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -192,7 +192,7 @@ def _datetime_to_int_days(cube): def _get_overlap(cubes): - """Return the intersection of all cubes' time arrays.""" + """Return bounds of the intersection of all cubes' time arrays.""" time_spans = [_datetime_to_int_days(cube) for cube in cubes] overlap = reduce(np.intersect1d, time_spans) if len(overlap) > 1: @@ -200,6 +200,12 @@ def _get_overlap(cubes): return +def _get_union(cubes): + """Return the union of all cubes' time arrays.""" + time_spans = [_datetime_to_int_days(cube) for cube in cubes] + return reduce(np.union1d, time_spans) + + def _slice_cube(cube, t_1, t_2): """ Efficient slicer. @@ -216,13 +222,6 @@ def _slice_cube(cube, t_1, t_2): return [idxs[0], idxs[-1]] -def _monthly_t(cubes): - """Rearrange time points for monthly data.""" - # get original cubes tpoints - days = {day for cube in cubes for day in _datetime_to_int_days(cube)} - return sorted(days) - - def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): """Construct a contiguous collection over time.""" for idx_cube, cube in enumerate(cubes): @@ -264,8 +263,9 @@ def _assemble_overlap_data(cubes, interval, statistic): def _assemble_full_data(cubes, statistic): """Get statistical data in iris cubes for FULL.""" - # all times, new MONTHLY data time axis - time_axis = [float(fl) for fl in _monthly_t(cubes)] + # Gather the unique time points in the union of all cubes + time_points = _get_union(cubes) + time_axis = [float(fl) for fl in time_points] # new big time-slice array shape new_shape = [len(time_axis)] + list(cubes[0].shape[1:]) From 3bd1e82b02603852dd3dd3fa74519753eb5279be Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 14:43:59 +0200 Subject: [PATCH 05/34] Add function to make all cubes use the same calendar. This function also checks the frequency of the cubes (previously in _datetime_to_int_days), this seems to be a much more logical place. _datetime_to_int_days now simplifies to a one-liner, and can probably be eliminated completely. --- esmvalcore/preprocessor/_multimodel.py | 61 +++++++++++++------ .../_multimodel/test_multimodel.py | 15 +++-- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 2b5e73901a..46c374ae63 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -158,8 +158,9 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube -def _datetime_to_int_days(cube): - """Return list of int(days) with respect to a common reference. +def _set_common_calendar(cubes): + """ + Make sure all cubes' use the same standard calendar. Cubes may have different calendars. This function extracts the date information from the cube and re-constructs a default calendar, @@ -170,25 +171,42 @@ def _datetime_to_int_days(cube): Doesn't work for (sub)daily data, because different calendars may have different number of days in the year. """ - # Extract date info from cube - years = [cell.point.year for cell in cube.coord('time').cells()] - months = [cell.point.month for cell in cube.coord('time').cells()] - - # Reconstruct default calendar - if not 0 in np.diff(years): - # yearly data - standard_dates = [datetime(year, 7, 1) for year in years] - elif not 0 in np.diff(months): - # monthly data - standard_dates = [datetime(year, month, 15) - for year, month in zip(years, months)] - else: - # (sub)daily data - raise ValueError("Multimodel only supports yearly or monthly data") + # The default time unit + t_unit = cf_units.Unit("days since 1850-01-01", calendar="standard") + + for cube in cubes: + # Extract date info from cube + years = [cell.point.year for cell in cube.coord('time').cells()] + months = [cell.point.month for cell in cube.coord('time').cells()] + + # Reconstruct default calendar + if not 0 in np.diff(years): + # yearly data + dates = [datetime(year, 7, 1) for year in years] + + elif not 0 in np.diff(months): + # monthly data + dates = [datetime(year, month, 15) + for year, month in zip(years, months)] + else: + # (sub)daily data + raise ValueError("Multimodel only supports yearly or monthly data") + + # Update the cubes' time coordinate (both point values and the units!) + cube.coord('time').points = [t_unit.date2num(date) for date in dates] + cube.coord('time').units = t_unit + # Reset bounds + cube.coord('time').bounds = None + cube.coord('time').guess_bounds() + # Remove aux coords that may differ + for auxcoord in cube.aux_coords: + if auxcoord.long_name in ['day_of_month', 'day_of_year']: + cube.remove_coord(auxcoord) - # Get the number of days starting from the reference - reference_date = datetime(1850, 1, 1) - return [(date - reference_date).days for date in standard_dates] + +def _datetime_to_int_days(cube): + """Return the cube's time point values as list of integers.""" + return cube.coord('time').points.astype(int).tolist() def _get_overlap(cubes): @@ -353,6 +371,9 @@ def multi_model_statistics(products, span, statistics, output_products=None): cubes = products statistic_products = {} + # Make cubes share the same calendar, so time points are comparable + _set_common_calendar(cubes) + if span == 'overlap': # check if we have any time overlap interval = _get_overlap(cubes) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 23552617f6..2f59b10278 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -9,7 +9,7 @@ import tests from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( - _assemble_full_data, _assemble_overlap_data, _compute_statistic, + _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, _datetime_to_int_days, _get_overlap, _plev_fix, _put_in_cube, _slice_cube) @@ -24,12 +24,12 @@ def setUp(self): mask3[0, 0, 0, 0] = True data3 = np.ma.array(data3, mask=mask3) - time = iris.coords.DimCoord([15, 45], + time = iris.coords.DimCoord([14, 45], standard_name='time', bounds=[[1., 30.], [30., 60.]], units=Unit('days since 1850-01-01', calendar='gregorian')) - time2 = iris.coords.DimCoord([45, 75, 105, 135], + time2 = iris.coords.DimCoord([45, 73, 104, 134], standard_name='time', bounds=[ [30., 60.], @@ -45,7 +45,7 @@ def setUp(self): units=Unit( 'days since 1850-01-01', calendar='gregorian')) - yr_time = iris.coords.DimCoord([15, 410], + yr_time = iris.coords.DimCoord([14., 410.], standard_name='time', bounds=[[1., 30.], [395., 425.]], units=Unit('days since 1850-01-01', @@ -204,10 +204,15 @@ def test_plev_fix(self): expected_data = np.ma.ones((3, 2, 2)) self.assert_array_equal(expected_data, fixed_data) + def test_set_common_calendar(self): + """Test set common calenar.""" + cubes = [self.cube1, self.cube2] + # TODO: complete this test + def test_raise_daily(self): """Test raise for daily input data.""" with self.assertRaises(ValueError): - _datetime_to_int_days(self.cube1_day) + _set_common_calendar([self.cube1_day]) From 7841a5f39356429d1b7b2af7900a8dda048378fd Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 16:04:56 +0200 Subject: [PATCH 06/34] Remove _datetime_to_int_days, as we can just use the time points --- esmvalcore/preprocessor/_multimodel.py | 19 +++++++------------ .../_multimodel/test_multimodel.py | 8 +------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 46c374ae63..969a206208 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -204,24 +204,19 @@ def _set_common_calendar(cubes): cube.remove_coord(auxcoord) -def _datetime_to_int_days(cube): - """Return the cube's time point values as list of integers.""" - return cube.coord('time').points.astype(int).tolist() - - def _get_overlap(cubes): """Return bounds of the intersection of all cubes' time arrays.""" - time_spans = [_datetime_to_int_days(cube) for cube in cubes] - overlap = reduce(np.intersect1d, time_spans) + time_spans = [cube.coord('time').points for cube in cubes] + overlap = reduce(np.intersect1d, time_spans).astype(int) if len(overlap) > 1: return [overlap[0], overlap[-1]] - return + return None def _get_union(cubes): """Return the union of all cubes' time arrays.""" - time_spans = [_datetime_to_int_days(cube) for cube in cubes] - return reduce(np.union1d, time_spans) + time_spans = [cube.coord('time').points for cube in cubes] + return reduce(np.union1d, time_spans).astype(int) def _slice_cube(cube, t_1, t_2): @@ -232,7 +227,7 @@ def _slice_cube(cube, t_1, t_2): of common time-data elements. """ time_pts = [t for t in cube.coord('time').points] - converted_t = _datetime_to_int_days(cube) + converted_t = cube.coord('time').points.astype(int).tolist() idxs = sorted([ time_pts.index(ii) for ii, jj in zip(time_pts, converted_t) if t_1 <= jj <= t_2 @@ -303,7 +298,7 @@ def _assemble_full_data(cubes, statistic): # loop through cubes and populate empty_arr with points for cube in cubes: - time_redone = _datetime_to_int_days(cube) + time_redone = cube.coord('time').points.astype(int).tolist() oidx = [time_axis.index(s) for s in time_redone] indices_list.append(oidx) for i in range(new_shape[0]): diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 2f59b10278..48df977716 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -10,7 +10,7 @@ from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, - _datetime_to_int_days, _get_overlap, _plev_fix, _put_in_cube, _slice_cube) + _get_overlap, _plev_fix, _put_in_cube, _slice_cube) class Test(tests.Test): @@ -165,12 +165,6 @@ def test_put_in_cube(self): stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) self.assert_array_equal(stat_cube.data, self.cube1.data) - def test_datetime_to_int_days(self): - """Test _datetime_to_int_days.""" - computed_dats = _datetime_to_int_days(self.cube1) - expected_dats = [14, 45] - self.assert_array_equal(computed_dats, expected_dats) - def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], From f95aa7c79adc06b18639f524aa4f7a384f2d1271 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 16:42:43 +0200 Subject: [PATCH 07/34] Simplify and rename _slice_cube --- esmvalcore/preprocessor/_multimodel.py | 25 ++++++------------- .../_multimodel/test_multimodel.py | 6 ++--- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 969a206208..f7bb9a8ee2 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -20,8 +20,6 @@ import iris import numpy as np -from ._time import regrid_time - logger = logging.getLogger(__name__) @@ -219,20 +217,11 @@ def _get_union(cubes): return reduce(np.union1d, time_spans).astype(int) -def _slice_cube(cube, t_1, t_2): - """ - Efficient slicer. - - Simple cube data slicer on indices - of common time-data elements. - """ - time_pts = [t for t in cube.coord('time').points] - converted_t = cube.coord('time').points.astype(int).tolist() - idxs = sorted([ - time_pts.index(ii) for ii, jj in zip(time_pts, converted_t) - if t_1 <= jj <= t_2 - ]) - return [idxs[0], idxs[-1]] +def _get_slice_parameters(cube, tmin, tmax): + """Get the lower and upper array indices for a given time interval.""" + time = cube.coord('time').points + idxs = np.argwhere((time >= tmin) & (time <= tmax)) + return [idxs.min(), idxs.max()] def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): @@ -254,12 +243,12 @@ def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): def _assemble_overlap_data(cubes, interval, statistic): """Get statistical data in iris cubes for OVERLAP.""" start, stop = interval - sl_1, sl_2 = _slice_cube(cubes[0], start, stop) + sl_1, sl_2 = _get_slice_parameters(cubes[0], start, stop) stats_dats = np.ma.zeros(cubes[0].data[sl_1:sl_2 + 1].shape) # keep this outside the following loop # this speeds up the code by a factor of 15 - indices = [_slice_cube(cube, start, stop) for cube in cubes] + indices = [_get_slice_parameters(cube, start, stop) for cube in cubes] for i in range(stats_dats.shape[0]): time_data = [ diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 48df977716..d55d6148c0 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -10,7 +10,7 @@ from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, - _get_overlap, _plev_fix, _put_in_cube, _slice_cube) + _get_overlap, _plev_fix, _put_in_cube, _get_slice_parameters) class Test(tests.Test): @@ -180,9 +180,9 @@ def test_assemble_full_data(self): expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) - def test_slice_cube(self): + def test_get_slice_parameters(self): """Test slice cube.""" - comp_slice = _slice_cube(self.cube1, 14, 45) + comp_slice = _get_slice_parameters(self.cube1, 14, 45) self.assert_array_equal([0, 1], comp_slice) def test_get_overlap(self): From db47522282693f66753cd9905e5c7e8bd369bf2d Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 22:33:16 +0200 Subject: [PATCH 08/34] Align _assemble_overlap_data more with _assemble_full_data --- esmvalcore/preprocessor/_multimodel.py | 63 +++++++++---------- .../_multimodel/test_multimodel.py | 26 +++++--- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index f7bb9a8ee2..ba9561a1f1 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -101,7 +101,6 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): if t_axis is None: times = template_cube.coord('time') else: - unit_name = template_cube.coord('time').units.name tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") times = iris.coords.DimCoord(t_axis, standard_name='time', @@ -202,26 +201,22 @@ def _set_common_calendar(cubes): cube.remove_coord(auxcoord) -def _get_overlap(cubes): +def _get_time_intersection(cubes): """Return bounds of the intersection of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] - overlap = reduce(np.intersect1d, time_spans).astype(int) - if len(overlap) > 1: - return [overlap[0], overlap[-1]] - return None + return reduce(np.intersect1d, time_spans).astype(int) -def _get_union(cubes): +def _get_time_union(cubes): """Return the union of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] return reduce(np.union1d, time_spans).astype(int) -def _get_slice_parameters(cube, tmin, tmax): - """Get the lower and upper array indices for a given time interval.""" +def _get_subset(cube, tmin, tmax): time = cube.coord('time').points idxs = np.argwhere((time >= tmin) & (time <= tmax)) - return [idxs.min(), idxs.max()] + return cube[idxs.min():idxs.max()+1] def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): @@ -240,33 +235,35 @@ def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): return ndatarr -def _assemble_overlap_data(cubes, interval, statistic): +def _assemble_overlap_data(cubes, statistic): """Get statistical data in iris cubes for OVERLAP.""" - start, stop = interval - sl_1, sl_2 = _get_slice_parameters(cubes[0], start, stop) - stats_dats = np.ma.zeros(cubes[0].data[sl_1:sl_2 + 1].shape) - - # keep this outside the following loop - # this speeds up the code by a factor of 15 - indices = [_get_slice_parameters(cube, start, stop) for cube in cubes] - - for i in range(stats_dats.shape[0]): - time_data = [ - cube.data[indx[0]:indx[1] + 1][i] - for cube, indx in zip(cubes, indices) - ] - stats_dats[i] = _compute_statistic(time_data, statistic) - stats_cube = _put_in_cube(cubes[0][sl_1:sl_2 + 1], - stats_dats, - statistic, - t_axis=None) + # Gather overlapping time points + new_times = _get_time_intersection(cubes).tolist() + tmin = min(new_times) + tmax = max(new_times) + n_times = len(new_times) + + # Target array to populate with computed statistics + new_shape = [n_times] + list(cubes[0].shape[1:]) + stats_data = np.ma.zeros(new_shape) + + # Prepare a list of cubes with matching times (so far just pointers) + # Keep this outside the following loop; 15x speedup (still true?) + cubelist = [_get_subset(cube, tmin, tmax) for cube in cubes] + + for i in range(n_times): + time_data = [cube.data[i] for cube in cubelist] + stats_data[i] = _compute_statistic(time_data, statistic) + + template_cube = cubelist[0] + stats_cube = _put_in_cube(template_cube, stats_data, statistic, new_times) return stats_cube def _assemble_full_data(cubes, statistic): """Get statistical data in iris cubes for FULL.""" # Gather the unique time points in the union of all cubes - time_points = _get_union(cubes) + time_points = _get_time_union(cubes) time_axis = [float(fl) for fl in time_points] # new big time-slice array shape @@ -360,8 +357,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): if span == 'overlap': # check if we have any time overlap - interval = _get_overlap(cubes) - if interval is None: + overlap = _get_time_intersection(cubes) + if len(overlap) <= 1: logger.info("Time overlap between cubes is none or a single point." "check datasets: will not compute statistics.") return products @@ -377,7 +374,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic if span == 'overlap': - statistic_cube = _assemble_overlap_data(cubes, interval, statistic) + statistic_cube = _assemble_overlap_data(cubes, statistic) elif span == 'full': statistic_cube = _assemble_full_data(cubes, statistic) statistic_cube.data = np.ma.array(statistic_cube.data, diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index d55d6148c0..e9124464bb 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -10,7 +10,7 @@ from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, - _get_overlap, _plev_fix, _put_in_cube, _get_slice_parameters) + _get_time_intersection, _plev_fix, _put_in_cube) class Test(tests.Test): @@ -39,6 +39,14 @@ def setUp(self): units=Unit( 'days since 1850-01-01', calendar='gregorian')) + time3 = iris.coords.DimCoord([104, 134], + standard_name='time', + bounds=[ + [90., 120.], + [120., 150.]], + units=Unit( + 'days since 1850-01-01', + calendar='gregorian')) day_time = iris.coords.DimCoord([1., 2.], standard_name='time', bounds=[[0.5, 1.5], [1.5, 2.5]], @@ -86,6 +94,9 @@ def setUp(self): coords_spec5 = [(time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] self.cube2 = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5) + coords_spec6 = [(time3, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube3 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec6) + coords_spec4_yr = [(yr_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] self.cube1_yr = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec4_yr) @@ -168,7 +179,7 @@ def test_put_in_cube(self): def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], - [14, 45], "mean") + "mean") expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) @@ -180,16 +191,11 @@ def test_assemble_full_data(self): expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) - def test_get_slice_parameters(self): - """Test slice cube.""" - comp_slice = _get_slice_parameters(self.cube1, 14, 45) - self.assert_array_equal([0, 1], comp_slice) - - def test_get_overlap(self): + def test_get_time_intersection(self): """Test get overlap.""" - full_ovlp = _get_overlap([self.cube1, self.cube1]) + full_ovlp = _get_time_intersection([self.cube1, self.cube1]) self.assert_array_equal([14, 45], full_ovlp) - no_ovlp = _get_overlap([self.cube1, self.cube2]) + no_ovlp = _get_time_intersection([self.cube1, self.cube3]) np.testing.assert_equal(None, no_ovlp) def test_plev_fix(self): From 127c69b231cb2da88374a3ee69650b77a646b889 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 22:49:34 +0200 Subject: [PATCH 09/34] Align _assemble_full_data more with _assemble_overlap_data --- esmvalcore/preprocessor/_multimodel.py | 46 +++++++++++--------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ba9561a1f1..fd0d5d479d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -204,13 +204,13 @@ def _set_common_calendar(cubes): def _get_time_intersection(cubes): """Return bounds of the intersection of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.intersect1d, time_spans).astype(int) + return reduce(np.intersect1d, time_spans) def _get_time_union(cubes): """Return the union of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.union1d, time_spans).astype(int) + return reduce(np.union1d, time_spans) def _get_subset(cube, tmin, tmax): @@ -255,48 +255,42 @@ def _assemble_overlap_data(cubes, statistic): time_data = [cube.data[i] for cube in cubelist] stats_data[i] = _compute_statistic(time_data, statistic) - template_cube = cubelist[0] - stats_cube = _put_in_cube(template_cube, stats_data, statistic, new_times) + template = cubelist[0] + stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube def _assemble_full_data(cubes, statistic): """Get statistical data in iris cubes for FULL.""" - # Gather the unique time points in the union of all cubes - time_points = _get_time_union(cubes) - time_axis = [float(fl) for fl in time_points] + # Gather time points in the union of all cubes + new_times = _get_time_union(cubes).tolist() + n_times = len(new_times) - # new big time-slice array shape - new_shape = [len(time_axis)] + list(cubes[0].shape[1:]) + # Target array to populate with computed statistics + new_shape = [n_times] + list(cubes[0].shape[1:]) + stats_data = np.ma.zeros(new_shape) # assemble an array to hold all time data # for all cubes; shape is (ncubes,(plev), lat, lon) new_arr = np.ma.empty([len(cubes)] + list(new_shape[1:])) - # data array for stats computation - stats_dats = np.ma.zeros(new_shape) - - # assemble indices list to chop new_arr on - indices_list = [] - # empty data array to hold time slices empty_arr = np.ma.empty(new_shape) - # loop through cubes and populate empty_arr with points + # Prepare a list mapping the cubes' times to the indices of new_times + indices_list = [] for cube in cubes: - time_redone = cube.coord('time').points.astype(int).tolist() - oidx = [time_axis.index(s) for s in time_redone] + oidx = [new_times.index(t) for t in cube.coord('time').points] indices_list.append(oidx) - for i in range(new_shape[0]): + + for i in range(n_times): # hold time slices only - new_datas_array = _full_time_slice(cubes, empty_arr, indices_list, + time_data = _full_time_slice(cubes, empty_arr, indices_list, new_arr, i) - # list to hold time slices - time_data = [] - for j in range(len(cubes)): - time_data.append(new_datas_array[j]) - stats_dats[i] = _compute_statistic(time_data, statistic) - stats_cube = _put_in_cube(cubes[0], stats_dats, statistic, time_axis) + stats_data[i] = _compute_statistic(time_data, statistic) + + template = cubes[0] + stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube From f873ea523dfae49bf4d026e9558da1f394b54699 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 13:52:42 +0200 Subject: [PATCH 10/34] Futher align assemble full and overlap data --- esmvalcore/preprocessor/_multimodel.py | 62 ++++++++++---------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fd0d5d479d..9522069821 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -219,43 +219,42 @@ def _get_subset(cube, tmin, tmax): return cube[idxs.min():idxs.max()+1] -def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): - """Construct a contiguous collection over time.""" - for idx_cube, cube in enumerate(cubes): - # reset mask - ndat.mask = True - ndat[indices[idx_cube]] = cube.data - if np.ma.is_masked(cube.data): - ndat.mask[indices[idx_cube]] = cube.data.mask +def _get_index(cube, time): + # return cube.coord('time').points.tolist().index(time) + cubetime = cube.coord('time').points + return int(np.argwhere(time == cubetime)) + + +def _get_time_slice(cubes, time): + """Fill time slice array with cubes' data if time in cube, else mask.""" + time_slice = [] + for j, cube in enumerate(cubes): + cube_time = cube.coord('time').points + if time in cube_time: + idx = int(np.argwhere(cube_time == time)) + subset = cube[idx].data else: - ndat.mask[indices[idx_cube]] = False - ndatarr[idx_cube] = ndat[t_idx] - - # return time slice - return ndatarr + subset = np.ma.empty(list(cubes[0].shape[1:])) + subset.mask = True + time_slice.append(subset) + return time_slice def _assemble_overlap_data(cubes, statistic): """Get statistical data in iris cubes for OVERLAP.""" # Gather overlapping time points new_times = _get_time_intersection(cubes).tolist() - tmin = min(new_times) - tmax = max(new_times) n_times = len(new_times) # Target array to populate with computed statistics new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) - # Prepare a list of cubes with matching times (so far just pointers) - # Keep this outside the following loop; 15x speedup (still true?) - cubelist = [_get_subset(cube, tmin, tmax) for cube in cubes] - - for i in range(n_times): - time_data = [cube.data[i] for cube in cubelist] + for i, time in enumerate(new_times): + time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) - template = cubelist[0] + template = cubes[0][:n_times] stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube @@ -270,23 +269,8 @@ def _assemble_full_data(cubes, statistic): new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) - # assemble an array to hold all time data - # for all cubes; shape is (ncubes,(plev), lat, lon) - new_arr = np.ma.empty([len(cubes)] + list(new_shape[1:])) - - # empty data array to hold time slices - empty_arr = np.ma.empty(new_shape) - - # Prepare a list mapping the cubes' times to the indices of new_times - indices_list = [] - for cube in cubes: - oidx = [new_times.index(t) for t in cube.coord('time').points] - indices_list.append(oidx) - - for i in range(n_times): - # hold time slices only - time_data = _full_time_slice(cubes, empty_arr, indices_list, - new_arr, i) + for i, time in enumerate(new_times): + time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) template = cubes[0] From 3b7b55bfd002afca2f32bdbf6981193948d7d795 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 14:05:51 +0200 Subject: [PATCH 11/34] Merge assemble full and overlap data. --- esmvalcore/preprocessor/_multimodel.py | 47 ++++--------------- .../_multimodel/test_multimodel.py | 8 ++-- 2 files changed, 12 insertions(+), 43 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9522069821..f1047a1581 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -213,18 +213,6 @@ def _get_time_union(cubes): return reduce(np.union1d, time_spans) -def _get_subset(cube, tmin, tmax): - time = cube.coord('time').points - idxs = np.argwhere((time >= tmin) & (time <= tmax)) - return cube[idxs.min():idxs.max()+1] - - -def _get_index(cube, time): - # return cube.coord('time').points.tolist().index(time) - cubetime = cube.coord('time').points - return int(np.argwhere(time == cubetime)) - - def _get_time_slice(cubes, time): """Fill time slice array with cubes' data if time in cube, else mask.""" time_slice = [] @@ -240,29 +228,13 @@ def _get_time_slice(cubes, time): return time_slice -def _assemble_overlap_data(cubes, statistic): - """Get statistical data in iris cubes for OVERLAP.""" - # Gather overlapping time points - new_times = _get_time_intersection(cubes).tolist() - n_times = len(new_times) - - # Target array to populate with computed statistics - new_shape = [n_times] + list(cubes[0].shape[1:]) - stats_data = np.ma.zeros(new_shape) - - for i, time in enumerate(new_times): - time_data = _get_time_slice(cubes, time) - stats_data[i] = _compute_statistic(time_data, statistic) - - template = cubes[0][:n_times] - stats_cube = _put_in_cube(template, stats_data, statistic, new_times) - return stats_cube - +def _assemble_data(cubes, statistic, span='overlap'): + """Get statistical data in iris cubes.""" + if span == 'overlap': + new_times = _get_time_intersection(cubes).tolist() + elif span == 'full': + new_times = _get_time_union(cubes).tolist() -def _assemble_full_data(cubes, statistic): - """Get statistical data in iris cubes for FULL.""" - # Gather time points in the union of all cubes - new_times = _get_time_union(cubes).tolist() n_times = len(new_times) # Target array to populate with computed statistics @@ -273,7 +245,7 @@ def _assemble_full_data(cubes, statistic): time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) - template = cubes[0] + template = cubes[0][:n_times] stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube @@ -351,10 +323,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic - if span == 'overlap': - statistic_cube = _assemble_overlap_data(cubes, statistic) - elif span == 'full': - statistic_cube = _assemble_full_data(cubes, statistic) + statistic_cube = _assemble_data(cubes, statistic, span) statistic_cube.data = np.ma.array(statistic_cube.data, dtype=np.dtype('float32')) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index e9124464bb..80eb36593e 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -9,7 +9,7 @@ import tests from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( - _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, + _assemble_data, _compute_statistic, _set_common_calendar, _get_time_intersection, _plev_fix, _put_in_cube) @@ -178,14 +178,14 @@ def test_put_in_cube(self): def test_assemble_overlap_data(self): """Test overlap data.""" - comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], - "mean") + comp_ovlap_mean = _assemble_data([self.cube1, self.cube1], + "mean", span='overlap') expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) def test_assemble_full_data(self): """Test full data.""" - comp_full_mean = _assemble_full_data([self.cube1, self.cube2], "mean") + comp_full_mean = _assemble_data([self.cube1, self.cube2], "mean", span='full') expected_full_mean = np.ma.ones((5, 3, 2, 2)) expected_full_mean.mask = np.ones((5, 3, 2, 2)) expected_full_mean.mask[1] = False From 168a701acea37ffd52222b55525e93d0c671ec26 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 15:49:06 +0200 Subject: [PATCH 12/34] Further simplify --- esmvalcore/preprocessor/_multimodel.py | 34 ++++++------------- .../_multimodel/test_multimodel.py | 12 ++----- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index f1047a1581..061f68e665 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -98,13 +98,8 @@ def _compute_statistic(data, statistic_name): def _put_in_cube(template_cube, cube_data, statistic, t_axis): """Quick cube building and saving.""" - if t_axis is None: - times = template_cube.coord('time') - else: - tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") - times = iris.coords.DimCoord(t_axis, - standard_name='time', - units=tunits) + tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") + times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits) coord_names = [c.long_name for c in template_cube.coords()] coord_names.extend([c.standard_name for c in template_cube.coords()]) @@ -201,22 +196,10 @@ def _set_common_calendar(cubes): cube.remove_coord(auxcoord) -def _get_time_intersection(cubes): - """Return bounds of the intersection of all cubes' time arrays.""" - time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.intersect1d, time_spans) - - -def _get_time_union(cubes): - """Return the union of all cubes' time arrays.""" - time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.union1d, time_spans) - - def _get_time_slice(cubes, time): """Fill time slice array with cubes' data if time in cube, else mask.""" time_slice = [] - for j, cube in enumerate(cubes): + for cube in cubes: cube_time = cube.coord('time').points if time in cube_time: idx = int(np.argwhere(cube_time == time)) @@ -230,17 +213,19 @@ def _get_time_slice(cubes, time): def _assemble_data(cubes, statistic, span='overlap'): """Get statistical data in iris cubes.""" + # New time array representing the union or intersection of all cubes + time_spans = [cube.coord('time').points for cube in cubes] if span == 'overlap': - new_times = _get_time_intersection(cubes).tolist() + new_times = reduce(np.intersect1d, time_spans) elif span == 'full': - new_times = _get_time_union(cubes).tolist() - + new_times = reduce(np.union1d, time_spans) n_times = len(new_times) # Target array to populate with computed statistics new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) + # Make time slices and compute stats for i, time in enumerate(new_times): time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) @@ -307,7 +292,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): if span == 'overlap': # check if we have any time overlap - overlap = _get_time_intersection(cubes) + times = [cube.coord('time').points for cube in cubes] + overlap = reduce(np.intersect1d, times) if len(overlap) <= 1: logger.info("Time overlap between cubes is none or a single point." "check datasets: will not compute statistics.") diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 80eb36593e..78979b13b1 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -9,8 +9,7 @@ import tests from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( - _assemble_data, _compute_statistic, _set_common_calendar, - _get_time_intersection, _plev_fix, _put_in_cube) + _assemble_data, _compute_statistic, _set_common_calendar, _plev_fix, _put_in_cube) class Test(tests.Test): @@ -173,7 +172,7 @@ def test_compute_min(self): def test_put_in_cube(self): """Test put in cube.""" cube_data = np.ma.ones((2, 3, 2, 2)) - stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) + stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=[1,2]) self.assert_array_equal(stat_cube.data, self.cube1.data) def test_assemble_overlap_data(self): @@ -191,13 +190,6 @@ def test_assemble_full_data(self): expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) - def test_get_time_intersection(self): - """Test get overlap.""" - full_ovlp = _get_time_intersection([self.cube1, self.cube1]) - self.assert_array_equal([14, 45], full_ovlp) - no_ovlp = _get_time_intersection([self.cube1, self.cube3]) - np.testing.assert_equal(None, no_ovlp) - def test_plev_fix(self): """Test plev fix.""" fixed_data = _plev_fix(self.cube2.data, 1) From 629a9d6f4f6f776bf8204d77e5d9998e985c00cb Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 16:17:06 +0200 Subject: [PATCH 13/34] Remove stuff about bounds and aux coords as it is not used anyway --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 061f68e665..3cc005109e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -172,11 +172,11 @@ def _set_common_calendar(cubes): months = [cell.point.month for cell in cube.coord('time').cells()] # Reconstruct default calendar - if not 0 in np.diff(years): + if 0 not in np.diff(years): # yearly data dates = [datetime(year, 7, 1) for year in years] - elif not 0 in np.diff(months): + elif 0 not in np.diff(months): # monthly data dates = [datetime(year, month, 15) for year, month in zip(years, months)] From f538fa85cc257aa8772d02446c3f8740b1e1f776 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 16:18:04 +0200 Subject: [PATCH 14/34] Remove stuff about bounds and aux coords as it is not used anyway --- esmvalcore/preprocessor/_multimodel.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 3cc005109e..dbb9729c1c 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -187,13 +187,6 @@ def _set_common_calendar(cubes): # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] cube.coord('time').units = t_unit - # Reset bounds - cube.coord('time').bounds = None - cube.coord('time').guess_bounds() - # Remove aux coords that may differ - for auxcoord in cube.aux_coords: - if auxcoord.long_name in ['day_of_month', 'day_of_year']: - cube.remove_coord(auxcoord) def _get_time_slice(cubes, time): From b0cc1ae4fb439edcaad68b7d83f104017cde650b Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 17:22:48 +0200 Subject: [PATCH 15/34] Clean up tests and add tests for new functions --- esmvalcore/preprocessor/_multimodel.py | 10 +- .../_multimodel/test_multimodel.py | 152 +++++++++--------- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index dbb9729c1c..3ace7e860e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -69,7 +69,7 @@ def _compute_statistic(data, statistic_name): # data is per time point # so we can safely NOT compute stats for single points if data.ndim == 1: - u_datas = [d for d in data] + u_datas = data else: u_datas = [d for d in data if not np.all(d.mask)] if len(u_datas) > 1: @@ -178,8 +178,10 @@ def _set_common_calendar(cubes): elif 0 not in np.diff(months): # monthly data - dates = [datetime(year, month, 15) - for year, month in zip(years, months)] + dates = [ + datetime(year, month, 15) + for year, month in zip(years, months) + ] else: # (sub)daily data raise ValueError("Multimodel only supports yearly or monthly data") @@ -258,11 +260,13 @@ def multi_model_statistics(products, span, statistics, output_products=None): statistics: str statistical measure to be computed. Available options: mean, median, max, min, std + Returns ------- list list of data products or cubes containing the multimodel stats computed. + Raises ------ ValueError diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 78979b13b1..7e98642cdb 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -8,65 +8,33 @@ import tests from esmvalcore.preprocessor import multi_model_statistics -from esmvalcore.preprocessor._multimodel import ( - _assemble_data, _compute_statistic, _set_common_calendar, _plev_fix, _put_in_cube) +from esmvalcore.preprocessor._multimodel import (_assemble_data, + _compute_statistic, + _get_time_slice, _plev_fix, + _put_in_cube, + _set_common_calendar) class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" + def setUp(self): """Prepare tests.""" - coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) - data2 = np.ma.ones((2, 3, 2, 2)) - data3 = np.ma.ones((4, 3, 2, 2)) - mask3 = np.full((4, 3, 2, 2), False) - mask3[0, 0, 0, 0] = True - data3 = np.ma.array(data3, mask=mask3) - - time = iris.coords.DimCoord([14, 45], - standard_name='time', - bounds=[[1., 30.], [30., 60.]], - units=Unit('days since 1850-01-01', - calendar='gregorian')) - time2 = iris.coords.DimCoord([45, 73, 104, 134], - standard_name='time', - bounds=[ - [30., 60.], - [60., 90.], - [90., 120.], - [120., 150.]], - units=Unit( - 'days since 1850-01-01', - calendar='gregorian')) - time3 = iris.coords.DimCoord([104, 134], - standard_name='time', - bounds=[ - [90., 120.], - [120., 150.]], - units=Unit( - 'days since 1850-01-01', - calendar='gregorian')) - day_time = iris.coords.DimCoord([1., 2.], - standard_name='time', - bounds=[[0.5, 1.5], [1.5, 2.5]], - units=Unit( - 'days since 1850-01-01', - calendar='gregorian')) - yr_time = iris.coords.DimCoord([14., 410.], - standard_name='time', - bounds=[[1., 30.], [395., 425.]], - units=Unit('days since 1850-01-01', - calendar='gregorian')) - yr_time2 = iris.coords.DimCoord([1., 367., 733., 1099.], - standard_name='time', - bounds=[ - [0.5, 1.5], - [366, 368], - [732, 734], - [1098, 1100], - ], - units=Unit('days since 1850-01-01', - calendar='gregorian')) + # Make various time arrays + time_args = { + 'standard_name': 'time', + 'units': Unit('days since 1850-01-01', calendar='gregorian') + } + monthly1 = iris.coords.DimCoord([14, 45], **time_args) + monthly2 = iris.coords.DimCoord([45, 73, 104, 134], **time_args) + monthly3 = iris.coords.DimCoord([104, 134], **time_args) + yearly1 = iris.coords.DimCoord([14., 410.], **time_args) + yearly2 = iris.coords.DimCoord([1., 367., 733., 1099.], **time_args) + daily1 = iris.coords.DimCoord([1., 2.], **time_args) + for time in [monthly1, monthly2, monthly3, yearly1, yearly2, daily1]: + time.guess_bounds() + + # Other dimensions are fixed zcoord = iris.coords.DimCoord([0.5, 5., 50.], standard_name='air_pressure', long_name='air_pressure', @@ -74,6 +42,7 @@ def setUp(self): [25., 250.]], units='m', attributes={'positive': 'down'}) + coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) lons = iris.coords.DimCoord([1.5, 2.5], standard_name='longitude', long_name='longitude', @@ -87,26 +56,29 @@ def setUp(self): units='degrees_north', coord_system=coord_sys) - coords_spec4 = [(time, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube1 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec4) + data1 = np.ma.ones((2, 3, 2, 2)) + data2 = np.ma.ones((4, 3, 2, 2)) + mask2 = np.full((4, 3, 2, 2), False) + mask2[0, 0, 0, 0] = True + data2 = np.ma.array(data2, mask=mask2) + + coords_spec1 = [(monthly1, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube1 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec1) - coords_spec5 = [(time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube2 = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5) + coords_spec2 = [(monthly2, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube2 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec2) - coords_spec6 = [(time3, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube3 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec6) + coords_spec3 = [(monthly3, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube3 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec3) - coords_spec4_yr = [(yr_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube1_yr = iris.cube.Cube(data2, - dim_coords_and_dims=coords_spec4_yr) + coords_spec4 = [(yearly1, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube4 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec4) - coords_spec5_yr = [(yr_time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube2_yr = iris.cube.Cube(data3, - dim_coords_and_dims=coords_spec5_yr) - coords_spec_day = [(day_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube1_day = iris.cube.Cube(data2, - dim_coords_and_dims=coords_spec_day) + coords_spec5 = [(yearly2, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube5 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec5) + coords_spec6 = [(daily1, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube6 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec6) def test_compute_statistic(self): """Test statistic.""" @@ -127,7 +99,7 @@ def test_compute_full_statistic_mon_cube(self): self.assert_array_equal(stats['mean'].data, expected_full_mean) def test_compute_full_statistic_yr_cube(self): - data = [self.cube1_yr, self.cube2_yr] + data = [self.cube4, self.cube5] stats = multi_model_statistics(data, 'full', ['mean']) expected_full_mean = np.ma.ones((4, 3, 2, 2)) expected_full_mean.mask = np.zeros((4, 3, 2, 2)) @@ -141,7 +113,7 @@ def test_compute_overlap_statistic_mon_cube(self): self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) def test_compute_overlap_statistic_yr_cube(self): - data = [self.cube1_yr, self.cube1_yr] + data = [self.cube4, self.cube4] stats = multi_model_statistics(data, 'overlap', ['mean']) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) @@ -172,19 +144,22 @@ def test_compute_min(self): def test_put_in_cube(self): """Test put in cube.""" cube_data = np.ma.ones((2, 3, 2, 2)) - stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=[1,2]) + stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=[1, 2]) self.assert_array_equal(stat_cube.data, self.cube1.data) def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_data([self.cube1, self.cube1], - "mean", span='overlap') + "mean", + span='overlap') expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) def test_assemble_full_data(self): """Test full data.""" - comp_full_mean = _assemble_data([self.cube1, self.cube2], "mean", span='full') + comp_full_mean = _assemble_data([self.cube1, self.cube2], + "mean", + span='full') expected_full_mean = np.ma.ones((5, 3, 2, 2)) expected_full_mean.mask = np.ones((5, 3, 2, 2)) expected_full_mean.mask[1] = False @@ -198,14 +173,39 @@ def test_plev_fix(self): def test_set_common_calendar(self): """Test set common calenar.""" + cube1 = self.cube1 + time1 = cube1.coord('time') + t_unit1 = time1.units + dates = t_unit1.num2date(time1.points) + + t_unit2 = Unit('days since 1850-01-01', calendar='gregorian') + time2 = t_unit2.date2num(dates) + cube2 = self.cube1.copy() + cube2.coord('time').points = time2 + cube2.coord('time').units = t_unit2 + _set_common_calendar([cube1, cube2]) + self.assertEqual(cube1.coord('time'), cube2.coord('time')) + + def test_get_time_slice_all(self): + """Test get time slice if all cubes have data.""" cubes = [self.cube1, self.cube2] - # TODO: complete this test + result = _get_time_slice(cubes, time=45) + expected = [self.cube1[1].data, self.cube2[0].data] + self.assert_array_equal(expected, result) + + def test_get_time_slice_part(self): + """Test get time slice if all cubes have data.""" + cubes = [self.cube1, self.cube2] + result = _get_time_slice(cubes, time=14) + masked = np.ma.empty(list(cubes[0].shape[1:])) + masked.mask = True + expected = [self.cube1[0].data, masked] + self.assert_array_equal(expected, result) def test_raise_daily(self): """Test raise for daily input data.""" with self.assertRaises(ValueError): - _set_common_calendar([self.cube1_day]) - + _set_common_calendar([self.cube6]) if __name__ == '__main__': From aef2578047cca45bb950267cc0618d901ffc65ce Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 29 Jun 2020 17:31:07 +0200 Subject: [PATCH 16/34] Valeriu's suggestions --- doc/recipe/preprocessor.rst | 6 +++++- esmvalcore/preprocessor/_multimodel.py | 11 +++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 0456f12108..56d4159dcf 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -663,6 +663,10 @@ from a statistical point of view, this is needed since weights are not yet implemented; also higher dimensional data is not supported (i.e. anything with dimensionality higher than four: time, vertical axis, two horizontal axes). +Input datasets may have different time coordinates. The multi-model statistics +preprocessor sets a common time coordinate on all datasets. As the number of +days in a year may vary between calendars, (sub-)daily data are not supported. + .. code-block:: yaml preprocessors: @@ -674,7 +678,7 @@ dimensionality higher than four: time, vertical axis, two horizontal axes). see also :func:`esmvalcore.preprocessor.multi_model_statistics`. -When calling the module inside diagnostic scripts, the input must be given +When calling the module inside diagnostic scripts, the input must be given as a list of cubes. The output will be saved in a dictionary where each entry contains the resulting cube with the requested statistic operations. diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 3ace7e860e..fc047e5c05 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -150,9 +150,9 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube -def _set_common_calendar(cubes): +def _unify_time_coordinates(cubes): """ - Make sure all cubes' use the same standard calendar. + Make sure all cubes' use the same standard calendar and time units. Cubes may have different calendars. This function extracts the date information from the cube and re-constructs a default calendar, @@ -189,6 +189,7 @@ def _set_common_calendar(cubes): # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] cube.coord('time').units = t_unit + cube.coord('time').guess_bounds() def _get_time_slice(cubes, time): @@ -271,6 +272,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): ------ ValueError If span is neither overlap nor full. + ValueError + If the time frequency of the input data not yearly or monthly. """ logger.debug('Multimodel statistics: computing: %s', statistics) @@ -284,8 +287,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): cubes = products statistic_products = {} - # Make cubes share the same calendar, so time points are comparable - _set_common_calendar(cubes) + # Reset time coordinates and make cubes share the same calendar + _unify_time_coordinates(cubes) if span == 'overlap': # check if we have any time overlap From 24294dffeb4ca6d91113f0d04dda78ea27741fcd Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 29 Jun 2020 17:37:35 +0200 Subject: [PATCH 17/34] fix tests --- esmvalcore/preprocessor/_multimodel.py | 1 + tests/unit/preprocessor/_multimodel/test_multimodel.py | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fc047e5c05..efd8267d10 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -189,6 +189,7 @@ def _unify_time_coordinates(cubes): # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] cube.coord('time').units = t_unit + cube.coord('time').bounds = None cube.coord('time').guess_bounds() diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 7e98642cdb..e0c01000d4 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -12,12 +12,11 @@ _compute_statistic, _get_time_slice, _plev_fix, _put_in_cube, - _set_common_calendar) + _unify_time_coordinates) class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" - def setUp(self): """Prepare tests.""" # Make various time arrays @@ -171,7 +170,7 @@ def test_plev_fix(self): expected_data = np.ma.ones((3, 2, 2)) self.assert_array_equal(expected_data, fixed_data) - def test_set_common_calendar(self): + def test_unify_time_coordinates(self): """Test set common calenar.""" cube1 = self.cube1 time1 = cube1.coord('time') @@ -183,7 +182,7 @@ def test_set_common_calendar(self): cube2 = self.cube1.copy() cube2.coord('time').points = time2 cube2.coord('time').units = t_unit2 - _set_common_calendar([cube1, cube2]) + _unify_time_coordinates([cube1, cube2]) self.assertEqual(cube1.coord('time'), cube2.coord('time')) def test_get_time_slice_all(self): @@ -205,7 +204,7 @@ def test_get_time_slice_part(self): def test_raise_daily(self): """Test raise for daily input data.""" with self.assertRaises(ValueError): - _set_common_calendar([self.cube6]) + _unify_time_coordinates([self.cube6]) if __name__ == '__main__': From 35210a5c6ba7f135f046a664f56b8f6b90fe3839 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 2 Jul 2020 14:02:54 +0200 Subject: [PATCH 18/34] Realize data before making time slices --- esmvalcore/preprocessor/_multimodel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fccdcc6d4b..67d4b3364d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -237,7 +237,7 @@ def _get_time_slice(cubes, time): idx = int(np.argwhere(cube_time == time)) subset = cube[idx].data else: - subset = np.ma.empty(list(cubes[0].shape[1:])) + subset = np.ma.empty(list(cube.shape[1:])) subset.mask = True time_slice.append(subset) return time_slice @@ -257,6 +257,9 @@ def _assemble_data(cubes, statistic, span='overlap'): new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) + # Realize all cubes at once instead of separately for each time slice + [cube.data for cube in cubes] + # Make time slices and compute stats for i, time in enumerate(new_times): time_data = _get_time_slice(cubes, time) From d2fddeba2199f2c20df8848004a5178737536db0 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 2 Jul 2020 15:19:55 +0200 Subject: [PATCH 19/34] Avoid codacy 'pointless-statement' message --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 67d4b3364d..0a7fddaf5a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -258,7 +258,7 @@ def _assemble_data(cubes, statistic, span='overlap'): stats_data = np.ma.zeros(new_shape) # Realize all cubes at once instead of separately for each time slice - [cube.data for cube in cubes] + _ = [cube.data for cube in cubes] # Make time slices and compute stats for i, time in enumerate(new_times): From cc51e1531a91d195160891e55941a0546711bfb7 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 4 Aug 2020 17:56:22 +0200 Subject: [PATCH 20/34] Address Bouwe's comments --- esmvalcore/preprocessor/_multimodel.py | 14 +++++++------- .../preprocessor/_multimodel/test_multimodel.py | 6 +----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 0a7fddaf5a..d213ec43a3 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -219,7 +219,10 @@ def _unify_time_coordinates(cubes): ] else: # (sub)daily data - raise ValueError("Multimodel only supports yearly or monthly data") + logger.warning( + "Multimodel encountered (sub)daily data. Attempting to" + " continue, but might fail for incompatible calendars.") + break # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] @@ -255,7 +258,7 @@ def _assemble_data(cubes, statistic, span='overlap'): # Target array to populate with computed statistics new_shape = [n_times] + list(cubes[0].shape[1:]) - stats_data = np.ma.zeros(new_shape) + stats_data = np.ma.zeros(new_shape, dtype=np.dtype('float32')) # Realize all cubes at once instead of separately for each time slice _ = [cube.data for cube in cubes] @@ -312,9 +315,6 @@ def multi_model_statistics(products, span, statistics, output_products=None): ------ ValueError If span is neither overlap nor full. - ValueError - If the time frequency of the input data not yearly or monthly. - """ logger.debug('Multimodel statistics: computing: %s', statistics) if len(products) < 2: @@ -350,8 +350,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic statistic_cube = _assemble_data(cubes, statistic, span) - statistic_cube.data = np.ma.array(statistic_cube.data, - dtype=np.dtype('float32')) + # statistic_cube.data = np.ma.array(statistic_cube.data, + # dtype=np.dtype('float32')) if output_products: # Add to output product and log provenance diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index d01142dacb..42b28a9525 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -17,6 +17,7 @@ class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" + def setUp(self): """Prepare tests.""" # Make various time arrays @@ -209,11 +210,6 @@ def test_get_time_slice_part(self): expected = [self.cube1[0].data, masked] self.assert_array_equal(expected, result) - def test_raise_daily(self): - """Test raise for daily input data.""" - with self.assertRaises(ValueError): - _unify_time_coordinates([self.cube6]) - if __name__ == '__main__': unittest.main() From 7c182a22605b2f84686357360b7594773f172c5e Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 5 Aug 2020 16:02:43 +0200 Subject: [PATCH 21/34] Update esmvalcore/preprocessor/_multimodel.py Co-authored-by: Bouwe Andela --- esmvalcore/preprocessor/_multimodel.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index d213ec43a3..09481b6c45 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -350,8 +350,6 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic statistic_cube = _assemble_data(cubes, statistic, span) - # statistic_cube.data = np.ma.array(statistic_cube.data, - # dtype=np.dtype('float32')) if output_products: # Add to output product and log provenance From a663ea68d0042ee7ad2591ed58ba596b0343737a Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 7 Aug 2020 10:20:30 +0200 Subject: [PATCH 22/34] Don't change the calendar if not necessary --- esmvalcore/preprocessor/_multimodel.py | 37 +++++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index d213ec43a3..86444d2857 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -185,21 +185,30 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube +def _get_consistent_time_unit(cubes): + """Return cubes' time unit if consistent, standard calendar otherwise.""" + t_units = [cube.coord('time').units for cube in cubes] + if len(set(t_units)) == 1: + return t_units[0] + return cf_units.Unit("days since 1850-01-01", calendar="standard") + + def _unify_time_coordinates(cubes): """ - Make sure all cubes' use the same standard calendar and time units. + Make sure all cubes' share the same time coordinate. + + This function extracts the date information from the cube and + reconstructs the time coordinate, resetting the actual dates to the + 15th of the month or 1st of july for yearly data (consistent with + `regrid_time`), so that there are no mismatches in the time arrays. - Cubes may have different calendars. This function extracts the date - information from the cube and re-constructs a default calendar, - resetting the actual dates to the 15th of the month or 1st of july for - yearly data (consistent with `regrid_time`), so that there are no - mismatches in the time arrays. + If cubes have different time units, it will use reset the calendar to + a default gregorian calendar with unit "days since 1850-01-01". - Doesn't work for (sub)daily data, because different calendars may have + Might not work for (sub)daily data, because different calendars may have different number of days in the year. """ - # The default time unit - t_unit = cf_units.Unit("days since 1850-01-01", calendar="standard") + t_unit = _get_consistent_time_unit(cubes) for cube in cubes: # Extract date info from cube @@ -219,10 +228,12 @@ def _unify_time_coordinates(cubes): ] else: # (sub)daily data - logger.warning( - "Multimodel encountered (sub)daily data. Attempting to" - " continue, but might fail for incompatible calendars.") - break + if cube.coord('time').units != t_unit: + logger.warning( + "Multimodel encountered (sub)daily data and inconsistent " + "time units or calendars. Attempting to continue, but " + "might produce unexpected results.") + dates = [cell.point for cell in cube.coord('time').cells()] # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] From c8b5f4b4d1cfe452953313b3f921c2e6111fd756 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 7 Aug 2020 12:09:43 +0200 Subject: [PATCH 23/34] Use template cube's calendar and don't slice by time --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ad478cebb1..82fba4901a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -133,7 +133,7 @@ def _compute_statistic(data, statistic_name): def _put_in_cube(template_cube, cube_data, statistic, t_axis): """Quick cube building and saving.""" - tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") + tunits = template_cube.coord('time').units times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits) coord_names = [c.long_name for c in template_cube.coords()] @@ -279,7 +279,7 @@ def _assemble_data(cubes, statistic, span='overlap'): time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) - template = cubes[0][:n_times] + template = cubes[0] stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube From fcb9955d0b3dfdb062444ae803bb9611c45d7060 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 10 Aug 2020 09:36:20 +0200 Subject: [PATCH 24/34] Use num2date rather than cells() method --- esmvalcore/preprocessor/_multimodel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 82fba4901a..6f2f306d73 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -212,8 +212,9 @@ def _unify_time_coordinates(cubes): for cube in cubes: # Extract date info from cube - years = [cell.point.year for cell in cube.coord('time').cells()] - months = [cell.point.month for cell in cube.coord('time').cells()] + coord = cube.coord('time') + years = [p.year for p in coord.units.num2date(coord.points)] + months = [p.year for p in coord.units.num2date(coord.points)] # Reconstruct default calendar if 0 not in np.diff(years): @@ -228,12 +229,13 @@ def _unify_time_coordinates(cubes): ] else: # (sub)daily data - if cube.coord('time').units != t_unit: + coord = cube.coord('time') + if coord.units != t_unit: logger.warning( "Multimodel encountered (sub)daily data and inconsistent " "time units or calendars. Attempting to continue, but " "might produce unexpected results.") - dates = [cell.point for cell in cube.coord('time').cells()] + dates = coord.units.num2date(coord.points) # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] From 46337f73ca9184cec7e295137a34f43069fa47fb Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 10 Aug 2020 09:37:49 +0200 Subject: [PATCH 25/34] Apply suggestions from code review Co-authored-by: Bouwe Andela --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 82fba4901a..cc0dc3c68e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -236,7 +236,7 @@ def _unify_time_coordinates(cubes): dates = [cell.point for cell in cube.coord('time').cells()] # Update the cubes' time coordinate (both point values and the units!) - cube.coord('time').points = [t_unit.date2num(date) for date in dates] + cube.coord('time').points = t_unit.date2num(dates) cube.coord('time').units = t_unit cube.coord('time').bounds = None cube.coord('time').guess_bounds() @@ -249,7 +249,7 @@ def _get_time_slice(cubes, time): cube_time = cube.coord('time').points if time in cube_time: idx = int(np.argwhere(cube_time == time)) - subset = cube[idx].data + subset = cube.data[idx] else: subset = np.ma.empty(list(cube.shape[1:])) subset.mask = True From c03358f4ea1c71ea33fa462ba304bb9ac6bad9a1 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 16:38:26 +0200 Subject: [PATCH 26/34] fix bug for monthly data in unify time --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fdf279d828..cba0a6d4ec 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -214,7 +214,7 @@ def _unify_time_coordinates(cubes): # Extract date info from cube coord = cube.coord('time') years = [p.year for p in coord.units.num2date(coord.points)] - months = [p.year for p in coord.units.num2date(coord.points)] + months = [p.month for p in coord.units.num2date(coord.points)] # Reconstruct default calendar if 0 not in np.diff(years): From efa63728abfd9d70b23aed65f62dcf84f9edcd1c Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 30 Sep 2020 15:19:30 +0200 Subject: [PATCH 27/34] add time bounds to output cube --- esmvalcore/preprocessor/_multimodel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index cba0a6d4ec..fe3bd7dfc3 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -135,6 +135,8 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): """Quick cube building and saving.""" tunits = template_cube.coord('time').units times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits) + times.bounds = None + times.guess_bounds() coord_names = [c.long_name for c in template_cube.coords()] coord_names.extend([c.standard_name for c in template_cube.coords()]) From bb8e9c5b6a06997557d2f3fcafe6a866c32c55c4 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 7 Jan 2021 09:18:59 +0100 Subject: [PATCH 28/34] Add missing var_name attribute to time coordinate --- esmvalcore/preprocessor/_multimodel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 8b4ac6b694..5a820dab9e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -134,7 +134,10 @@ def _compute_statistic(data, statistic_name): def _put_in_cube(template_cube, cube_data, statistic, t_axis): """Quick cube building and saving.""" tunits = template_cube.coord('time').units - times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits) + times = iris.coords.DimCoord(t_axis, + standard_name='time', + units=tunits, + var_name='time') times.bounds = None times.guess_bounds() From 13ac659cffd31a1b769c51860e1ea6a977212327 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 7 Jan 2021 17:12:02 +0100 Subject: [PATCH 29/34] Force regrid of daily data and be more consistent with regrid time. The new regression test for the daily statistics of a 365-day calendar failed because some days had time points at 00:00 and others at 12:00. Since we're planning to move time handling to regrid time altogether, this behaviour is consistent with regrid_time (see issue #744). --- esmvalcore/preprocessor/_multimodel.py | 35 ++++++++++++++------------ 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 5a820dab9e..8892f40957 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -9,7 +9,6 @@ It operates on different (time) spans: - full: computes stats on full dataset time; - overlap: computes common time overlap between datasets; - """ import logging @@ -28,10 +27,8 @@ def _plev_fix(dataset, pl_idx): """Extract valid plev data. - this function takes care of situations - in which certain plevs are completely - masked due to unavailable interpolation - boundaries. + this function takes care of situations in which certain plevs are + completely masked due to unavailable interpolation boundaries. """ if np.ma.is_masked(dataset): # keep only the valid plevs @@ -50,9 +47,9 @@ def _plev_fix(dataset, pl_idx): def _quantile(data, axis, quantile): """Calculate quantile. - Workaround for calling scipy's mquantiles with arrays of >2 dimensions - Similar to iris' _percentiles function, see their discussion: - https://github.com/SciTools/iris/pull/625 + Workaround for calling scipy's mquantiles with arrays of >2 + dimensions Similar to iris' _percentiles function, see their + discussion: https://github.com/SciTools/iris/pull/625 """ # Ensure that the target axis is the last dimension. data = np.rollaxis(data, axis, start=data.ndim) @@ -199,8 +196,7 @@ def _get_consistent_time_unit(cubes): def _unify_time_coordinates(cubes): - """ - Make sure all cubes' share the same time coordinate. + """Make sure all cubes' share the same time coordinate. This function extracts the date information from the cube and reconstructs the time coordinate, resetting the actual dates to the @@ -220,27 +216,34 @@ def _unify_time_coordinates(cubes): coord = cube.coord('time') years = [p.year for p in coord.units.num2date(coord.points)] months = [p.month for p in coord.units.num2date(coord.points)] + days = [p.day for p in coord.units.num2date(coord.points)] # Reconstruct default calendar if 0 not in np.diff(years): # yearly data - dates = [datetime(year, 7, 1) for year in years] + dates = [datetime(year, 7, 1, 0, 0, 0) for year in years] elif 0 not in np.diff(months): # monthly data dates = [ - datetime(year, month, 15) + datetime(year, month, 15, 0, 0, 0) for year, month in zip(years, months) ] - else: - # (sub)daily data - coord = cube.coord('time') + elif 0 not in np.diff(days): + # daily data + dates = [ + datetime(year, month, day, 0, 0, 0) + for year, month, day in zip(years, months, days) + ] if coord.units != t_unit: logger.warning( "Multimodel encountered (sub)daily data and inconsistent " "time units or calendars. Attempting to continue, but " "might produce unexpected results.") - dates = coord.units.num2date(coord.points) + else: + raise ValueError( + "Multimodel statistics preprocessor currently does not " + "support sub-daily data.") # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = t_unit.date2num(dates) From 1becb92c1b15e23919b002630c49d2a687418807 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 7 Jan 2021 17:51:45 +0100 Subject: [PATCH 30/34] Add long name to new time coordinates --- esmvalcore/preprocessor/_multimodel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 8892f40957..c899b07627 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -134,7 +134,8 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits, - var_name='time') + var_name='time', + long_name='time') times.bounds = None times.guess_bounds() From 2bd2d627491251389f81cf0076a133794dab52ad Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 12 Jan 2021 17:23:10 +0100 Subject: [PATCH 31/34] Temporarily bypass coordinate check The reference data contains some metadata that is redundant or inconsistent. After updating the reference data, this can be reverted. --- tests/sample_data/multimodel_statistics/test_multimodel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sample_data/multimodel_statistics/test_multimodel.py b/tests/sample_data/multimodel_statistics/test_multimodel.py index a527a9b913..d46482ec33 100644 --- a/tests/sample_data/multimodel_statistics/test_multimodel.py +++ b/tests/sample_data/multimodel_statistics/test_multimodel.py @@ -150,7 +150,8 @@ def multimodel_regression_test(cubes, span, name): # Compare coords for this_coord, other_coord in zip(result_cube.coords(), reference_cube.coords()): - assert this_coord == other_coord + # assert this_coord == other_coord + np.all(this_coord.points == other_coord.points) # temporary bypass # remove Conventions which are added by Iris on save reference_cube.attributes.pop('Conventions', None) From df689b81799c767445824466eaff125e56094598 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 13 Jan 2021 12:01:15 +0100 Subject: [PATCH 32/34] Revert "Temporarily bypass coordinate check" This reverts commit 2bd2d627491251389f81cf0076a133794dab52ad. --- tests/sample_data/multimodel_statistics/test_multimodel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/sample_data/multimodel_statistics/test_multimodel.py b/tests/sample_data/multimodel_statistics/test_multimodel.py index d46482ec33..a527a9b913 100644 --- a/tests/sample_data/multimodel_statistics/test_multimodel.py +++ b/tests/sample_data/multimodel_statistics/test_multimodel.py @@ -150,8 +150,7 @@ def multimodel_regression_test(cubes, span, name): # Compare coords for this_coord, other_coord in zip(result_cube.coords(), reference_cube.coords()): - # assert this_coord == other_coord - np.all(this_coord.points == other_coord.points) # temporary bypass + assert this_coord == other_coord # remove Conventions which are added by Iris on save reference_cube.attributes.pop('Conventions', None) From 395a675c811720897f7b6730b12c7915ae51a64f Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 13 Jan 2021 12:57:42 +0100 Subject: [PATCH 33/34] Update reference data --- .../timeseries_daily_365_day-full-mean.nc | Bin 18962 -> 25378 bytes .../timeseries_daily_365_day-overlap-mean.nc | Bin 25378 -> 25378 bytes .../timeseries_daily_gregorian-full-mean.nc | Bin 18954 -> 25558 bytes ...timeseries_daily_gregorian-overlap-mean.nc | Bin 25558 -> 25558 bytes ...ies_daily_proleptic_gregorian-full-mean.nc | Bin 18962 -> 25378 bytes ..._daily_proleptic_gregorian-overlap-mean.nc | Bin 25378 -> 25378 bytes .../timeseries_monthly-full-mean.nc | Bin 18962 -> 22899 bytes .../timeseries_monthly-overlap-mean.nc | Bin 22899 -> 22899 bytes 8 files changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/sample_data/multimodel_statistics/timeseries_daily_365_day-full-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_daily_365_day-full-mean.nc index 59aeda4cb86305d43bb002137f46918db4527d9e..d581f72c1c3a85fc1f19bf45e9cb169fb2092f25 100644 GIT binary patch literal 25378 zcmeHP3tUxI*54NpQAw$MXNG*fQba89QDo>JP!o~ynUx91%}51?d+t5=oO@8})2Sva zDw|$2MrBh?R%%&J52Mmvekz$|R$rlMrj}CKq)pkk_Bnf9E;qnzs;`+JTNnSc_St)# zz1Lp*an|1FUOP4;>!h~b+X@+CVnmdPvoHD2_bMdJc4C<4`+0oP1Wp{AF)1RpHX^~k zA9<2Wwsov?@l2$H^G|cIZG>>it4=+rBtk?+2!3EV#$F)PxkP&rL;q&wAh)cf)Lo_( zxjhtrDv^AoEsW6TF7p(X6rXoiVzFDBU3l4fDM`bUQW7sOye##+AxT4$lTu@oWNMd) zwdJ9t*yk?RD2tTl$crXqdB_?xBx#skM*IE?<6F<2h_T~iPLXV+tF(t5Evni%7~4yT zR+Ky1mhLSs(zujTB1;{A<;KnJY>k~HT&}J%tIOrw9O=jrjFBB>u9M@>CxJ4gi)5;%p7FS${sg)iV&S6 zgl)%#`C7imtsUn{LUf@NvWA64^WDYLP*e_8>#WIRGlY0asZD{@OoI?5a)n>?r#R=% z#|Z#>d0J4#@(B@9c$d8RR$5npJ36G20P`ky9!;hMYk1~4t4I- zDZEuZrg3W$c`|il7VSt0{o9nCza5&W(d+Z?ebVFTA|vs39{lo|(`(8W@jx-ng}a>RuEbtHsDG`L(grU9#%*p&KuHg>=1I8T?IU)#Or zG-X|`^)8WS>&%sKewNJGaevG~N5y}2xyH&ESCnJAJa7h8EN@a^{&@GfNzPD%T)gNK zBf2ak39ak_S1Qcu%ym{&j2r@(lLjNiVr8S-BtwClC%Wkw?n0T)B|6fa)Rks9d*Vt# zNl95@QE|Rz&vP`)(HP}CDPATIP8e!nQTOay_P8YLEA1e*Lduo{XdFq;7&jp^XY#m- zIcflv^MNcj4~#qwg-eyXT}&mAL0VNPKOl@B7{(_OFT|R`Yfl=3iP?55DciNL>7EiS z#;XW!tMGVb*TM9UsDxbF@bqlR9M9?H3@-!6cTfUaJJG>-E)S<;WH~%ab3Ds&R$2A( zf3FIn770Q5EGIVXxh#iQQ7oV3v=2*vv(j6Kuy~fwLj}uW`EsU^f#X@ey|@X9XZdoq zB?G7D_>N9=F#dV9tmqUL&*?jd#dCS270ST!xja1Vb3DsYtC?W>)c#KX?hXXwSw2rz zEQjT*MO-jF%RfC#4$JQu7SD3d2#aSq@nP{Sr&m~fp|Zn-uy~gLUt#f_zIPMxvlhl0NG14~-G?N{Z)jM)`#B%%h6R;-}PS7GmnC z#TR8w8k=jsa75US6q%0#?_w-b4?e)Js_G4|fxCv%v&X>B+{_=!&Jt9e`V;m&m$S?) z$hqEe?DCVj=8T*&+_4MgK;?I2#;dL^O>4f6>_TQAcd&%Pk~+_1ZwZ&6DQu#S;PjPq zZdw%X?e2R%KZBn#_;;Dr;r{i$qry{}J+CWw^hGX4JFuP@omi>%hK}%+%8A5!Vl3ZR zcDnuCXz6m4wnrZOsg9I-zy5)BNtW~6Q-9QR<4J&%FiyRUvyX#uY5RK)HM5P9_fx!p z?1y8NC15Y0dD_v*ozTrhk9ZS%Tm}lG4J_*H$LfxLjCs2pr@T{+ zxa@Ky5ncR|g^XPm3Il%BmHXj+N5_#JIxe|gM;D^BjB?B!Q;wQ#7S_FO;kvmN77%4c z_;G2ZAM0v#=x^xQzE;OvqRN*o%ze#5T9t)u_gENLY@uqtg>QzKc=sF=hjUHrpJL*Y z-<$Zj*hIfB<=EW09AA8I;Wnbh`z<8yvoPQZ6Bqp5#NgkVNT&2_ECXkj8yI-Rfj(R+svL*Dh_VP83J_Ln1Xy@doS+m1*J@3*q> zW^V&}KLelL=f@+<{g`;VA8StYV{@H_Yrn8?=^~O()UC6Hn>$(X^fz&8qKT`&Gcf;{ zfveUSNPNOTU6zI3M4Mj>;O3VC$V?7k&)@(Gh8ZY5-@t=Z?zIaI{G-4?WuAe})l}A( zEKHhWA!VwChwl$yDN!7irFM{kBOm$E>tjD&yvC2CfAAxIriCx^Ej;>V0D-Rpxc069 zE+NXCYoeOyjD99w=xbu-nFc=TYasSX3-g|$vVRxAd&dI!3IW`a5x|y_1|Aw^VEl)E z?Az(b*8>O#`jL1jfPViVnVSOmU<;K!5Wu5m0Jp8yF@mV`!#ZC0n~pn&>zFcBNBaya z_c#kPJ`7-4Z2$)*1hAXv(R3YInL38-nR`T%5T)PP)G2^gME^W!V#Oh9OKR8lYp7jK zKkgx$JGID<`9y=e`|)8HKPo1h(5St;eQ%((Fwt(EfuqkEI5tm5hOT3LzK**u)8SgE zV4pZKfAJcIoQO{T%ziX#s`u#qXEcYRBUjU~Nr9}r& z7Zt#;znED3XA@<91K(9p8yxgw?;$@8cBQ)|m0lG<8zT2W($jzd(!Vm%^Gg#;mYR6z zZWFcitYa<{eX0y}xy!(7n=RySv0%kpxQl4s74(cEYG2a9(f9xskPRJ3HZf+tfs5xF zI9O{TYLA6;AG5G^m4#~RD^sb@%(=wG-EI@3VPe=A6K$eR+|fh18}i zbkwZU5nWAfO|+vjfOWS7urG&fU}6B<`%&BVq5S9Txc&ki-zVt!wvUeT&ZPfNI@-KP z_1Y1@h>rqTy(<8V?DT%3M=0$T*(7s-jt3KU6m0S#lIYmN04f$xnG5I{L?fq>{S#FW z^5g0xKemtXPr1koE5+mtw~QrfB#s=rFA;?kv(5R^?f!)hef3IwouvE!lD=p zSGTd?{(|iKbE*TCe+kjmANp`gtq=Rkd`R~Auz}eQ@{mA)&7illoA3a>|i0u^heK<#?;G9LJ`Wql)O&>%H*Y=EX%160YzfneuGx z;lr~-XbeoX@b^R-Lz65lr}1Y6jX&4s_|R^K4z95gU$j~^@d`>`{{j|s#5a6hY~^>aGzw*p8lC;O$ov6U$8B@+ci z$<$u?pXr!J^GshC*#(VNFZTDNav6=uRTieh!eOGsw*t_&1(5ZKj>**DHy+jT&u?{H zwM9qAH>o}}4(=ctxtzu~>i6FK09NGDxY(7-eTEOi|KdaYdwf`E`7nVf?P0R7M`=8r zL*vTjCNetF_|TE`Gs26SkzPcldyzhh+AYe9k&#}c-fm$19R?O(Z{S&?f&~UriOftd z{(HO^=5`HJcW5}~*05)`hMwQ~u|gQg>}sHWHv_#>3@kXyz}S5nY7S|*aIJG7Tdh!0PUK~v#9OOm2FHHo9a;r?- zN|ae-B8jNO%U(om^5T@KUaXkr#fCGyxWA_tZy6-pqIPX(;@emgkJ31{{cbAP#oLaS zr+U%4jSu|?dvW>zAD$TN#cL@hB2uYsX?>APlyQjaL2Z2P8yY53J$r7{FmMyq;a&|r z?$+=Cr8(B!#Jcv>cRQH);(&qMJ~fbVh6g`n$A=xAU~2Lnk&!_qZO9xZaa&(`qLSsLEB(u0{-d9Y)uAN?-$qae|b92x^3?d``nq9yG$jET~) zewGL0F7lw)2oIha>cMEskHQ6h9GmOMu6cg6zRZv5Zd!{C@S^bOKbva&4 z&i3LU+2R*h8n`t;^C_)Gt|DCyoMPacGB3{Yc=6XKG<-$2wl9X-s7k64-JUGmYLBq-R?xXg-N5i76G>=jrJoCH&UKvSq#v9cCYsilF z``|t7!>BwTI?wiD-F~vsJ=9-5vXDpf-MkzNNfRyXeVNuYG@t$RWgQo7)KPICjSKhd zc={b3tG4TybgzYuM0z2u`-r~2z(VYJ3!ks{V*S%zWZmvX_dk14@RSz`YrV+++Jrb_ zV&H3Jb3}{oAbTacg32(*=fj8NedssEhxOxp=t4B?brXAEF;TUh`pH`+KKRf?+y^wj zkPZ*d@S^+$FM4e7(t2Patp^tFHgvL0)4;c+NB1=9htz)`p#IzX0}ZQoYPf;gq5GL$ z3_jb3t%K=V=>{?~sZX^ru&f;++0FxGL-zTKThvR7i(W!LqlLR@x9zJ^^**wtWO;|i z5A^Wc03)B6IFTc2qt#X&?@aS4i8kUq^+K~n;a3WqIK+}Ci{02E78&1cIDsQ-b8G5` zhrdfE%ygvJztg=tJKsK=Lh)1oP?Rcf$Dev2U1GYzJcShss}xo%tW_vFI{D%irYp=- zSfQ{=VYR|qg`$(nuP|L}CRPl_Jif3G@c*YvVGuA1dF;3Y%W3u8Ia~026 zs(8jq#WSu{JY$XG8S50!7^nITW3u8Ia~026s(8jq#WSu{JY$XG8S50!7{~ojVR9Je zDxR@a@r;#M5g@B4+VIag3nR$Dy*5a zs>w1xf8T)YQIq9ibh%L?S8{6U7;?&VGDc-*3TMwJ_H0T*;XwyfY?^)gRfvyOEFDs9 zCM-JV(~In-YoACbm5Wa6pJ%T`J?l z@mBWEPtK?bnKH4gNIDT$WXk15o_PA7A5Wi?AR{Y2_~Af&fme$!F45|5`SQz(@(bp< zlj1p?C4SnrmfSa)^;yY?V58eEf0xsC!7oF2`Tc_f%}t)KcbXyEUYQE9j-Dbz)8CO4 zsPlTYO4n>2k-_aPHXz2>Tq*IV9%belG1{f0Kbs@Z+bP>D+CHc0isECsZ8Z9fV^%?N zq2~!@THNw{&M%U-^!RXKO4WvUf6WaNX-m?Mc*8^n%z(f*SZp$-yr0zH9pS&%WM#4Mu$@60FwP+e4NOX5%kI`w`reV{!P5Nn=XW{ zbRn*OZuBr=4>|VWD8%}`e@YXdNx39LYW7wC;7OvIF67yHdA9YekMA>t{j%7{DZs4j zuKc5zD^sL6pDMUxN8TbylzTt`Y6cx{Q#}65vnmlO8pds-Srn>#Zk zmtZVIHdk{;gnpQ|jXl54&Y$O&hmp9fayphHImNtvc4{6ZS|SLc7#c%nhmhMI;5HLV8io0Q%EqhF|2Enr9 z&6CWJb$9SI-PHVFI?wH+S>VLxe>oJe;HKvPi+-!qx-7g5j6AI~&l(P$#^n-#@8{+> zH(AGqq?8AIOUv9I51sC9Zqg2Y;jK^aOx&d7Ir$yk2X~13`^A9fBS^u)F{UBbu z{VS_uIgrWjEdy=V(e;Tfj^&7Syv6a1#++kvKni*Azol=C=OrGpr4nV}h6!C%%U_h? z`WCw9l)2rWS;Zw~+U3oyufvp7q@pG_oLbTn!=`!hdTSw zd($!wCGST!^W%}ZGU?B?A1&AYa-@27?Y-mdxv&{I)b;w|}&>20J)yC0Da{aO@zsci|X- zy>OMcotBrQsvWB6ZJ~6URP{j7bgkH~T>G>$0w~?>I+CewE*0ZCHd5uBmN9EUQhc`lofhAl}{hAu?AK_s7r!My@Cx4kEOC)~hRu2ZboPkI27}=8EFMAxfTK z;vXt&?rf8ber?hTQPP3Tzp3Vm;-MmjR?hYUmKTC9no6Az!)JQ9;wruc>NODR-Jstu zJOS^!9Pv|_V~{~p=@V~-chuNU!s3D?*{$m_=$Qqh)KLB)G??z~bY|dcYOUs{sXAS%xpJ*Ez delta 2466 zcmZ`)drXs86u;lsuMf&s*EXe;KClqMf+8*QvQk7SQ1Jn_C>l3`4KR+bI4Ulhsl%uX z+aCjt#%MMqYW6VGC3eYXScGM2Gz*ToX!gg%@M>qBU^ljgg<=bU?< zzkBYftHSURVbCL*8;0_0wKAeKG@4PP7m+|VMkD#|SU|oA<*ci(Y zTxkpi!=m9MsPHVaFxN+U=mDAzObJeL$_YartDqYiwL_?=L@nb01n6F#tCnjKQo~eR zS{aFnp1^BI`)+r9d+dcyx39d!Tj2HKpWEy8)87&=BY81G8UzzTClm&Z8rg@C7D0Ew z2@UP?vTePc+j@Gpx5w;9zVnkDH0dOELDG-~le8n+x$7qJis^d(`Hxv91JEr+#6O%c zA=%72Uhnh*rWPoGF^?PF?^niyr9J_|$Qi!Ax!QOzaY#2M zfGb5OSRj^?4t;u$cryo1>aCP%qs&`+Biu>RnK_b?Qf#dFkG{^Xv@R2SA*6_VRnVGh zg}wNsn+oQL-b%IFd-*NMP7+6bFpf_WpB%z7MzPO_sa6r&*2+s}T$2O|&2rr+TkT#o zW9PGFM(kC?e%YF@h#qmB4jH^yQ`Z+L2MCs(1g`GJcB&O6r>=nX7BhrR4avO8JHFi^{LjI!*Nqx1lvNl zVX>-k|2f_Tjv%D03-$TCD4wk(EJ4ze1(OQnS*dehD`6|E zQ%VwNKtmUs=Cg4&z=KvGL2b9iyGp^B?S;Dwz4HiXBK=+!Sw|g-s=^{M z;Wo6XN!PNJvBNQeq@CMe>Kx{6{Ho{7Gg!+3qc%T$Uy`F~mOsUl33^(N!@i&m#w&Dk z$CUIr8w3|b2zbh2SG`xfO9raVu{neIMCcx0O*~d)7Qtj#3O(YY# z$4W<}u2^mSbnXg?gzMDsKI)la+{d1n0$4~9iub>L*$>GeGgmY1r|{W(Y|JuqFOt>u zrOE0gBe$%{nADdiYn}A_9?z%DHHXaa)`f!ANGSm2UMYDLgSAE8M6_l?C2YiFGZ*gr zvS=Sw=n{V=A272h%}L)mm6|_FU|c$iQF7qs*(Z=&|C*?C!Ta^!nzFF^>>-y8L!p5F z%$`#QirX}3-JFr9p?DIcRBl^`t462!bS2{)UU9}n^ubVMQ_I&_k3VQohCPP^Qq-0G zqrSoe$#pT~J4xtiu@}A3um=WocGfLoT=)J`C=XFm&rIx^t5&w$kihY`J7na#UR2c(wluzQLJ#{PIwmSUGc(&2@@a zcg#8lIjYjU3;%PIe;f{-t8QLXhlig`MvD*DHxwkE-SVZ%q`a8aR&e9cJ|zVhS!YNL Qw3Z9vbR|vW=eB_OFP-nELjV8( diff --git a/tests/sample_data/multimodel_statistics/timeseries_daily_365_day-overlap-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_daily_365_day-overlap-mean.nc index 060b6120adc62ab88924faa9d1fb6a5b28df1402..d581f72c1c3a85fc1f19bf45e9cb169fb2092f25 100644 GIT binary patch delta 947 zcmZ9LziSg=9Ea~cuQf?Yn))MYQmGapwO+idQmZB9TBH=AHXhW{K|Dbm1SPqlf>Vk* z2{8?Pag9(XLDGX;+U3;2P9+tbjemfU1{}IYKX3Wo;=x0B-n>uXLz1mMv9%|>h!EE) zDlF~m(#fZ^BzDCB^{MZ){crQLWUr0*yK0aUReF&b7cqL-KTiA8Df*d-4Lo>Me7m@E zdqGjEls{q8Saw)@^~RkgXL{7XpB+-QwanO#@1E3w02U|$2b4es)Ib9?K>%8y4LYC)5?9az16ZI498dxkPy-Fn1OaG)Ht2vJNEqmW z0W44i4k&>NsDTD(f&jEY8+1SqBuw) z>(YBz@(h0xZNqFfmp;pm&7%X^v32Q}?9PVWq3qaP`YSs&j|MNJW9!m+STYrMCd_7Y zX%1Fr^XRVZ*t+ykcImJy%Z?5EWXFbmvSY(O*$s!?w(QujPj+nBCp$LmlU*k44rRxN zeX?W2J{=t!_Q8_burpyc8}`BKY}lu#{Y7&@or)mGjaQAJghI8Gj4 z%FP_Vg*)Lb|3mq?A6z9ZUKPqZE-K2#j>zL*aX{9MkJ`x8k3oMZen*hK+DH1#YwgL+ iBL^qzf^vmZ+Lzqs3N2ro^q&}+UGn{D}4EJv_ay5oAm7=}t&2wOH-ZxA@6M(YKORusEXqhNRL z;zt)<`nNeUlV6^n znog$}OH;TjLPG(cI%lp7P&PWKUkbPt^~vmm_xhdPJx)GG#gLaOAsf|!0Zh;YZO{R( zu#Ei z93@~SX;nHSI%zGM5uMlSZi`MD_K8j!_K8j!_KD7Cb*rM2hJB)whJB)whJB**TivGU zq+y@vq+wqOoiyx&v4GV@U`c7%2P;X#KG8{Q>50gMa>T(L;v2-Zcrvhj{49RzcBU}q z#GR}AhR3%a;f!@Ed~sn#Hx82OOmp@c2N|oS%p3SWZ1mj|ioc3Dd&SkTQ~2l$_4E1p zCEe|eahf{UO`-9}ZualF_TlVzJif+hIJQImwO0H}rw@(kw_=*o&si^CJ~ne=?tcE} b^!0_=dAgh$rC3*=9&{et`b0gH___BFxAze~ diff --git a/tests/sample_data/multimodel_statistics/timeseries_daily_gregorian-full-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_daily_gregorian-full-mean.nc index 01a2eafdec11fe75234228a78f7de78e26812481..11e7691e8587fbef313bf86c1097394a9d7555c8 100644 GIT binary patch literal 25558 zcmeHP3wRVowyq%oLJ}kj@>T`}jEi6p5D+4PKM6rVA_*WOO9F`!33*M=yJr#vRutSt zT^AI^D4?>S1Q!KYk+6V9MU78j)oZ-M%0n~=?1~Y+r@HG*CIbnAkIQ~v3M&6pb=B$W zQ>W^jI#u0wr6eY|jO-XGWQd3m;UdQP$$x%S;lmkb4yXBg2LGrB$E75WZxUVAq?>U) ztfeaUZi@Nig-#}>rkNPoL|Ejni#t3`+StXl`f_wCa+uFrtzKC-k$ zL>u&wpXbQV)2NE1X3}d$$n=ohyKkR?MjMUm!dd?efMHw7xDJ+)uvZ}>mUcAnvBN)Tl$XXX#hY>*;5=8$f z(f`LVA;P2O6*E^}Sw4~*Rdj_+3sJ4^$4rhAlRKDKiu#(kRAJK?NvlbdCYE4&B14u6 z&5a?!k$u)+i0)DQv<(&%V&{&8p(Zb;gz$nM(g*=O2VPu%l}W>3GW}#;M2d`S%q!n0 z+&;ie*Tah!<<+z0MRfO{%7*CvMBQw_S^kwG#~f;)>fXqsLri&ag&n)nV0dy`N}>?= zDJfA95K^WUE>eX{#8T2MF`}t?MU7~|*z&;oPvp=zJ^!MKfE*ajje#*e-=3F6BSQ{f zDRF%7-2A+0)9H#SV-=IkJAG(^5E}pZt?|Gd%a-J{v?S3+Nxp@+mS^`j#zpPv2Vzg>Lq*D7`!4{VBhn}ZQwnOuLuVo8w~ zEa9dQ@x&S6y2+-%;5yTF?xikjlCz3-5u#l&QE6&SxN-_Gg`ipwkC2kfDJir<%vUqb z`|7^jW6kQ&^R&gWHl(W!cYQPXG)fw zkk2o96UPP!?ZD@($a;H`am|j9YtdxDm4^JRZ^TU3s8^( z7o7fGNV-=@`btU*@!VCLT8_lHF|sDx9>#A@`nZ&7B&#Uc4jr)@|+w?|5$ z3{0Qf!^)fUnU1n%0_6w9n)N%H5Xfixtg4s}(^p1ZpghyRB!mvrzceJD>2wLnXFAr9 ze5TViB!9V*=WZeSO#iZwd@kR;o_sRGjKI&!{n9fepWAtPNIthyu{FRSzOPa}fqbST zX9OAezD!5X3^H&&(=lg=+We_%y!1AU2g-9kneawnI-IYRWuQDiuPQDU@BrkDMr`Cj zBL^Be(8z&C4m5J0kpqn!Xyia62YxjUNcW)WCE=CzNOdpM{VBct(l5_Gki?wXIayTV z6yHZs3!?{0KL_s**tE>NtjwaUX?dBmv#FF^vC363dm1uxil%AVvuOh%Q?nOk%gW(& zF_LZ?A;|l98&{Nf2(3Ikr^uRKTUm$+!{^_eJU%7WSa399uZFD3yBK$?8z17OD!VEe zWnl{JF)&m0|D|eynkMO3YJSflT%*S;m2*wEF5j7-oRLe0x^7~g~MvTpll9-~wXSxS0R zsMPS#QseK`hM4@?t{B98Vl-<=w%2uoA5_}~%qK?Q_btbAozo$E+Ol$HXXnYiDpiQr zvY~a1XJ{&)K`vgiVk^?j5-kLKpT_TPHKGB>d&ar=|LAr`V3gco|DRla2dCw#JI^;L z)~F!=6wKM!XW%d=K1y;TD$j{2`A&TLuoE9Ib>f|q9^_iQc=TQm9;7(ChX*^ldhqmp zE-URj)~|Bn*?%~(DlPvz8B?46r$)x8EFUbH zKFoc>hwN28)a>`6;(!k?Thl}%l7`D*`>oYFI z9d;pop9_~==Ehx}sr?V@xR~PZI315%rQ?(BUUc8)#ln*=RG)C+%_ujvT;Rrq9o@Kx z>e&0}Xx~%E;#eJ@b=9%Dn~pV?=y>aOA97#zL8Cb31s^&->%;XMd`P*>ixVBacrwb1 zdm_DPddvgscOKZU^h9>m6ZF!&k|j`nk4-XI75*1?Hht*H-wq`p1u zM8f}i5&lmv=6vWu@p~R@f8C9moo$>=|=RAZXEsEjkIsv*m1~#WnoTSKF5iz%bZxy z)`k83TsZu=2PdEMpyEXjz9+hmz3;&ZigTaQ;9I1jd5wl12Q@4lX~&xs+x=e0{AoIN zZr4%0O~>3`Bp=m+}G;pMB^nUU&oD4=y-C4j++WSaCtrGoKN*^9#rLe&}o(jkNT<2 zE*&$v`*1}UiX>;1JABx8LdUXZKFmMi!|@s)dYtfM>=8dod^*N0&{5Ky+7#`BkH*4Y z8WYo|dNFCH7h7q(uD?jfWt((FZ6>-gIwn#4_GulqwItW0eOR30L$%L|)M6*`$cfhR zgw!9qzH{KY^?uy8(T~GJ{g^b&j~*31B)v*wY^Mud-*ut#dneYAtS#*AM1P86q94`_ zKU&W8Vbd%hI(VpkP90a=?LyKWE)1l;9rB^tX%0t&lwKHPM7jcGTD)2VFAhUoo*z4OEizW@Y3rJJXq;Kzjy37 zQfbFiyEKe`N5jVhy|`tp7gH9HT+;luXSW+|K5%1qTRYw$xoI=jPI9Q>#<_NkZEZ*T zU%goRoEJORdJwVRg8}Q@aIbY^LL~9zcTQwc8)n_%fbH*ge7?tym_vT_{kI>3?)Tw= zr9Na1(NRPFeI(tFE;suzdaoBReM9qs&5g+m-FWv<3BI6MI->+zDONpSjKLd=arF5T zgl#Ut$}3ASp?e9|CKh9CyJD<3Qi6UIpRFoE7m8coEZB?R1FV1ZOF{F;e%Wq_uj5!Bk`#p#RvC$uF?Vx zbB_%z(>2VQpyAc`G(1TB@36s%na`43babMoy%S#)JFtjirzUnB3MZZu|8E*?N6At< zcHC>nfJ_(epYB5I15Rw&@5GS;Ck~O$>hw>lyU&R$I=e9L5*G&j-i3|nE_~n9htDal zKcXY-8y(p%>F7an{^gqYlD6p#3^GAk2l{Z#nV6>rQkp*HJ-Zr^7=!di{mQu%=^?O-DP66V9eibXZ}> z=vBn0B$_)ZeoSqdP2;xxD>~l$2kFIZ9RqKrx(DrOxu0Y+f}Xdj9h)xKu(F$m?HBlP zS1YRH)$xw5qs@LVIvw=lY0^E5t)z!`IpHOJYQNio+vYj2KF*6peZ7d^>_H{Xi91J; zK1lW;sizln+j2#6yn1q7y3;m{W;!+@t@mZ_1JLj_(I(DSRuM?&BNk~ ze9WS9@e^D~xt-!n7rMJ$Nb|bT?K%xJZqYE{xD7pz*pRZ;hJ*8MxaUP3<-||XQb!iW z!rzmQobE+hTe2O51LIda@R#L;LuuSo9CXl!Qi`h&(!6}YAj z=-=5;*_&kk1{*#aYs2tqh44)*L`F?M27Hr`k|lXKd`}+M*0?a^KQ0{m7ui#LT^K@h zX4N}1uN}3+PH{#{2R?~%VD;q=Bz1S-{Wu>+UG2l>$4L*6o>>yp`@!Q`rQs}oJC{p0XzP%&`$bF!^AZjJ_3z} z;U1*9Xl&R?uhKkMP5O0enjOm~*zw38HEgzi%Z|F#=z-XIcq0i@U8)Ufk)8Ur+5J5cPk$cbgt2LqpSqVtn9hUaQnaVOas zO&m1t9r%>=eE&O~q^F(umSW+27e-NRm+Z#FNp2($@}NK2qLZ)l;+cM4;#y9j(Y7nDwQOqhIOR zSM0+szYjxp79gvl0L7aMu<7{%Y+G4?ma7V|nry&`f4i{hei!g37hdh{LUAnVx;=$> zb5|k0EG$IV{6cJsEQGUJA=dP9Vq=^W>mGN&wSx2$$?+hH{ZU8g(V706_zWkQYbD}^%W*4%uraOuvB5W z!YYNLi>j|ML1Bi%GS$zOAy}=_j4`S|8RJx%F;%4*3sjnMu}U+RsWfAyN;6igG-HgC z2gW#+W=vIS#sZaQT&&WJWh%{BsnU$qD$N+9n5xo@1uD(BSfv@uRGP6;r5USLnlVO=C&oCHW=vIS#sZaQ zT&&WJWh%{BsnU$qD$N+9#w%l-N;9UaG-H8Gr*0c5!k-wq^2J40Olv+d;oYgP{COQF z8uPm-nV9gWgI^!L=D>smdHK#B69ag#&y3f1Wll_Bl=?uHH!$$$m~vq5-C}0#<+Q>d z6b33Z{y5J$5Na^=q`d|iA_LC;wG)OMJ;g;mIJDH?R^;2oSy^ITPjhzFIH@Ynu1fN3 zRfXD5Y>zc?Ry9u6=l2^hW_X%3AzM?FXAG6;7<9@s4a3JI3DYi%?ngx^JY|B)U2dFy z72;nimky~m5ImQDns1n{^OX5z*=7;(h9zo$aiO|@Gvj=rCA@_q6qcK*@lms5>DNt` z&hqFC(=FcA*!j6(_}C;_SUyMsMShUm*1_wg$ZR*6DW5nwPKDUUFm> zmu4vBEMG4U$R8DgUM~}(?Xklz zPWf($+)r!_zYqtE{%=vIGwXCyoi3^)1a-Q%PCwP{#bB|hBWHDFY;4ABJAa`^Y9wI%&yeWontWY)#&z!Em;x8JO59 zCwkO-M}|J3LX7XpnJy@>qUS8`n&3BWrIo-BoR#UeS#3FaGqSCH2Mvhp-M4pNYh2v5 z#{Y)S3`y<6G$hj+IwEvx*%n6k&&bTpmWPqJt+Ka*JL9&Z?A!9mX5Oz!A<;eS%vrWxxNS*|hsd=C6v{jd5;)al7!NvI=9B3RDq^zEB> zEcN{Sr)i{cwj40@Kkr2ul~2as2Uapa)!o5rx<371kelr|YyB_h1}0pe{=fNrRqL|x zGBC1Q=QMLTSdGi6j<2U?Hl(bhgG$N+z6C|uHXEJpZAfX$Wf@biEgx;Zf{mH>yjf{E zo-@39&Gi{gnx^tE$hYNaIgV_)jS-~s4>lCC@|}%(U-9Pr0n_7FYO%!1HnGP|xxWHN z=_%=YJ#MG}7()XdH?_^J90`H1mHbv7H#z2{l(Kj9>^>SGYsR{Y4YJc7yr&ZlM&Kh3fktuT%*z4KnM&3R#FaozleCqdXWnG4@MuC} zY{Nb@T6pAx8Apr%P$`oKP%#&ug~Jv#R^uTIpn(Bg%aYNTKn zAg6D=U_hj@j-w?bNJ3>p4P$t@yk*yhoM9hJGE_y*kcKh5MOL9fCYQyv&am#Lj2OB`Rv2VV|wnN6M_m@GEhkUh7!(n+WbR5iAuVxW7ek$B8=K z8Z0LwSk4BY@HRR+f2D5y$z%=dm_M6k)QX$ec?H{UKgT+j6B*Amv{^^L&(1j}XQVS3 z$CFRz9Fr4L(2b4Fr1RB?X8ps+Ml`2+$~Jn&c&<~wLMPvY;^)oECJ$7$ZCYM_kv6j- z`#OY4IhsA|C)?IzGBu+-H>wf;dpU6W{b&}y`jC%@p85qtb3gi1wN*J!F{Xa&_;F<& zTUANkk8a@Ok*TuiFSQ>n?S45^9qgSy-q>AiCT+r2ltZuOO(399ySq3jvhPCk)k3yK zdEhacH6%^>YN3^q=9L&SjAT$}U<3H)3m*I0(7{cY`p3iyxjWxhEdRFE2r-d1QXg*IqS4%r*>Rpj$3-dqY$sCg zb)OW&>F32_(Xs=QYGXLRIAHXDGbx2eF#Aec(MB-kAh2g*3V>lS%gbiZ%h}urrS!L5 z?E=UY5xK){1XGUqAcYb(>1i>7DLKAXZmY^@gx`h(#@VF`x#; znN3X#DQh!C@TXh(!Bu`E)N;VI2G|e8lEKL3LO(R{#?|H(WhMo_x4Y+$qs9h%WaxO( zS$JeVlDDNQvd4dxM}~GI4Fw~QCj~F;<$mRlw%IhO8^f>0fnU!f!`&JzC#FY6$(d!# z2Y>q2o~x1YPvd~0QTR*HMwMoJa`ye=9vS*b$=P~jq@FyJN9Lw8dSuk+z|U8sAau#g zDw&OhP+iC!6THnS6)39!_Q+&s+O&ooGQp>aG5eejAv5?=5kC0!4^4P|ye>>T+dm6Q?j2&y> zVpCx|yMNxnXrXD6OC@%kt~Eu+sD)e>$=W|xCnwt*s>6$wbV8AYjV{J%Ll6;4jCWT( z^;JW-rb#q^3YDG$7!}g3%=}L7evpO)a6_$*z zy88_EfL;}316QhaM0!>~7mId}`5EQv{A&}iSG2)mj~yquxY(P;7|FR*QGONUOyIc5 z+FwrG7}Spj9Iuq_s%>swgm!aJg>wR`6p4^$=ioic8sWEvvjPfImhbKP>)@i^1qUlF zu%ylc*Ri(S}d~{!TsPh^L(y(7qdMcs<^)j0=!u zPy{Q)3^Sou^@x}A-~-i0iFQi-UbVn=gObi6jUmAn7W_x8b!fH9=46fCCBbT=4L0GM z?*FphHQF3qyd83r*ij#x!Z(Rcj^n8|?;33))}3OSoiI%TBx{Q429wR<)iH7r+Y)TA z4mO)?MH-uzD#4CokWO9W#(0{!#|RU06ZG5vPzOnF@tW&^t(nh8q8xDu^~yh8S=bV; z!J2UK(y&O>vyJM=8eUawP%|BStZnb@0qDQ#(=1qD*=~O9q}R zr1!S*WUb&P=r&P*)&gx6`QmOT&PX=QDNKbgB?o*{VN1Oc&?Gyzlox>0S>bz+w|c^T zF5!}LnRX5f)NbZMB^+r*S2%#HCWJ_ZQbn^V9E8uM1ePzP)29o4v~G>84u0z zi*dKPO_w({LXHb0_pEV?fbN+< ziEHPwlYjk?oEH^$ELjhdy(hIG4pk8=#*f9Ctn%lh>0|q9>SYSNt#YZ@7=q*SHb?9H zFV4{vYVR6VD)il(OHv4~ww(CK%jOB(`w=EhSRt3{z57B@WseRWH*U6y-y6C{s*)QxL#@@MCzX>s*ho67jU?A+ o|I^Di2g66=S61reH0?#CTI&6y6BGlkwpKyBv4A?iD-sm{1)fHlc>n+a diff --git a/tests/sample_data/multimodel_statistics/timeseries_daily_gregorian-overlap-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_daily_gregorian-overlap-mean.nc index 5e2c46e578b0ff942c887a3acd74a4191e6358c2..11e7691e8587fbef313bf86c1097394a9d7555c8 100644 GIT binary patch delta 986 zcmZ9LziSg=7{}iWCnPCJ&JIZ~X%ht#n_SHg5v4?vLu?UjV<9?KVi6q#O)^ArF%-IG zNJAe=C*$CtgT#Z|k1VwaHNC0e68``(Ex0)Zzi)kS=?foTKDp;}@Q~zaUpm^C4A~`3 z(K*+Se6XkAG<&WeL8(n|)XTIPDbYyvOeMbBn@g3ucNlBa+e(6T*-IBAVR`R&;vKDD zkC|(cXh0fJT?Lt#o}JB=>lbb&NFVO@&D3}H59)98dLQPO!Bb;A2R6xOB3JQQYAorl7d z8QS3Cj1zA0P#9L^p)jn-Lt)A+4R|=~goiv7h81}z3@h?bm=fF8(8ZiCprI3n6=8-j ztf-+Aro>6tf@&@fQU{*eo$XWi4qbi0p5TV};w?rsS;~yfFX7H{ErBPrdf-Yco^$~Q^G;W2hNi#SeK_`uxSQTND8IFih zn#SWIlxFaZ2&GXoYa(nm!(|al!xlv-4OS~q6np7iz1Xp!Idbw zpxM<&(MiJ=VQbQ`#V9&y)XJ&|2{GzyiXkY$WG;A8h>{OS%}=Y zbLUKZ-~JHhn6oy2zns&H$yKY{rqCWH5C}Qyd@gmkOnE*!$S-ar2`ds^ ze;qnPL!3JI;y8T}_xovo(Rk0#FS&t4Q9tUADBRvR*ChiQlC=|)>F>4w>E9F?QqFgiXno4J?C iil@*qiI^jf-s z>Q(jY>bZUNh?Mi1v}+<{YTQ^vidf^2|9r1fmrOH7)T@5~)~=?8#?BKKOB-3$Vlht+HRTA#h!(O|^SEIoP^Ki2*i$9wgzW2LMH4H{TN<%tYgDm*uq z3`h1^oymGp)!o)vP>3C`Ck>V=t+@@>7v$9}B1UMUjhHYpW#ojB<1unZ>X?a>glN@3 z7pMib(xy2No2^A?cK%*yp&E~eJ@TC0)J0<0=r($u>M;J4(fxqZ1^e$R zU1W_hn{!C%AX)i=T6D2gmaa(`4NWm2e@zJ?8PyI~jH<>e#+q1lpEaGYJHkK}cIcYFL%vUzLS2C2id!lNe;jWeCETRR?No{C`GbXM? zc8kW1b^7M?T8DQk*OvoG_GQQRj?PhF_BREA7AWzr zT%|I&ufp?{T?fl=PzAZQ;pN$oIiJhR8D1vNk5K{|nc2a7ZV#7ZWI61mIiKa2tE?*d zH>!rHMM6+M%dv)Cm*wy(isiGMW?|*;QF@CB%V+uQDp(H7motS-oX_%&#Z5>)%a^My znYcXXw=lDV`J2_UqGebzb@uzW7xt(JURgc*tN zm;0rASU$J&;;?*fr&4QBKYU#^zXbDHj`RpJ@pV~_^b9g_KFcvZL{)x)8ZW)g;=%Hq zPfK_su^i4)hzZsP&S8TDANfqD(pYoJ~O^%|(xK)nX)HBhgCdJX(y8jxXL zGfKiw)*Lx9gGDmRFXQqY0~wK%o0CPY`9bU>q=nW2GS0#82ke?HFUwY#H6zcKn@y$U z6RUhG=17Arr*MXrol7qeY?`AmTUM6Q21eyN{!n$6pzf&$VfQP!%Ey%n ztIWVIUzzWmkxPaLcA)~O{EW=_=85a~*L{y1LgpBEu!X^vn%CrL3Adm&tmPiTVm&dMxl--b1L37Ah{Sqg)ITZg zXk0f+h8(5sQN(_-DWxi||LLYae)GB){HW{3kpQz`ta=#d7zg9hW-C9h<26dgPl*C@ z9F9@8fTMu&89bDi&)#|Z6|!~LwC(&t;@y4>UgF0qLw!h0@?q8v4;Jq7z)vT`_X)mAJI*UP}-~ruRW-v$-_GK_0n-U$^A0m#q4{%Nc)`^ zOESGE8Scg5VO|XCSA>m&iqL6?j<0s=NX*iaKT}8hd^hfx>&BWDZj5`-jfXczeIseh zpNbK?z8G3mG45zsjJ-39(9u?e+sfRqZ+4^2C2j?qMKD@ga%U&+R^P7sW?6{8GiSBbe z`IV$i&$vhZC*8 zabW3b2X6HF@wLZ~=im3?jeS1!O7@`P*feZ~En}$}6JlNULgUs_h_}1#d!xwmv-ok^x01Yb!YdG3~ z$`JY9v}47qb{sxU=f3fv?=lbGyWfKumv~_7QT52M)(L@Lh8UR^9o;X-SnweR%C}K76*xhYe*u zj9uiz>brbM_{@#J6Ft4dg?2ls@76hS0nv5W>v(>;j$4Q6I5|{Dppy=3oR0TaxskEP zjhs1dbkB981NEIX(T#mO{it}$j|>{y#}@n1=xRSkP4GiMn#(}&NAT8;6+O7zYqH^l31^q=HLk89j0IqkxQL^%)mvA4vJhaM+;{*xaEwvhc0 zwS35n_{Y2`denVtmDC#bZmP>hj^6y!y+9UsLrkHohV-6#N1M{iJeY#e9npE-#M|d zP{$Idj_81n)4$iTd$EpHL{n^Ttjlm?;WIAu`m+lQqg^=G$c3?y#julKSol;CP7qDa zD#Dq}B8+*=gWT0*r>EQ~{)ZdK$VL|pb7R#@#pwHDFhuyd{p zZ_mxJ1}&n6ZhMkcpolwAL_!@i~LBYemw?1&Ku#! zfbV?R|BVmjF*Ht^J5l&p%#ZgikqZzo!ipX-?J$K4ajM!Wbi zt*akPPx;XLqz{O8;>6c9Zf|nHQsTgaf6)-z(}8HR-?fv-zh(H5Yx84Kt{*GOUp=|q zfv}yZ+8d1z%AC?pCxtaXed><~I=tSp9PQ;vYVA2;3bXx2{ z-$f2=EB2$i?#I4%)Rwkn&l`N0@`MkA54v!eXk2Tu8KU@pPIOPC`d8{GC3=Qz{zamF z!~Jj(og}~Y)im-mzjHyGMz(f?18e3uP<+0Q1sCYZwbML6wEJxzcGBFqcqZ8wjopDq zU0D9H3sZHn52p)j$uArtzp(WZ9}dL(aP!?>d`;9lo%}8N+S>;?;UPNNi+o+Y16!8+ zu=IW(<~8-<#%LP9H0JjDy~xlUNOCwZWV?oLq?d>bH0+GgFmaR@-A8ya_f-#4$~<^x zkO%XK*0t2|c^e{!9l~YDm;;6AOSEmL8(Vj{(KE-5Wz^nTZQbZg)agq*zOJxi?)7%O zc8eWT{!s{Rf*s4Zy0MM?#@dhFc#i0?SPu@wc+m4=J8ti3$BwJ)crw+FlsR?`B#LU~ z#nP5u^0Qui+sliqscq|t=KNJdGomZs(qP}J;lyeUJ(g=YL+gXB)4Vu+lzcK#io=6^ zqBiZF5UreeXsi<-jdY?s--)MhaAJ|yhuJRjx!-!Bf8&MAF7PN+#m*PVP_3571G#ojoA(6)V zm!~u&pClW#`tT*$a_0HeXU%=kTGH4h+TE6HxPykP$xg=*wGtZEpRr>at)<%CPUUIt zN+F%M7(nw`e_GRis9_+H^AhqCJOrQr#!g$OAprM(VB*A;PxXLz9q_~F*UB4h6j@E=s3)d z<|OOb3NN}gr@1WFi{`hJ{V$+3NS+-VH9K~8wPVmlcC6o7h~94%;;ETlESOF6-zHk~ zZ1&*ub{<^Qn(SKcO1f7)v{>jNhs=mS`g&7J13QH80DXdT^ z+Nt^qlN4qs3@9v7Sf;Q-p=ht_D@;6(%XnP#92HqOeS1g+kFu)mNCLFhgOf>gVz>tW-Q> ztm;q31jRF^E1t1H@r;WV&seH>#&X3oRw|w`R@nn%g5nv|70+0pc*ezwXDn4bW4YoP zD;3WetL%v}LGg^~if1fPJmX@;GnOizv0U+tm5OJKRrbi3pm@e~#WNNto^i3_8A}z< zSgv@+O2sqADtl&3P&{M0;u#AR&$w9ejHQZaELS{ZrQ#W5)p%e`P&{M0;u#AR&$w9e zjHQZaELS{ZrQ#W5)p%k|P&{M0;u#AR&$w9ejHQZaELS{ZrQ#W5)p%u0P&{M0;u#AR zFXsNs@aKWDFOpWjys+ntMpKgBxo+FiCo#py7lA2BcOU=!WdBblC&|;dUNj|$$NOZy zxz9EwiBZlAvb=%8KVPUo=8?^2))D3vey1>4q5j}Es(~MbsOepUOihBp{`HfuJQ=>M z=sqyK)Sp-7JH$Dy#5&ioch%UbD%VL;Jbx&_Zz*^m6|chT*sGc#>+|;wNKKw7H>1lJ zC2}RFmX0C2OeZ2ab)+!g`B*QfB9tCCLFEoMcE1X7ROQkp)jHBGgFc&PEL|^BdqFL< zTYreL618t?{;63r%Gh6MiHuf)BIemN-fEV7q{}=RCbwpo0r7^$o1gK?<3`HD@Xg$~m}8qcFS}11r_01od#xqko6In%1Q zMr*p1_*1ts^Bpl7t)oA?B8D82eHLZx)3ikLHr;5AKI51%GcU{jj500mdEVz2L9g_9 zb6{f0>mUA_FG!>>%%zeeT>@tFxies&~Zf|}NB zbLZIQ32p&f^R-qoB>8nxBg5!t<~vArW-;jjxESfoQ0lkw6Xq!SQ|F|`ylsp-evxSj zSVX|+s3%{vw5a<9A-`JV?fk#Y3&IiORbxf&p)bA-_r4>>nrEvX*V2mTzaaDYw%6{R ze$VhpsVStJw(`bkQ>t_zsgK9SijgwoO7k`CORG=EiHC_7hSG(&b>r{>!f-i;a};9h zvHJ&$6H+e8kec1@zI1{pqXW5jUhZwZ?C2_wFdh~!G8-`C&fni5ZjdDs%}*7wtwZk> zPe~br1@#xDoE1-rPvpUNL7$h=w@+W3c=_YMTzt38s!u-D^LB5d!(leNj%YW z7I#hPm$q`1z&D&_^Vs9;IeD4caec4om(aU!@4j&f34@IPb?q6FOE8upyQ{7(LO(1U zZOpHkwt3ldGZME|PRBeY%e;cZ{CU|0T2AJS*@fA&^J(>%$3^S$|3?FA{@*3Lo+fiS z?;7*}*{AylWEGmbrM(){eF`0jI%jvctkT6?5E!$2nXE!<5@}F!b{F5t5^4=`Q|IPi zz1(lZbFxf*`h_&`pU(fP^F+0-4K)oBYFgF$w@+%1!@tm7)ri3Bc#mZFNo7 z){v5NgKt4$w%ty%~3wMX3Ms@LKwkP!UwfL|-(CdZt#bdHXm+m8nD zqMcEWUXFfslllzdX=U^iqhV6|-z>4;8!;!jLaTA(J#VY#Ee^rXp=eoMDm)~eLyLkn%@)Z|0lLg!QLjE%Hm`rcRWF)en5e%CvON0i)o|LEPi|}AD?3gUEy($=X=qbNyv^-Ites&Gix;j?< zCPMp6gj$6N?QapS$bfRkiUr3N_?|*;T9q9w}bd@@Oj6?J4V?w z@296k2_3c;qnCKjh-5l!DHi_m6DhSm{kAk<^naAJ1|yk$C0DaXGVLJv?!q(xW8o@K zn>{bRs*y_QZ|kZD5z|D19|W6r#2XrvvB`*skxa|cAN9nkjC%NWG*HJ*662BNw;G8H z_tG4RdYw%3N-dd9Oo&GKFHQs{Wo`5TSZ z?K`8ZuGVI3Tz1Wy$SjdO9z!=6St8jTG|+6US4$*2g-H1xk$)l063NaXQtn^kA1bSB zZLr!-* mrE_#6+h*753T{Y0&voPv0G&0u5n3UYD;V}t|L4D11OEnl3`4KR+bI4Ulhsl%uX z+aCjt#%MMqYW6VGC3eYXScGM2Gz*ToX!gg%@M>qBU^ljgg<=bU?< zzkBYftHSURVbCL*8;0_0wKAeKG@4PP7m+|VMkD#|SU|oA<*ci(Y zTxkpi!=m9MsPHVaFxN+U=mDAzObJeL$_YartDqYiwL_?=L@nb01n6F#tCnjKQo~eR zS{aFnp1^BI`)+r9d+dcyx39d!Tj2HKpWEy8)87&=BY81G8UzzTClm&Z8rg@C7D0Ew z2@UP?vTePc+j@Gpx5w;9zVnkDH0dOELDG-~le8n+x$7qJis^d(`Hxv91JEr+#6O%c zA=%72Uhnh*rWPoGF^?PF?^niyr9J_|$Qi!Ax!QOzaY#2M zfGb5OSRj^?4t;u$cryo1>aCP%qs&`+Biu>RnK_b?Qf#dFkG{^Xv@R2SA*6_VRnVGh zg}wNsn+oQL-b%IFd-*NMP7+6bFpf_WpB%z7MzPO_sa6r&*2+s}T$2O|&2rr+TkT#o zW9PGFM(kC?e%YF@h#qmB4jH^yQ`Z+L2MCs(1g`GJcB&O6r>=nX7BhrR4avO8JHFi^{LjI!*Nqx1lvNl zVX>-k|2f_Tjv%D03-$TCD4wk(EJ4ze1(OQnS*dehD`6|E zQ%VwNKtmUs=Cg4&z=KvGL2b9iyGp^B?S;Dwz4HiXBK=+!Sw|g-s=^{M z;Wo6XN!PNJvBNQeq@CMe>Kx{6{Ho{7Gg!+3qc%T$Uy`F~mOsUl33^(N!@i&m#w&Dk z$CUIr8w3|b2zbh2SG`xfO9raVu{neIMCcx0O*~d)7Qtj#3O(YY# z$4W<}u2^mSbnXg?gzMDsKI)la+{d1n0$4~9iub>L*$>GeGgmY1r|{W(Y|JuqFOt>u zrOE0gBe$%{nADdiYn}A_9?z%DHHXaa)`f!ANGSm2UMYDLgSAE8M6_l?C2YiFGZ*gr zvS=Sw=n{V=A272h%}L)mm6|_FU|c$iQF7qs*(Z=&|C*?C!Ta^!nzFF^>>-y8L!p5F z%$`#QirX}3-JFr9p?DIcRBl^`t462!bS2{)UU9}n^ubVMQ_I&_k3VQohCPP^Qq-0G zqrSoe$#pT~J4xtiu@}A3um=WocGfLoT=)J`C=XFm&rIx^t5&w$kihY`J7na#UR2c(wluzQLJ#{PIwmSUGc(&2@@a zcg#8lIjYjU3;%PIe;f{-t8QLXhlig`MvD*DHxwkE-SVZ%q`a8aR&e9cJ|zVhS!YNL Qw3Z9vbR|vW=eB_OFP-nELjV8( diff --git a/tests/sample_data/multimodel_statistics/timeseries_daily_proleptic_gregorian-overlap-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_daily_proleptic_gregorian-overlap-mean.nc index 29c55592c3b3da6bf3c3f700de8a0442972cac1a..f61371381b94d29ca48672770a2264987d8efe34 100644 GIT binary patch delta 934 zcmZA0Plys>9LDi^huKg=oud4^t3r(ARh%csMKWtlz!0rYM$B=+pJnRMcX0CK1y< z{xrXkxstLUCX!(>q^1$2w&9~5^ud7`;52uuE)sAM*PLtzvDIPWTXB~UR`>PHQ?paZZ;a2@zUcolwsTp$1>#A zrEfCi<QWkOz{{mV4jnH&Nfj9;y>LT@yt?#AhP+&A%8-}7 zNR|v2y|5!gUbvzRdEtsOxpDo}$uG{J;6bfE`* z@J^$L6y%@)MJPiBYEXwJn9znU^q>#k6naQO4hm3&GE|@jb!dVKZRkP|`ru8YhZN+} zPSpy0&ojIZ{`<*-q{scPiQwPIdD@HzT(`v1X&b8{mPNbqz|etMEJPL$Lw4BVp;$WY z^H3~{e(+E%rfwzBsdnc}pc6}{C{{x(i_!^nVzDsgc^J0CB_4{U(;5%Ovgk1n#bRTm z^Ki@#H+d))uE;~Na77-9#m33v;kX@kcqkUG$V0JkMIMU9B3l}|h~4=#bYkI(SPijo zMGc);EJ}GEM(uEkhhpi*Li&1^?zQCNt*2Z|#^Tq1%holo=Q? PCOk8Tf6>quejon>+x7*S diff --git a/tests/sample_data/multimodel_statistics/timeseries_monthly-full-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_monthly-full-mean.nc index bd41893e62a19c9a782ec375ea16811e14674a00..dde68ce93237217dd678924caadadbaf58e75e77 100644 GIT binary patch delta 3874 zcmaJ^3s98T6+ZXhe_>z9g0L(s%A&Zgt0KrlMP7>pNJK!;M6n$;N(ZgNMu-h|bkU+Q zsi@0|9Ze0zs;Sd9qh=@7X*;R5wzW;;Xmp)SrZ#ly*rvvWYMSwt*!x6Tlitxi_nv$1 zdHv^p_t5tWx%DR5?bSLKhkQ?D#eo=pm!KCVEtxFV77a)!;z5$$fXAlfj{;pDfCF20 zO~kom9s%4(W?{gXhMMLiR$Ssq#YB%ogJpQhn2+c682p*XXWbI0uB&RQsRRh&0oOEb zs}I}L9PyK`pYN%!sch8ZkPs&z$#-FTSsK%qv&=1w*`>|obZ*{YhF#K)OrzMU<`nH7 zrxR{5mZoXMB#!~~SKI9&|5*H~XL6{)NIu5Xwm$t&>n=^hZ2wrxWg-N3PQcBn-)QNb zrOs`6c@I=9pe~z014y9kmEP7ov1F<)U#ijW{WuUqvO<0 zx^aim>3C2o4o#dWXsC=)5!5Se`c*+C8SOMZddNSTTn_nN81WQkUD-G(qpGRt37Fzz z#{kF~0>#uhfPr89bp{ydB>Zm7CXz6TZPNc+Wv+afTRuJp;MF(c_lLKd&J(<1HYFxe z2{u$hi3i&)**L~*!owCjZZw;S6VI7bFk-QrhqxVpG#o`aMr-*nXDqf+PK(vfmAbjg zhf0Su++}r`TW1LVI4=Bqt36F|vZ4;O&cIKt4vjh!Ys*ekX1$L~)!Q80qD;nc7}R3W z<_IeNQ=(3;m9;hXYxvn;!=Dz#CmzX=WV?05I(*LdoD~CEI=LE6y%P?kAJT$Q z_OEir8>|rYRF$CRPYLR-ll$}K{z5^!7Yn-Vn}U`v7Zh3q9c&eJV2z;N0YU90f)15) z8U#6k^Oa&l(e4_tRy3zptQCa@u~zhalRPi!z%yd4Xkdw0E84J>t!n|qgywpM5KwgJ zTVk!~snuew=rF5vG8YWwfgC1W7wf|Utst9$BS4)*)FW&` zhv4?siPbeh0GuIzdfl&suXS=KSZ%XGXtCd5li<42CDGw^%x61~h8N0m?MtekQ#04L zjGo2gQAjlYFheovj6?KTM(8kiu3Fo|Ki%gt?idH-6I~&XVNH9Rg14LhIG#x% zz=fZd%}-!CW?rfHAPM(7OMLb9|9&$XOCo-z^3lLAgo<d zi<@xhHSSLg!;*L^MzLhBkbYQa-nsWY@VCR6$14`pR#DJTWJW2-vpTFj;xLiU4Vv1b6$&Yvu6~QZPctqh?3=I>OI;JrBC6)=uK6s;R7i z#i`7y9Dkm%z7X#hRbr2pfED>Uv@_d0dpucy7pjtyS=2I-FkBxvn13YkBJ=o>@%co2 zv3Oi217LMv6)o=EnFTphb8_iF{Lq;m&+1S~QMLG=;xYCpUJE)6xzy(n z%Y-q!$9vxp%NxE_4L2AXwSXJX8Jv!b66;&!-H-7xpIG-}fG!g9k6}%vgO!F3sx;JV z)u$)LN9~0C$?QO36r1T}^G0r&*RrjJ(=Wp1WNcd>v>A=J$W+wprue3MWZs!b^PFZ z`knCoqL+x)TM~Y;xP^ie-Y9)i29YP*f*vl*gE;HP*0OEZNOS)Z0pX)5gii>%se~wm zI0-uP`hub}C-u$bSJAk4Q>_r7;Esd<@Xvl;L+wdSwxNAYE4wE90~A-8lLyyxn-X%T3*C zC%oeuRg}VD*U4JRM=OH$dTkqLdT~+tZ^uNcxsKtV4RIOq{C# zJivCWfi6$0(m=vu7_9Ip4N-)YFtu$;P|8HNMP&QRt?uO7#H!WqKy9^ux<5cZZofZ7 zKjnTl3}pzZ5PTeU!n%M-We6aoMsOw&2n?P3`Tf?W*c?@kR2~*pk|0Cu>R7$`rj&?` zuYE(Gw8kQF!FT{eS1?}I>tY5w6KIN)Zcm{@8V+X8bY`lb4iAI*_!ls%tfJC{a9^WS zFciTC!3&+LkkT*}AqCt}6e71crsXds+LPC2FwG4^ z_gz-kQjG1eA?$=QuLJxFN7)@-Aw75EJzimXBU^l*U?PnVt7(N7NriTJUik0oLcMqX ze!F?Wf+#x9H;6A`^o$?{@E-1eL`@?+GX4sqdTLrUw!S-cRkZ_$Jj78tvL21J1fdf0 z&770XPna{>v5}LzFw5K#fMK#y%E_6V4oY(VuD1y! zw%SmhPAzRv8X2iu4K{~g$<#h3(q_LBb{cFx+04%>ZSEip&K%eEDYJ2t9!z2!dhCC~ zpr{do4%nSr0J+8C7dto`5wh2!t+N)!A|w(yAv1~5?(OBZX_3sdrtQP;QZKFKWn^Bw z#Br}T`;%sPr?_V4Acvh(1Ie;P=i5w)Y?ErmqFNGUCxh*2NjrWQ(A>-^q;-`DZh3xdCq>9_Gi(jzba$d;jEd_#2>Mz1B z7tB@I^Ipi6d=dJUOEZT9Sp?F`Dxp)Dmfhy0bYM1qCO-VjwvYTdByM?HHJo~-5{oXW z5p%BSSYX1H&V+mmK#Qw(p12f(YzA9qG4FA1Za;F-n!aq=?z)<6_v(|kY}tg@)n?*?*&V;kGN{Bd{1o@~4< z5f*d&5T?B`dmG#c6x1q+5BymxD+2$d~)5O-7xOBw*z?RS{-d!JBPVxP=)Z_(P(wl>%{(;;TC?n0H z%U}I-bAikfrg{eZ7Y-n;51b(M0XGQk;02)-{GjrIQ29^@-4F?(6JjB>Ln4G`NQKZ3 zG9dJV90=V|0HFnJA@m0aFkR08WiVWDfk?pU1`h}yMjQA*_%NCw0K$jS7lNVsq4a`q z2p>ilL__#6Iv^gz_hDdwFc=sVk|7FU^oKMEA4VU@g79JVgggizMi&%qj#Dt@;sr)+ zJ}`zD7`8gU)dmLqd2t~SSBHUt6G-{2P!M3S0|^*Tp02IS_-yh~Z4Y3q@aagiEa@_- z-z>=M!#FuZH_FJAfq@;UjNut0kO1oD0CA9wf*J#geg=l$E&m%fCpuL!PR`ZmWjSSD zW;VHB&0zC7{dz`5(aCy-(yRuCx(0@mH@eD9E;NjoJUg6+GrcG^J-;Y3F%M|zM8?e- H#@uQEFUYar delta 776 zcmYk)KTH!*90%~05$_b&Ij@B4eXcUyI~RcFf@ zV}0mhIrZMyT@RjMFPR_rC=UbQ&4cyfqBRI$F~CN4AO5w1__)K1AC<7WIG^cvHk3PU z%)(odF6P7cEnT>+w&CZ9x99cLu}1@E_*@MXfK~vgblV1p7ay-wn%`c%#n$$4@PE_F z`l5-oo~Se0{ztsmNGz%vYwMq!+1L}M8anhNFQ#tf$l3sTdYIIYyZtB0)e&;(EIE6D z%wHx$6J(=8F5BeNHB#><;qZ=Ca37$Nrw-}(AZ2OZqAZ01WohG-JKwcdq%2)HPgy#C zN#p_mJpkxs8X$#9%F@*-%F^02W$D6##*PYFx$E82 zMA*HOy9$6;^z7dBXS(#;VTn1WKG>}M!uy>`=UH!+u^1*&QGY6)(v!*T&V|p1VmO@| z>kwOsjYLWJWy+O_IR<`S!%#kI-<-KxyfTla+ Date: Thu, 14 Jan 2021 09:56:32 +0100 Subject: [PATCH 34/34] Typo Co-authored-by: Stef Smeets --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index c899b07627..750768195a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -27,7 +27,7 @@ def _plev_fix(dataset, pl_idx): """Extract valid plev data. - this function takes care of situations in which certain plevs are + This function takes care of situations in which certain plevs are completely masked due to unavailable interpolation boundaries. """ if np.ma.is_masked(dataset):