From e3c25d7a15effe2f39c9c46677e2885eebe547d6 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Jun 2020 13:50:24 +0100 Subject: [PATCH 01/12] regrid time before applying mm --- esmvalcore/preprocessor/_multimodel.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9ea3798caf..cc35f4e425 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -20,6 +20,8 @@ import iris import numpy as np +from ._time import regrid_time + logger = logging.getLogger(__name__) @@ -164,6 +166,7 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): def _datetime_to_int_days(cube): """Return list of int(days) converted from cube datetime cells.""" + cube = _align_yearly_axes(cube) time_cells = [cell.point for cell in cube.coord('time').cells()] time_unit = cube.coord('time').units.name time_offset = _get_time_offset(time_unit) @@ -183,6 +186,21 @@ def _datetime_to_int_days(cube): return days +def _align_yearly_axes(cube): + """Perform a time-regridding operation to align time axes for yr data.""" + time_cells = [cell.point for cell in cube.coord('time').cells()] + years = [date_obj.year for date_obj in time_cells] + # be extra sure that the first point is not in the previous year + if len(years) > 2: + delta = int(years[2]) - int(years[1]) + else: + delta = int(years[1]) - int(years[0]) + if delta == 1: + return regrid_time(cube, 'yr') + else: + return cube + + def _get_overlap(cubes): """ Get discrete time overlaps. From a8d1f2a0e1eb4070dc866369a81ddc1257611217 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Jun 2020 13:50:39 +0100 Subject: [PATCH 02/12] added test cases --- .../_multimodel/test_multimodel.py | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index d8d440d51a..248305f5ba 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -48,7 +48,22 @@ def setUp(self): units=Unit( 'days since 1950-01-01', calendar='gregorian')) - + yr_time = iris.coords.DimCoord([15, 410], + standard_name='time', + bounds=[[1., 30.], [395., 425.]], + units=Unit( + 'days since 1950-01-01', + calendar='gregorian')) + yr_time2 = iris.coords.DimCoord([1., 367., 733., 1099.], + standard_name='time', + bounds=[ + [0.5, 1.5], + [366, 368], + [732, 734], + [1098, 1100], ], + units=Unit( + 'days since 1950-01-01', + calendar='gregorian')) zcoord = iris.coords.DimCoord([0.5, 5., 50.], standard_name='air_pressure', long_name='air_pressure', @@ -75,6 +90,14 @@ def setUp(self): coords_spec5 = [(time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] self.cube2 = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5) + coords_spec4_yr = [(yr_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube1_yr = iris.cube.Cube(data2, + dim_coords_and_dims=coords_spec4_yr) + + coords_spec5_yr = [(yr_time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube2_yr = iris.cube.Cube(data3, + dim_coords_and_dims=coords_spec5_yr) + def test_get_time_offset(self): """Test time unit.""" result = _get_time_offset("days since 1950-01-01") @@ -92,16 +115,21 @@ def test_compute_statistic(self): self.assert_array_equal(stat_median, expected_median) def test_compute_full_statistic_cube(self): - data = [self.cube1, self.cube2] - stats = multi_model_statistics(data, 'full', ['mean']) - expected_full_mean = np.ma.ones((2, 3, 2, 2)) - expected_full_mean.mask = np.zeros((2, 3, 2, 2)) - expected_full_mean.mask[1] = True - self.assert_array_equal(stats['mean'].data, expected_full_mean) + datas = [[self.cube1_yr, self.cube2_yr]] + for data in datas: + stats = multi_model_statistics(data, 'full', ['mean']) + expected_full_mean = np.ma.ones((2, 3, 2, 2)) + expected_full_mean.mask = np.zeros((2, 3, 2, 2)) + expected_full_mean.mask[1] = True + print(stats['mean'].data) + self.assert_array_equal(stats['mean'].data, expected_full_mean) def test_compute_overlap_statistic_cube(self): - data = [self.cube1, self.cube1] + data = [self.cube1, self.cube2] + # [self.cube1_yr, self.cube2_yr]] + # for data in datas: stats = multi_model_statistics(data, 'overlap', ['mean']) + print(stats) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) From 19a4789255c721573422ec5786c905964b73202c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Jun 2020 14:21:00 +0100 Subject: [PATCH 03/12] fixed test --- .../_multimodel/test_multimodel.py | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 248305f5ba..eb8725d413 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -114,22 +114,31 @@ def test_compute_statistic(self): self.assert_array_equal(stat_mean, expected_mean) self.assert_array_equal(stat_median, expected_median) - def test_compute_full_statistic_cube(self): - datas = [[self.cube1_yr, self.cube2_yr]] - for data in datas: - stats = multi_model_statistics(data, 'full', ['mean']) - expected_full_mean = np.ma.ones((2, 3, 2, 2)) - expected_full_mean.mask = np.zeros((2, 3, 2, 2)) - expected_full_mean.mask[1] = True - print(stats['mean'].data) - self.assert_array_equal(stats['mean'].data, expected_full_mean) - - def test_compute_overlap_statistic_cube(self): + def test_compute_full_statistic_mon_cube(self): data = [self.cube1, self.cube2] - # [self.cube1_yr, self.cube2_yr]] - # for data in datas: + stats = multi_model_statistics(data, 'full', ['mean']) + expected_full_mean = np.ma.ones((2, 3, 2, 2)) + expected_full_mean.mask = np.zeros((2, 3, 2, 2)) + expected_full_mean.mask[1] = True + self.assert_array_equal(stats['mean'].data, expected_full_mean) + + def test_compute_full_statistic_yr_cube(self): + data = [self.cube1_yr, self.cube2_yr] + stats = multi_model_statistics(data, 'full', ['mean']) + expected_full_mean = np.ma.ones((4, 3, 2, 2)) + expected_full_mean.mask = np.zeros((4, 3, 2, 2)) + expected_full_mean.mask[2:4] = True + self.assert_array_equal(stats['mean'].data, expected_full_mean) + + def test_compute_overlap_statistic_mon_cube(self): + data = [self.cube1, self.cube1] + stats = multi_model_statistics(data, 'overlap', ['mean']) + expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) + self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) + + def test_compute_overlap_statistic_yr_cube(self): + data = [self.cube1_yr, self.cube1_yr] stats = multi_model_statistics(data, 'overlap', ['mean']) - print(stats) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) From 358f3f70ad50a7cd97cd200d1ed24d7ef08755e2 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Jun 2020 14:30:16 +0100 Subject: [PATCH 04/12] fixed codacy --- .../_multimodel/test_multimodel.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index eb8725d413..4aa1075c5e 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -55,15 +55,15 @@ def setUp(self): 'days since 1950-01-01', calendar='gregorian')) yr_time2 = iris.coords.DimCoord([1., 367., 733., 1099.], - standard_name='time', - bounds=[ - [0.5, 1.5], - [366, 368], - [732, 734], - [1098, 1100], ], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + standard_name='time', + bounds=[ + [0.5, 1.5], + [366, 368], + [732, 734], + [1098, 1100], ], + units=Unit( + 'days since 1950-01-01', + calendar='gregorian')) zcoord = iris.coords.DimCoord([0.5, 5., 50.], standard_name='air_pressure', long_name='air_pressure', From 298fcef94fae32df855d336f359ee0fc01d06dba Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jun 2020 12:56:52 +0100 Subject: [PATCH 05/12] adjusted for full span with no days at the middle of the month --- esmvalcore/preprocessor/_multimodel.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index cc35f4e425..0f77b56498 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -164,12 +164,10 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube -def _datetime_to_int_days(cube): +def _datetime_to_int_days(cube, overlap=False): """Return list of int(days) converted from cube datetime cells.""" cube = _align_yearly_axes(cube) time_cells = [cell.point for cell in cube.coord('time').cells()] - time_unit = cube.coord('time').units.name - time_offset = _get_time_offset(time_unit) # extract date info real_dates = [] @@ -179,10 +177,18 @@ def _datetime_to_int_days(cube): # wrong overlap indices # NOTE: this workaround is good only # for monthly data - real_date = datetime(date_obj.year, date_obj.month, 1, 0, 0, 0) + if overlap: + real_date = datetime(date_obj.year, date_obj.month, 1, 0, 0, 0) + else: + real_date = datetime(date_obj.year, + date_obj.month, 15, 0, 0, 0) real_dates.append(real_date) + # get the number of days starting from the reference unit + time_unit = cube.coord('time').units.name + time_offset = _get_time_offset(time_unit) days = [(date_obj - time_offset).days for date_obj in real_dates] + return days @@ -214,7 +220,7 @@ def _get_overlap(cubes): """ all_times = [] for cube in cubes: - span = _datetime_to_int_days(cube) + span = _datetime_to_int_days(cube, overlap=True) start, stop = span[0], span[-1] all_times.append([start, stop]) bounds = [range(b[0], b[-1] + 1) for b in all_times] @@ -232,7 +238,7 @@ def _slice_cube(cube, t_1, t_2): of common time-data elements. """ time_pts = [t for t in cube.coord('time').points] - converted_t = _datetime_to_int_days(cube) + converted_t = _datetime_to_int_days(cube, overlap=True) idxs = sorted([ time_pts.index(ii) for ii, jj in zip(time_pts, converted_t) if t_1 <= jj <= t_2 From b11184eede51ef788af1f74982aaae8a693a80f8 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jun 2020 12:57:05 +0100 Subject: [PATCH 06/12] added test --- tests/unit/preprocessor/_multimodel/test_multimodel.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 4aa1075c5e..6443c16393 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -171,12 +171,18 @@ def test_put_in_cube(self): stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) self.assert_array_equal(stat_cube.data, self.cube1.data) - def test_datetime_to_int_days(self): + def test_datetime_to_int_days_overlap(self): """Test _datetime_to_int_days.""" - computed_dats = _datetime_to_int_days(self.cube1) + computed_dats = _datetime_to_int_days(self.cube1, overlap=True) expected_dats = [0, 31] self.assert_array_equal(computed_dats, expected_dats) + def test_datetime_to_int_days_no_overlap(self): + """Test _datetime_to_int_days.""" + computed_dats = _datetime_to_int_days(self.cube1) + expected_dats = [14, 45] + self.assert_array_equal(computed_dats, expected_dats) + def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], From bddd18d13544b2056f18fda2d3f256e92d9b7eb2 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jun 2020 17:57:04 +0100 Subject: [PATCH 07/12] peter suggestion Co-authored-by: Peter Kalverla --- esmvalcore/preprocessor/_multimodel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 0f77b56498..d19ef1671a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -194,8 +194,7 @@ def _datetime_to_int_days(cube, overlap=False): def _align_yearly_axes(cube): """Perform a time-regridding operation to align time axes for yr data.""" - time_cells = [cell.point for cell in cube.coord('time').cells()] - years = [date_obj.year for date_obj in time_cells] + years = [cell.point.year for cell in cube.coord('time').cells()] # be extra sure that the first point is not in the previous year if len(years) > 2: delta = int(years[2]) - int(years[1]) From 6e0d1a6d8506b0d5b143e0db4fe0a600fdc899d5 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jun 2020 17:57:35 +0100 Subject: [PATCH 08/12] peter suggestion Co-authored-by: Peter Kalverla --- esmvalcore/preprocessor/_multimodel.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index d19ef1671a..086762893d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -196,11 +196,7 @@ def _align_yearly_axes(cube): """Perform a time-regridding operation to align time axes for yr data.""" years = [cell.point.year for cell in cube.coord('time').cells()] # be extra sure that the first point is not in the previous year - if len(years) > 2: - delta = int(years[2]) - int(years[1]) - else: - delta = int(years[1]) - int(years[0]) - if delta == 1: + if not 0 in np.diff(years): return regrid_time(cube, 'yr') else: return cube From a32bcb646f2bc2348d8dd4bb55527c8189ccbf02 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 18 Jun 2020 15:48:51 +0100 Subject: [PATCH 09/12] reverted a couple useless changes and added solution by Peter --- esmvalcore/preprocessor/_multimodel.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 086762893d..2e6e882bf6 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -111,10 +111,12 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): if t_axis is None: times = template_cube.coord('time') else: + unit_name = template_cube.coord('time').units.name + tunits = cf_units.Unit(unit_name, calendar="standard") times = iris.coords.DimCoord( t_axis, standard_name='time', - units=template_cube.coord('time').units) + units=tunits) coord_names = [c.long_name for c in template_cube.coords()] coord_names.extend([c.standard_name for c in template_cube.coords()]) @@ -164,7 +166,7 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube -def _datetime_to_int_days(cube, overlap=False): +def _datetime_to_int_days(cube): """Return list of int(days) converted from cube datetime cells.""" cube = _align_yearly_axes(cube) time_cells = [cell.point for cell in cube.coord('time').cells()] @@ -175,13 +177,7 @@ def _datetime_to_int_days(cube, overlap=False): # real_date resets the actual data point day # to the 1st of the month so that there are no # wrong overlap indices - # NOTE: this workaround is good only - # for monthly data - if overlap: - real_date = datetime(date_obj.year, date_obj.month, 1, 0, 0, 0) - else: - real_date = datetime(date_obj.year, - date_obj.month, 15, 0, 0, 0) + real_date = datetime(date_obj.year, date_obj.month, 1, 0, 0, 0) real_dates.append(real_date) # get the number of days starting from the reference unit @@ -215,7 +211,7 @@ def _get_overlap(cubes): """ all_times = [] for cube in cubes: - span = _datetime_to_int_days(cube, overlap=True) + span = _datetime_to_int_days(cube) start, stop = span[0], span[-1] all_times.append([start, stop]) bounds = [range(b[0], b[-1] + 1) for b in all_times] @@ -233,7 +229,7 @@ def _slice_cube(cube, t_1, t_2): of common time-data elements. """ time_pts = [t for t in cube.coord('time').points] - converted_t = _datetime_to_int_days(cube, overlap=True) + converted_t = _datetime_to_int_days(cube) idxs = sorted([ time_pts.index(ii) for ii, jj in zip(time_pts, converted_t) if t_1 <= jj <= t_2 From 8aae8895169faea2befacd70cd3bd75075de4721 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 18 Jun 2020 15:49:12 +0100 Subject: [PATCH 10/12] removed useless test and fixed other --- tests/unit/preprocessor/_multimodel/test_multimodel.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 6443c16393..4cb7533d49 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -171,16 +171,10 @@ def test_put_in_cube(self): stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) self.assert_array_equal(stat_cube.data, self.cube1.data) - def test_datetime_to_int_days_overlap(self): - """Test _datetime_to_int_days.""" - computed_dats = _datetime_to_int_days(self.cube1, overlap=True) - expected_dats = [0, 31] - self.assert_array_equal(computed_dats, expected_dats) - def test_datetime_to_int_days_no_overlap(self): """Test _datetime_to_int_days.""" computed_dats = _datetime_to_int_days(self.cube1) - expected_dats = [14, 45] + expected_dats = [0, 31] self.assert_array_equal(computed_dats, expected_dats) def test_assemble_overlap_data(self): From 8f6070120ab96a46b8ad918fc6daf08553bcdf53 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 18 Jun 2020 16:02:59 +0100 Subject: [PATCH 11/12] codacy huff --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 2e6e882bf6..247ddefa5d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -192,7 +192,7 @@ def _align_yearly_axes(cube): """Perform a time-regridding operation to align time axes for yr data.""" years = [cell.point.year for cell in cube.coord('time').cells()] # be extra sure that the first point is not in the previous year - if not 0 in np.diff(years): + if 0 not in np.diff(years): return regrid_time(cube, 'yr') else: return cube From 9eb8de5a48a8dc06aa271651ba7297318fe7f875 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 19 Jun 2020 14:16:06 +0100 Subject: [PATCH 12/12] cleaned statement for codacy --- esmvalcore/preprocessor/_multimodel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 247ddefa5d..a7e68143c4 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -194,8 +194,7 @@ def _align_yearly_axes(cube): # be extra sure that the first point is not in the previous year if 0 not in np.diff(years): return regrid_time(cube, 'yr') - else: - return cube + return cube def _get_overlap(cubes):