From e3189bdabffaa2583090b7c99bc0e3859834927f Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 2 Oct 2020 15:03:24 +0200 Subject: [PATCH 01/19] Do not push --- esmvalcore/cmor/check.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 3bf891803c..a346f4160a 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -3,6 +3,8 @@ from enum import IntEnum import cf_units +import datetime +import calendar as clr import iris.coord_categorisation import iris.coords import iris.exceptions @@ -461,6 +463,37 @@ def _check_coord_bounds(self, cmor, coord, var_name): coord.var_name, var_name ) + def _check_time_bounds(self, time): + cmor = self._cmor_var.coordinates['time'] + if cmor.must_have_bounds == 'yes' and not time.has_bounds(): + if self.automatic_fixes: + freq = self.frequency + for step in len(time.points): + month = time.cell(step).point.month + year = time.cell(step).point.year + if freq in ['mon', 'mo']: + next_month = clr.nextmonth(year, month) + min_bound = time.units.date2num( + datetime.datetime(year, month, 1, 0, 0) + ) + max_bound = time.units.date2num( + datetime.datetime( + next_month[0], next_month[1], 1, 0, 0) + ) + elif freq == 'yr': + min_bound = time.units.date2num( + datetime.datetime(year, 1, 1, 0, 0) + ) + max_bound = time.units.date2num( + datetime.datetime(year+1, 1, 1, 0, 0) + ) + # to deal with other freqs + time.bounds = np.array([min_bound, max_bound]) + self.report_warning( + 'Added guessed bounds to coordinate {0} from var time', + time.var_name + ) + def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" if coord.ndim > 1: @@ -606,9 +639,7 @@ def _check_time_coord(self): calendar=coord.units.calendar)) simplified_cal = self._simplify_calendar(coord.units.calendar) coord.units = cf_units.Unit(coord.units.origin, simplified_cal) - attrs = self._cube.attributes - parent_time = 'parent_time_units' if parent_time in attrs: if attrs[parent_time] in 'no parent': @@ -638,6 +669,7 @@ def _check_time_coord(self): tol = 0.001 intervals = {'dec': (3600, 3660), 'day': (1, 1)} freq = self.frequency + self._check_time_bounds(coord) if freq.lower().endswith('pt'): freq = freq[:-2] if freq in ['mon', 'mo']: @@ -686,7 +718,7 @@ def _check_time_coord(self): msg = '{}: Frequency {} does not match input data' self.report_error(msg, var_name, freq) break - + self._check_time_bounds(coord) # remove time_origin from attributes coord.attributes.pop('time_origin', None) From 0444ef46764b2cd63b581e7018b004c3ef1265d5 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 27 Oct 2020 12:55:07 +0100 Subject: [PATCH 02/19] Add bounds for other frequencies --- esmvalcore/cmor/check.py | 41 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index a346f4160a..4a837479f3 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -4,7 +4,6 @@ import cf_units import datetime -import calendar as clr import iris.coord_categorisation import iris.coords import iris.exceptions @@ -463,22 +462,22 @@ def _check_coord_bounds(self, cmor, coord, var_name): coord.var_name, var_name ) - def _check_time_bounds(self, time): + def _check_time_bounds(self, freq, time): cmor = self._cmor_var.coordinates['time'] if cmor.must_have_bounds == 'yes' and not time.has_bounds(): if self.automatic_fixes: - freq = self.frequency - for step in len(time.points): + bounds = [] + for step, point in enumerate(time.points): month = time.cell(step).point.month year = time.cell(step).point.year if freq in ['mon', 'mo']: - next_month = clr.nextmonth(year, month) + next_month = self._get_next_month(month, year) min_bound = time.units.date2num( datetime.datetime(year, month, 1, 0, 0) ) max_bound = time.units.date2num( datetime.datetime( - next_month[0], next_month[1], 1, 0, 0) + next_month[1], next_month[0], 1, 0, 0) ) elif freq == 'yr': min_bound = time.units.date2num( @@ -487,12 +486,27 @@ def _check_time_bounds(self, time): max_bound = time.units.date2num( datetime.datetime(year+1, 1, 1, 0, 0) ) - # to deal with other freqs - time.bounds = np.array([min_bound, max_bound]) - self.report_warning( - 'Added guessed bounds to coordinate {0} from var time', - time.var_name - ) + else: + delta = { + 'day': 12/24, + '6hr': 3/24, + '3hr': 1.5/24, + '1hr': 0.5/24, + } + min_bound = point-delta[freq] + max_bound = point+delta[freq] + bounds.append([min_bound, max_bound]) + time.bounds = np.array(bounds) + self.report_warning( + 'Added guessed bounds to coordinate {0} from var {1}', + time.var_name, self._cmor_var.short_name + ) + + def _get_next_month(self, month, year): + if month != 12: + return month+1, year + else: + return 1, year+1 def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" @@ -669,7 +683,6 @@ def _check_time_coord(self): tol = 0.001 intervals = {'dec': (3600, 3660), 'day': (1, 1)} freq = self.frequency - self._check_time_bounds(coord) if freq.lower().endswith('pt'): freq = freq[:-2] if freq in ['mon', 'mo']: @@ -718,7 +731,7 @@ def _check_time_coord(self): msg = '{}: Frequency {} does not match input data' self.report_error(msg, var_name, freq) break - self._check_time_bounds(coord) + self._check_time_bounds(freq, coord) # remove time_origin from attributes coord.attributes.pop('time_origin', None) From 844d9bb114b5b501d905d397ca3b31488041725f Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 27 Oct 2020 14:59:33 +0100 Subject: [PATCH 03/19] Fix tests --- tests/unit/cmor/test_cmor_check.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 63006a7782..8bb8699fd2 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -355,8 +355,9 @@ def test_check_missing_time_strict_flag(self): def test_check_missing_coord_strict_flag(self): """Test check fails for missing coord other than lat and lon with --cmor-check strict""" - self.var_info.coordinates = {'height2m': CoordinateInfoMock('height2m') - } + self.var_info.coordinates.update( + {'height2m': CoordinateInfoMock('height2m')} + ) self._check_fails_in_metadata(automatic_fixes=False) def test_check_bad_var_standard_name_relaxed_flag(self): @@ -431,8 +432,9 @@ def test_check_missing_time_relaxed_flag(self): def test_check_missing_coord_relaxed_flag(self): """Test check reports warning for missing coord other than lat and lon with --cmor-check relaxed""" - self.var_info.coordinates = {'height2m': CoordinateInfoMock('height2m') - } + self.var_info.coordinates.update( + {'height2m': CoordinateInfoMock('height2m')} + ) self._check_warnings_on_metadata(automatic_fixes=False, check_level=CheckLevels.RELAXED) @@ -511,8 +513,9 @@ def test_check_missing_time_none_flag(self): def test_check_missing_coord_none_flag(self): """Test check reports warning for missing coord other than lat, lon and time with --cmor-check ignore""" - self.var_info.coordinates = {'height2m': CoordinateInfoMock('height2m') - } + self.var_info.coordinates.update( + {'height2m': CoordinateInfoMock('height2m')} + ) self._check_warnings_on_metadata(automatic_fixes=False, check_level=CheckLevels.IGNORE) From 3891ea4d55021c175cbf9afb80a92fdc35957768 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 27 Oct 2020 16:37:04 +0100 Subject: [PATCH 04/19] Fix KeyError --- esmvalcore/cmor/check.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 2a506da97d..09e3e51066 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -466,7 +466,9 @@ def _check_coord_bounds(self, cmor, coord, var_name): ) def _check_time_bounds(self, freq, time): - cmor = self._cmor_var.coordinates['time'] + for coord in self._cmor_var.coordinates: + if self._cmor_var.coordinates[coord].axis == 'T': + cmor = self._cmor_var.coordinates[coord] if cmor.must_have_bounds == 'yes' and not time.has_bounds(): if self.automatic_fixes: bounds = [] From 8d59972690e09ae6fee3023f938f76e860e14183 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 27 Oct 2020 17:11:21 +0100 Subject: [PATCH 05/19] Change for --- esmvalcore/cmor/check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 09e3e51066..e3ebb8901a 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -466,9 +466,9 @@ def _check_coord_bounds(self, cmor, coord, var_name): ) def _check_time_bounds(self, freq, time): - for coord in self._cmor_var.coordinates: - if self._cmor_var.coordinates[coord].axis == 'T': - cmor = self._cmor_var.coordinates[coord] + times = {'time', 'time1', 'time2', 'time3'} + key = times.intersection(self._cmor_var.coordinates) + cmor = self._cmor_var.coordinates[" ".join(key)] if cmor.must_have_bounds == 'yes' and not time.has_bounds(): if self.automatic_fixes: bounds = [] From 9aa51b7d3867fe4a719ac22c649cad2d17fb4bc9 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 28 Oct 2020 10:19:42 +0100 Subject: [PATCH 06/19] Add dec frequency --- esmvalcore/cmor/check.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index e3ebb8901a..ea5d7adceb 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -491,6 +491,13 @@ def _check_time_bounds(self, freq, time): max_bound = time.units.date2num( datetime.datetime(year+1, 1, 1, 0, 0) ) + elif freq == 'dec': + min_bound = time.units.date2num( + datetime.datetime(year, 1, 1, 0, 0) + ) + max_bound = time.units.date2num( + datetime.datetime(year+10, 1, 1, 0, 0) + ) else: delta = { 'day': 12/24, @@ -506,7 +513,6 @@ def _check_time_bounds(self, freq, time): 'Added guessed bounds to coordinate {0} from var {1}', time.var_name, self._cmor_var.short_name ) - def _get_next_month(self, month, year): if month != 12: return month+1, year From 1c9c57dc064087e794ddb2e9fa4e3ba6fb02db2c Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 28 Oct 2020 10:20:13 +0100 Subject: [PATCH 07/19] Add tests --- tests/unit/cmor/test_cmor_check.py | 80 +++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 8bb8699fd2..ae68f5dc74 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -905,6 +905,78 @@ def test_frequency_not_supported(self): """Fail at metadata if frequency is not supported.""" self._check_fails_in_metadata(frequency='wrong_freq') + def test_hourly_time_bounds(self): + """Test time bounds are properly guessed for hourly freq""" + cube = self.get_cube(self.var_info, frequency='hr') + original_bounds = self.cube.coord('time').bounds + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert original_bounds.all() == guessed_bounds.all() + + def test_3hourly_time_bounds(self): + """Test time bounds are properly guessed for 3hourly freq""" + cube = self.get_cube(self.var_info, frequency='3hr') + original_bounds = self.cube.coord('time').bounds + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert original_bounds.all() == guessed_bounds.all() + + def test_6hourly_time_bounds(self): + """Test time bounds are properly guessed for 6hourly freq""" + cube = self.get_cube(self.var_info, frequency='6hr') + original_bounds = self.cube.coord('time').bounds + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert original_bounds.all() == guessed_bounds.all() + + def test_daily_time_bounds(self): + """Test time bounds are properly guessed for daily freq""" + original_bounds = self.cube.coord('time').bounds + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert original_bounds.all() == guessed_bounds.all() + + def test_monthly_time_bounds(self): + """Test time bounds are properly guessed for monthly freq""" + cube = self.get_cube(self.var_info, frequency='mon') + original_bounds = cube.coord('time').bounds + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert original_bounds.all() == guessed_bounds.all() + + def test_yearly_time_bounds(self): + """Test time bounds are properly guessed for yearly freq""" + cube = self.get_cube(self.var_info, frequency='yr') + original_bounds = cube.coord('time').bounds + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert original_bounds.all() == guessed_bounds.all() + + def test_dec_time_bounds(self): + """Test time bounds are properly guessed for dec freq""" + cube = self.get_cube(self.var_info, frequency='dec') + original_bounds = cube.coord('time').bounds + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert original_bounds.all() == guessed_bounds.all() + + def test_no_time_bounds(self): + """Test time bounds are not guessed for instantaneous data""" + self.var_info.coordinates['time'].must_have_bounds = 'no' + self.var_info.coordinates['time1'] = ( + self.var_info.coordinates.pop('time')) + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert guessed_bounds is None + def _check_fails_on_data(self): checker = CMORCheck(self.cube, self.var_info) checker.check_metadata() @@ -1103,7 +1175,13 @@ def _get_time_values(dim_spec): delta = 30 elif frequency == 'day': delta = 1 - elif frequency.ends_with('hr'): + elif frequency == 'yr': + delta = 360 + elif frequency == 'dec': + delta = 3600 + elif frequency.endswith('hr'): + if frequency == 'hr': + frequency = '1hr' delta = float(frequency[:-2]) / 24 else: raise Exception('Frequency {} not supported'.format(frequency)) From 56328e47f718783c6fd975c4b24ef04e82cbb43c Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 28 Oct 2020 11:45:57 +0100 Subject: [PATCH 08/19] Fix bug in 1hr frequency --- esmvalcore/cmor/check.py | 3 ++ tests/unit/cmor/test_cmor_check.py | 50 +++++++++++++++--------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index ea5d7adceb..37eedf340f 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -513,6 +513,7 @@ def _check_time_bounds(self, freq, time): 'Added guessed bounds to coordinate {0} from var {1}', time.var_name, self._cmor_var.short_name ) + def _get_next_month(self, month, year): if month != 12: return month+1, year @@ -724,6 +725,8 @@ def _check_time_coord(self): interval = intervals[freq] target_interval = (interval[0] - tol, interval[1] + tol) elif freq.endswith('hr'): + if freq == 'hr': + freq = '1hr' frequency = freq[:-2] if frequency == 'sub': frequency = 1.0 / 24 diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index ae68f5dc74..175437dbf2 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -908,9 +908,10 @@ def test_frequency_not_supported(self): def test_hourly_time_bounds(self): """Test time bounds are properly guessed for hourly freq""" cube = self.get_cube(self.var_info, frequency='hr') - original_bounds = self.cube.coord('time').bounds - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) + original_bounds = cube.coord('time').bounds + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency='hr') guessed_bounds = self.cube.coord('time').bounds assert original_bounds.all() == guessed_bounds.all() @@ -918,8 +919,9 @@ def test_3hourly_time_bounds(self): """Test time bounds are properly guessed for 3hourly freq""" cube = self.get_cube(self.var_info, frequency='3hr') original_bounds = self.cube.coord('time').bounds - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency = '3hr') guessed_bounds = self.cube.coord('time').bounds assert original_bounds.all() == guessed_bounds.all() @@ -927,8 +929,9 @@ def test_6hourly_time_bounds(self): """Test time bounds are properly guessed for 6hourly freq""" cube = self.get_cube(self.var_info, frequency='6hr') original_bounds = self.cube.coord('time').bounds - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency='6hr') guessed_bounds = self.cube.coord('time').bounds assert original_bounds.all() == guessed_bounds.all() @@ -941,31 +944,28 @@ def test_daily_time_bounds(self): assert original_bounds.all() == guessed_bounds.all() def test_monthly_time_bounds(self): - """Test time bounds are properly guessed for monthly freq""" + """Test time bounds are guessed for monthly freq""" cube = self.get_cube(self.var_info, frequency='mon') - original_bounds = cube.coord('time').bounds - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) - guessed_bounds = self.cube.coord('time').bounds - assert original_bounds.all() == guessed_bounds.all() + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency='mon') + assert self.cube.coord('time').bounds is not None def test_yearly_time_bounds(self): - """Test time bounds are properly guessed for yearly freq""" + """Test time bounds are guessed for yearly freq""" cube = self.get_cube(self.var_info, frequency='yr') - original_bounds = cube.coord('time').bounds - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) - guessed_bounds = self.cube.coord('time').bounds - assert original_bounds.all() == guessed_bounds.all() + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency='yr') + assert self.cube.coord('time').bounds is not None def test_dec_time_bounds(self): - """Test time bounds are properly guessed for dec freq""" + """Test time bounds are guessed for dec freq""" cube = self.get_cube(self.var_info, frequency='dec') - original_bounds = cube.coord('time').bounds - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) - guessed_bounds = self.cube.coord('time').bounds - assert original_bounds.all() == guessed_bounds.all() + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency='dec') + assert self.cube.coord('time').bounds is not None def test_no_time_bounds(self): """Test time bounds are not guessed for instantaneous data""" From a7dc456e6e7d9090ec3ebfef48fedc7dc13b257b Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 28 Oct 2020 11:51:45 +0100 Subject: [PATCH 09/19] Fix flake --- tests/unit/cmor/test_cmor_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 175437dbf2..7726b82da7 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -921,7 +921,7 @@ def test_3hourly_time_bounds(self): original_bounds = self.cube.coord('time').bounds cube.coord('time').bounds = None self.cube = cube - self._check_cube(automatic_fixes=True, frequency = '3hr') + self._check_cube(automatic_fixes=True, frequency='3hr') guessed_bounds = self.cube.coord('time').bounds assert original_bounds.all() == guessed_bounds.all() From 11217b7df2746d2053e7beef0da8a3d1366657f4 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 28 Oct 2020 17:41:53 +0100 Subject: [PATCH 10/19] Fix style issues --- esmvalcore/cmor/check.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 37eedf340f..87c2ed5c82 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -2,8 +2,8 @@ import logging from enum import IntEnum -import cf_units import datetime +import cf_units import iris.coord_categorisation import iris.coords import iris.exceptions @@ -479,34 +479,34 @@ def _check_time_bounds(self, freq, time): next_month = self._get_next_month(month, year) min_bound = time.units.date2num( datetime.datetime(year, month, 1, 0, 0) - ) + ) max_bound = time.units.date2num( datetime.datetime( next_month[1], next_month[0], 1, 0, 0) - ) + ) elif freq == 'yr': min_bound = time.units.date2num( datetime.datetime(year, 1, 1, 0, 0) - ) + ) max_bound = time.units.date2num( - datetime.datetime(year+1, 1, 1, 0, 0) - ) + datetime.datetime(year + 1, 1, 1, 0, 0) + ) elif freq == 'dec': min_bound = time.units.date2num( datetime.datetime(year, 1, 1, 0, 0) - ) + ) max_bound = time.units.date2num( - datetime.datetime(year+10, 1, 1, 0, 0) - ) + datetime.datetime(year + 10, 1, 1, 0, 0) + ) else: delta = { - 'day': 12/24, - '6hr': 3/24, - '3hr': 1.5/24, - '1hr': 0.5/24, + 'day': 12 / 24, + '6hr': 3 / 24, + '3hr': 1.5 / 24, + '1hr': 0.5 / 24, } - min_bound = point-delta[freq] - max_bound = point+delta[freq] + min_bound = point - delta[freq] + max_bound = point + delta[freq] bounds.append([min_bound, max_bound]) time.bounds = np.array(bounds) self.report_warning( @@ -514,11 +514,11 @@ def _check_time_bounds(self, freq, time): time.var_name, self._cmor_var.short_name ) - def _get_next_month(self, month, year): + @staticmethod + def _get_next_month(month, year): if month != 12: - return month+1, year - else: - return 1, year+1 + return month + 1, year + return 1, year + 1 def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" From 8e4bc5a99c964cdc7fd7788a4094a170c82d5f8d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Jan 2021 13:29:35 +0100 Subject: [PATCH 11/19] Fix format --- esmvalcore/cmor/check.py | 213 ++++++++++++++++++--------------------- 1 file changed, 97 insertions(+), 116 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 87c2ed5c82..a174c7c4e7 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -1,8 +1,8 @@ """Module for checking iris cubes against their CMOR definitions.""" +import datetime import logging from enum import IntEnum -import datetime import cf_units import iris.coord_categorisation import iris.coords @@ -12,8 +12,7 @@ from .table import CMOR_TABLES -CheckLevels = IntEnum( - 'CheckLevels', 'DEBUG STRICT DEFAULT RELAXED IGNORE') +CheckLevels = IntEnum('CheckLevels', 'DEBUG STRICT DEFAULT RELAXED IGNORE') """Level of strictness of the checks. Attributes @@ -87,8 +86,7 @@ def __init__(self, self.automatic_fixes = automatic_fixes def check_metadata(self, logger=None): - """ - Check the cube metadata. + """Check the cube metadata. Perform all the tests that do not require to have the data in memory. @@ -110,7 +108,6 @@ def check_metadata(self, logger=None): If errors are found. If fail_on_error attribute is set to True, raises as soon as an error is detected. If set to False, it perform all checks and then raises. - """ if logger is not None: self._logger = logger @@ -151,7 +148,6 @@ def check_data(self, logger=None): If errors are found. If fail_on_error attribute is set to True, raises as soon as an error is detected. If set to False, it perform all checks and then raises. - """ if logger is not None: self._logger = logger @@ -174,7 +170,6 @@ def report_errors(self): ------ CMORCheckError If any errors were reported before calling this method. - """ if self.has_errors(): msg = 'There were errors in variable {}:\n{}\nin cube:\n{}' @@ -189,7 +184,6 @@ def report_warnings(self): ---------- logger: logging.Logger Given logger - """ if self.has_warnings(): msg = 'There were warnings in variable {}:\n{}\n'.format( @@ -203,7 +197,6 @@ def report_debug_messages(self): ---------- logger: logging.Logger Given logger. - """ if self.has_debug_messages(): msg = 'There were metadata changes in variable {}:\n{}\n'.format( @@ -226,32 +219,27 @@ def _check_var_metadata(self): if self.automatic_fixes: self.report_warning( 'Standard name for {} changed from {} to {}', - self._cube.var_name, - self._cube.standard_name, - self._cmor_var.standard_name - ) + self._cube.var_name, self._cube.standard_name, + self._cmor_var.standard_name) self._cube.standard_name = self._cmor_var.standard_name else: - self.report_error( - self._attr_msg, self._cube.var_name, 'standard_name', - self._cmor_var.standard_name, self._cube.standard_name - ) + self.report_error(self._attr_msg, self._cube.var_name, + 'standard_name', + self._cmor_var.standard_name, + self._cube.standard_name) # Check long_name if self._cmor_var.long_name: if self._cube.long_name != self._cmor_var.long_name: if self.automatic_fixes: self.report_warning( 'Long name for {} changed from {} to {}', - self._cube.var_name, - self._cube.long_name, - self._cmor_var.long_name - ) + self._cube.var_name, self._cube.long_name, + self._cmor_var.long_name) self._cube.long_name = self._cmor_var.long_name else: - self.report_error( - self._attr_msg, self._cube.var_name, 'long_name', - self._cmor_var.long_name, self._cube.long_name - ) + self.report_error(self._attr_msg, self._cube.var_name, + 'long_name', self._cmor_var.long_name, + self._cube.long_name) # Check units if (self.automatic_fixes and self._cube.attributes.get( @@ -263,10 +251,9 @@ def _check_var_metadata(self): units = self._get_effective_units() if self._cube.units != units: if not self._cube.units.is_convertible(units): - self.report_error( - f'Variable {self._cube.var_name} units ' - f'{self._cube.units} can not be ' - f'converted to {self._cmor_var.units}') + self.report_error(f'Variable {self._cube.var_name} units ' + f'{self._cube.units} can not be ' + f'converted to {self._cmor_var.units}') else: self.report_warning( f'Variable {self._cube.var_name} units ' @@ -282,10 +269,9 @@ def _check_var_metadata(self): self.report_warning('{}: attribute {} not present', self._cube.var_name, attr) elif self._cube.attributes[attr] != attr_value: - self.report_error( - self._attr_msg, self._cube.var_name, - attr, attr_value, - self._cube.attributes[attr]) + self.report_error(self._attr_msg, self._cube.var_name, + attr, attr_value, + self._cube.attributes[attr]) def _get_effective_units(self): """Get effective units.""" @@ -321,13 +307,13 @@ def _check_multiple_coords_same_stdname(self): for coord in self._cube.coords(): if coord.standard_name: if coord.standard_name in standard_names: - coords = [c.var_name for c in self._cube.coords( - standard_name=coord.standard_name) + coords = [ + c.var_name for c in self._cube.coords( + standard_name=coord.standard_name) ] self.report_error( 'There are multiple coordinates with ' - f'standard_name "{coord.standard_name}": {coords}' - ) + f'standard_name "{coord.standard_name}": {coords}') else: standard_names.add(coord.standard_name) @@ -339,8 +325,8 @@ def _check_dim_names(self): else: try: cube_coord = self._cube.coord(var_name=coordinate.out_name) - if (cube_coord.standard_name is None and - coordinate.standard_name == ''): + if (cube_coord.standard_name is None + and coordinate.standard_name == ''): pass elif cube_coord.standard_name != coordinate.standard_name: self.report_critical( @@ -378,11 +364,11 @@ def _check_dim_names(self): if coordinate.standard_name in ['time', 'latitude', 'longitude'] or \ coordinate.requested: - self.report_critical( - self._does_msg, coordinate.name, 'exist') + self.report_critical(self._does_msg, + coordinate.name, 'exist') else: - self.report_error( - self._does_msg, coordinate.name, 'exist') + self.report_error(self._does_msg, coordinate.name, + 'exist') def _check_coords(self): """Check coordinates.""" @@ -452,18 +438,15 @@ def _check_coord_bounds(self, cmor, coord, var_name): except ValueError as ex: self.report_warning( 'Can not guess bounds for coordinate {0} ' - 'from var {1}: {2}', coord.var_name, var_name, ex - ) + 'from var {1}: {2}', coord.var_name, var_name, ex) else: self.report_warning( 'Added guessed bounds to coordinate {0} from var {1}', - coord.var_name, var_name - ) + coord.var_name, var_name) else: self.report_warning( 'Coordinate {0} from var {1} does not have bounds', - coord.var_name, var_name - ) + coord.var_name, var_name) def _check_time_bounds(self, freq, time): times = {'time', 'time1', 'time2', 'time3'} @@ -478,26 +461,20 @@ def _check_time_bounds(self, freq, time): if freq in ['mon', 'mo']: next_month = self._get_next_month(month, year) min_bound = time.units.date2num( - datetime.datetime(year, month, 1, 0, 0) - ) + datetime.datetime(year, month, 1, 0, 0)) max_bound = time.units.date2num( - datetime.datetime( - next_month[1], next_month[0], 1, 0, 0) - ) + datetime.datetime(next_month[1], next_month[0], 1, + 0, 0)) elif freq == 'yr': min_bound = time.units.date2num( - datetime.datetime(year, 1, 1, 0, 0) - ) + datetime.datetime(year, 1, 1, 0, 0)) max_bound = time.units.date2num( - datetime.datetime(year + 1, 1, 1, 0, 0) - ) + datetime.datetime(year + 1, 1, 1, 0, 0)) elif freq == 'dec': min_bound = time.units.date2num( - datetime.datetime(year, 1, 1, 0, 0) - ) + datetime.datetime(year, 1, 1, 0, 0)) max_bound = time.units.date2num( - datetime.datetime(year + 10, 1, 1, 0, 0) - ) + datetime.datetime(year + 10, 1, 1, 0, 0)) else: delta = { 'day': 12 / 24, @@ -511,8 +488,7 @@ def _check_time_bounds(self, freq, time): time.bounds = np.array(bounds) self.report_warning( 'Added guessed bounds to coordinate {0} from var {1}', - time.var_name, self._cmor_var.short_name - ) + time.var_name, self._cmor_var.short_name) @staticmethod def _get_next_month(month, year): @@ -534,15 +510,15 @@ def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): if cmor.stored_direction == 'increasing': if coord.points[0] > coord.points[1]: if not self.automatic_fixes or coord.ndim > 1: - self.report_critical( - self._is_msg, var_name, 'increasing') + self.report_critical(self._is_msg, var_name, + 'increasing') else: self._reverse_coord(coord) elif cmor.stored_direction == 'decreasing': if coord.points[0] < coord.points[1]: if not self.automatic_fixes or coord.ndim > 1: - self.report_critical( - self._is_msg, var_name, 'decreasing') + self.report_critical(self._is_msg, var_name, + 'decreasing') else: self._reverse_coord(coord) @@ -568,9 +544,9 @@ def _check_coord_values(self, coord_info, coord, var_name): l_fix_coord_value = self._check_longitude_min( coord, var_name) else: - self.report_critical( - self._vals_msg, var_name, - '< {} ='.format('valid_min'), valid_min) + self.report_critical(self._vals_msg, var_name, + '< {} ='.format('valid_min'), + valid_min) if coord_info.valid_max: valid_max = float(coord_info.valid_max) @@ -580,14 +556,14 @@ def _check_coord_values(self, coord_info, coord, var_name): l_fix_coord_value = self._check_longitude_max( coord, var_name) else: - self.report_critical( - self._vals_msg, var_name, - '> {} ='.format('valid_max'), valid_max) + self.report_critical(self._vals_msg, var_name, + '> {} ='.format('valid_max'), + valid_max) if l_fix_coord_value: if coord.ndim == 1: - lon_extent = iris.coords.CoordExtent( - coord, 0.0, 360., True, False) + lon_extent = iris.coords.CoordExtent(coord, 0.0, 360., True, + False) self._cube = self._cube.intersection(lon_extent) else: new_lons = coord.points.copy() @@ -605,16 +581,14 @@ def _check_coord_values(self, coord_info, coord, var_name): def _check_longitude_max(self, coord, var_name): if np.any(coord.points > 720): self.report_critical( - f'{var_name} longitude coordinate has values > 720 degrees' - ) + f'{var_name} longitude coordinate has values > 720 degrees') return False return True def _check_longitude_min(self, coord, var_name): if np.any(coord.points < -360): self.report_critical( - f'{var_name} longitude coordinate has values < -360 degrees' - ) + f'{var_name} longitude coordinate has values < -360 degrees') return False return True @@ -650,9 +624,8 @@ def _check_time_coord(self): var_name = coord.var_name if not coord.is_monotonic(): - self.report_error( - 'Time coordinate for var {} is not monotonic', var_name - ) + self.report_error('Time coordinate for var {} is not monotonic', + var_name) if not coord.units.is_time_reference(): self.report_critical(self._does_msg, var_name, @@ -660,9 +633,8 @@ def _check_time_coord(self): else: old_units = coord.units coord.convert_units( - cf_units.Unit( - 'days since 1850-1-1 00:00:00', - calendar=coord.units.calendar)) + cf_units.Unit('days since 1850-1-1 00:00:00', + calendar=coord.units.calendar)) simplified_cal = self._simplify_calendar(coord.units.calendar) coord.units = cf_units.Unit(coord.units.origin, simplified_cal) attrs = self._cube.attributes @@ -765,7 +737,6 @@ def has_errors(self): ------- bool: True if there are pending errors, False otherwise. - """ return len(self._errors) > 0 @@ -776,7 +747,6 @@ def has_warnings(self): ------- bool: True if there are pending warnings, False otherwise. - """ return len(self._warnings) > 0 @@ -787,12 +757,11 @@ def has_debug_messages(self): ------- bool: True if there are pending debug messages, False otherwise. - """ return len(self._debug_messages) > 0 def report(self, level, message, *args): - """Generic method to report a message from the checker + """Generic method to report a message from the checker. Parameters ---------- @@ -837,7 +806,6 @@ def report_critical(self, message, *args): Message for the error. *args: arguments to format the message string. - """ self.report(CheckLevels.RELAXED, message, *args) @@ -850,7 +818,6 @@ def report_error(self, message, *args): Message for the error. *args: arguments to format the message string. - """ self.report(CheckLevels.DEFAULT, message, *args) @@ -863,7 +830,6 @@ def report_warning(self, message, *args): Message for the warning. *args: arguments to format the message string. - """ self.report(CheckLevels.STRICT, message, *args) @@ -876,7 +842,6 @@ def report_debug_message(self, message, *args): Message for the debug logger. *args: arguments to format the message string - """ self.report(CheckLevels.DEBUG, message, *args) @@ -901,21 +866,23 @@ def _get_cmor_checker(table, var_info = CMOR_TABLES['custom'].get_variable(mip, short_name) def _checker(cube): - return CMORCheck( - cube, - var_info, - frequency=frequency, - fail_on_error=fail_on_error, - check_level=check_level, - automatic_fixes=automatic_fixes) + return CMORCheck(cube, + var_info, + frequency=frequency, + fail_on_error=fail_on_error, + check_level=check_level, + automatic_fixes=automatic_fixes) return _checker -def cmor_check_metadata(cube, cmor_table, mip, - short_name, frequency, +def cmor_check_metadata(cube, + cmor_table, + mip, + short_name, + frequency, check_level=CheckLevels.DEFAULT): - """Check if metadata conforms to variable's CMOR definiton. + """Check if metadata conforms to variable's CMOR definition. None of the checks at this step will force the cube to load the data. @@ -933,18 +900,23 @@ def cmor_check_metadata(cube, cmor_table, mip, Data frequency. check_level: CheckLevels Level of strictness of the checks. - """ - checker = _get_cmor_checker(cmor_table, mip, - short_name, frequency, + checker = _get_cmor_checker(cmor_table, + mip, + short_name, + frequency, check_level=check_level) checker(cube).check_metadata() return cube -def cmor_check_data(cube, cmor_table, mip, short_name, frequency, +def cmor_check_data(cube, + cmor_table, + mip, + short_name, + frequency, check_level=CheckLevels.DEFAULT): - """Check if data conforms to variable's CMOR definiton. + """Check if data conforms to variable's CMOR definition. The checks performed at this step require the data in memory. @@ -962,16 +934,18 @@ def cmor_check_data(cube, cmor_table, mip, short_name, frequency, Data frequency check_level: CheckLevels Level of strictness of the checks. - """ - checker = _get_cmor_checker(cmor_table, mip, short_name, frequency, + checker = _get_cmor_checker(cmor_table, + mip, + short_name, + frequency, check_level=check_level) checker(cube).check_data() return cube def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level): - """Check if cube conforms to variable's CMOR definiton. + """Check if cube conforms to variable's CMOR definition. Equivalent to calling cmor_check_metadata and cmor_check_data consecutively. @@ -990,10 +964,17 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level): Data frequency. check_level: enum.IntEnum Level of strictness of the checks. - """ - cmor_check_metadata(cube, cmor_table, mip, short_name, frequency, + cmor_check_metadata(cube, + cmor_table, + mip, + short_name, + frequency, check_level=check_level) - cmor_check_data(cube, cmor_table, mip, short_name, frequency, + cmor_check_data(cube, + cmor_table, + mip, + short_name, + frequency, check_level=check_level) return cube From fd75ee5d155b0f761a85ef8230a1f712c720765d Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 26 Jan 2021 08:35:58 +0100 Subject: [PATCH 12/19] Fix docstring --- esmvalcore/cmor/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index dbb58ae928..bb39105c36 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -813,7 +813,7 @@ def has_debug_messages(self): return len(self._debug_messages) > 0 def report(self, level, message, *args): - """Generic method to report a message from the checker. + """Report a message from the checker. Parameters ---------- From dafe506cd58befbad8ca5727864bd952bbbd9ec1 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 26 Jan 2021 12:55:46 +0100 Subject: [PATCH 13/19] Create common time bounds function --- esmvalcore/cmor/check.py | 41 ++------------------------------ esmvalcore/preprocessor/_time.py | 41 +++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index bb39105c36..d0db299538 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -1,5 +1,4 @@ """Module for checking iris cubes against their CMOR definitions.""" -import datetime import logging from enum import IntEnum @@ -501,53 +500,17 @@ def _check_coord_bounds(self, cmor, coord, var_name): coord.var_name, var_name) def _check_time_bounds(self, freq, time): + from esmvalcore.preprocessor._time import _get_time_bounds times = {'time', 'time1', 'time2', 'time3'} key = times.intersection(self._cmor_var.coordinates) cmor = self._cmor_var.coordinates[" ".join(key)] if cmor.must_have_bounds == 'yes' and not time.has_bounds(): if self.automatic_fixes: - bounds = [] - for step, point in enumerate(time.points): - month = time.cell(step).point.month - year = time.cell(step).point.year - if freq in ['mon', 'mo']: - next_month = self._get_next_month(month, year) - min_bound = time.units.date2num( - datetime.datetime(year, month, 1, 0, 0)) - max_bound = time.units.date2num( - datetime.datetime(next_month[1], next_month[0], 1, - 0, 0)) - elif freq == 'yr': - min_bound = time.units.date2num( - datetime.datetime(year, 1, 1, 0, 0)) - max_bound = time.units.date2num( - datetime.datetime(year + 1, 1, 1, 0, 0)) - elif freq == 'dec': - min_bound = time.units.date2num( - datetime.datetime(year, 1, 1, 0, 0)) - max_bound = time.units.date2num( - datetime.datetime(year + 10, 1, 1, 0, 0)) - else: - delta = { - 'day': 12 / 24, - '6hr': 3 / 24, - '3hr': 1.5 / 24, - '1hr': 0.5 / 24, - } - min_bound = point - delta[freq] - max_bound = point + delta[freq] - bounds.append([min_bound, max_bound]) - time.bounds = np.array(bounds) + time.bounds = _get_time_bounds(time, freq) self.report_warning( 'Added guessed bounds to coordinate {0} from var {1}', time.var_name, self._cmor_var.short_name) - @staticmethod - def _get_next_month(month, year): - if month != 12: - return month + 1, year - return 1, year + 1 - def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" if coord.ndim > 1: diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index e0101bf0f8..c1131ed7bb 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -718,7 +718,7 @@ def regrid_time(cube, frequency): # uniformize bounds cube.coord('time').bounds = None - cube.coord('time').guess_bounds() + cube.coord('time').bounds = _get_time_bounds(cube.coord('time'), frequency) # remove aux coords that will differ reset_aux = ['day_of_month', 'day_of_year'] @@ -737,6 +737,45 @@ def regrid_time(cube, frequency): return cube +def _get_time_bounds(time, freq): + bounds = [] + for step, point in enumerate(time.points): + month = time.cell(step).point.month + year = time.cell(step).point.year + if freq in ['mon', 'mo']: + next_month = _get_next_month(month, year) + min_bound = time.units.date2num( + datetime.datetime(year, month, 1, 0, 0)) + max_bound = time.units.date2num( + datetime.datetime(next_month[1], next_month[0], 1, 0, 0)) + elif freq == 'yr': + min_bound = time.units.date2num( + datetime.datetime(year, 1, 1, 0, 0)) + max_bound = time.units.date2num( + datetime.datetime(year + 1, 1, 1, 0, 0)) + elif freq == 'dec': + min_bound = time.units.date2num( + datetime.datetime(year, 1, 1, 0, 0)) + max_bound = time.units.date2num( + datetime.datetime(year + 10, 1, 1, 0, 0)) + else: + delta = { + 'day': 12 / 24, + '6hr': 3 / 24, + '3hr': 1.5 / 24, + '1hr': 0.5 / 24, + } + min_bound = point - delta[freq] + max_bound = point + delta[freq] + bounds.append([min_bound, max_bound]) + return np.array(bounds) + + +def _get_next_month(month, year): + if month != 12: + return month + 1, year + return 1, year + 1 + def low_pass_weights(window, cutoff): """Calculate weights for a low pass Lanczos filter. From 64cdeeefadb06cd09f7b1534cbb8f61daf93b71b Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 26 Jan 2021 13:06:22 +0100 Subject: [PATCH 14/19] Fix flake --- esmvalcore/preprocessor/_time.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index c1131ed7bb..ff63b8d0c3 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -776,6 +776,7 @@ def _get_next_month(month, year): return month + 1, year return 1, year + 1 + def low_pass_weights(window, cutoff): """Calculate weights for a low pass Lanczos filter. From 85151f22281ad70e345cb43affe606fae5e481db Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Thu, 28 Jan 2021 16:56:39 +0100 Subject: [PATCH 15/19] Update esmvalcore/preprocessor/_time.py Co-authored-by: Peter Kalverla --- esmvalcore/preprocessor/_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index ff63b8d0c3..8fefe37539 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -743,11 +743,11 @@ def _get_time_bounds(time, freq): month = time.cell(step).point.month year = time.cell(step).point.year if freq in ['mon', 'mo']: - next_month = _get_next_month(month, year) + next_month, next_year = _get_next_month(month, year) min_bound = time.units.date2num( datetime.datetime(year, month, 1, 0, 0)) max_bound = time.units.date2num( - datetime.datetime(next_month[1], next_month[0], 1, 0, 0)) + datetime.datetime(next_year, next_month, 1, 0, 0)) elif freq == 'yr': min_bound = time.units.date2num( datetime.datetime(year, 1, 1, 0, 0)) From f2b65bc94688e459bfb3b67061a1262e8f110664 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 29 Jan 2021 14:41:01 +0100 Subject: [PATCH 16/19] Add warning for missing bounds --- esmvalcore/cmor/check.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index d0db299538..bce2c9d031 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -510,6 +510,10 @@ def _check_time_bounds(self, freq, time): self.report_warning( 'Added guessed bounds to coordinate {0} from var {1}', time.var_name, self._cmor_var.short_name) + else: + self.report_warning( + 'Coordinate {0} from var {1} does not have bounds', + time.var_name, self._cmor_var.short_name) def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" From c0d84d2ae511b1d6ab5dfef9dc17196d3f1157b0 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 1 Feb 2021 09:50:42 +0100 Subject: [PATCH 17/19] Improve tests --- tests/unit/cmor/test_cmor_check.py | 73 ++++------------------ tests/unit/preprocessor/_time/test_time.py | 58 +++++++++++++++++ 2 files changed, 70 insertions(+), 61 deletions(-) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 6cd392ca29..f47a0a2ad5 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -921,67 +921,18 @@ def test_frequency_not_supported(self): """Fail at metadata if frequency is not supported.""" self._check_fails_in_metadata(frequency='wrong_freq') - def test_hourly_time_bounds(self): - """Test time bounds are properly guessed for hourly freq""" - cube = self.get_cube(self.var_info, frequency='hr') - original_bounds = cube.coord('time').bounds - cube.coord('time').bounds = None - self.cube = cube - self._check_cube(automatic_fixes=True, frequency='hr') - guessed_bounds = self.cube.coord('time').bounds - assert original_bounds.all() == guessed_bounds.all() - - def test_3hourly_time_bounds(self): - """Test time bounds are properly guessed for 3hourly freq""" - cube = self.get_cube(self.var_info, frequency='3hr') - original_bounds = self.cube.coord('time').bounds - cube.coord('time').bounds = None - self.cube = cube - self._check_cube(automatic_fixes=True, frequency='3hr') - guessed_bounds = self.cube.coord('time').bounds - assert original_bounds.all() == guessed_bounds.all() - - def test_6hourly_time_bounds(self): - """Test time bounds are properly guessed for 6hourly freq""" - cube = self.get_cube(self.var_info, frequency='6hr') - original_bounds = self.cube.coord('time').bounds - cube.coord('time').bounds = None - self.cube = cube - self._check_cube(automatic_fixes=True, frequency='6hr') - guessed_bounds = self.cube.coord('time').bounds - assert original_bounds.all() == guessed_bounds.all() - - def test_daily_time_bounds(self): - """Test time bounds are properly guessed for daily freq""" - original_bounds = self.cube.coord('time').bounds - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) - guessed_bounds = self.cube.coord('time').bounds - assert original_bounds.all() == guessed_bounds.all() - - def test_monthly_time_bounds(self): - """Test time bounds are guessed for monthly freq""" - cube = self.get_cube(self.var_info, frequency='mon') - cube.coord('time').bounds = None - self.cube = cube - self._check_cube(automatic_fixes=True, frequency='mon') - assert self.cube.coord('time').bounds is not None - - def test_yearly_time_bounds(self): - """Test time bounds are guessed for yearly freq""" - cube = self.get_cube(self.var_info, frequency='yr') - cube.coord('time').bounds = None - self.cube = cube - self._check_cube(automatic_fixes=True, frequency='yr') - assert self.cube.coord('time').bounds is not None - - def test_dec_time_bounds(self): - """Test time bounds are guessed for dec freq""" - cube = self.get_cube(self.var_info, frequency='dec') - cube.coord('time').bounds = None - self.cube = cube - self._check_cube(automatic_fixes=True, frequency='dec') - assert self.cube.coord('time').bounds is not None + def test_time_bounds(self): + """Test time bounds are guessed for all frequencies""" + freqs = ['hr', '3hr', '6hr', 'day', 'mon', 'yr', 'dec'] + guessed = [] + for freq in freqs: + cube = self.get_cube(self.var_info, frequency=freq) + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency=freq) + if cube.coord('time').bounds is not None: + guessed.append(True) + assert len(guessed) == len(freqs) def test_no_time_bounds(self): """Test time bounds are not guessed for instantaneous data""" diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index e40054b18d..1caba16e45 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -8,6 +8,7 @@ import iris.coords import numpy as np import pytest +import datetime from cf_units import Unit from iris.cube import Cube from numpy.testing import assert_array_almost_equal, assert_array_equal @@ -764,6 +765,17 @@ def test_regrid_time_year(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 assert_array_equal(diff_cube.data, expected) + # test bounds are set at [01-01-YEAR 00:00, 01-01-NEXT_YEAR 00:00] + timeunit_1 = newcube_1.coord('time').units + for i, time in enumerate(newcube_1.coord('time').points): + year_1 = timeunit_1.num2date(time).year + expected_minbound = timeunit_1.date2num( + datetime.datetime(year_1, 1, 1)) + expected_maxbound = timeunit_1.date2num( + datetime.datetime(year_1+1, 1, 1)) + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTimeMonthly(tests.Test): @@ -800,6 +812,24 @@ def test_regrid_time_mon(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 assert_array_equal(diff_cube.data, expected) + # test bounds are set at + # [01-MONTH-YEAR 00:00, 01-NEXT_MONTH-YEAR 00:00] + timeunit_1 = newcube_1.coord('time').units + for i, time in enumerate(newcube_1.coord('time').points): + month_1 = timeunit_1.num2date(time).month + year_1 = timeunit_1.num2date(time).year + next_month = month_1 + 1 + next_year = year_1 + if month_1 == 12: + next_month = 1 + next_year += 1 + expected_minbound = timeunit_1.date2num( + datetime.datetime(year_1, month_1, 1)) + expected_maxbound = timeunit_1.date2num( + datetime.datetime(next_year, next_month, 1)) + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTimeDaily(tests.Test): @@ -846,6 +876,13 @@ def test_regrid_time_day(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 12/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 12/24 + expected_maxbound = time + 12/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTime6Hourly(tests.Test): @@ -892,6 +929,13 @@ def test_regrid_time_6hour(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 3/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 3/24 + expected_maxbound = time + 3/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTime3Hourly(tests.Test): @@ -938,6 +982,13 @@ def test_regrid_time_3hour(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 1.5/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 1.5/24 + expected_maxbound = time + 1.5/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTime1Hourly(tests.Test): @@ -984,6 +1035,13 @@ def test_regrid_time_hour(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 0.5/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 0.5/24 + expected_maxbound = time + 0.5/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestTimeseriesFilter(tests.Test): From 88ef9fb255b3bed6bc522cbaeb50b3712c26626d Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 11 Feb 2021 17:25:49 +0100 Subject: [PATCH 18/19] Add test for differing calendars --- tests/unit/preprocessor/_time/test_time.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index 1caba16e45..8c980700a1 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -35,13 +35,13 @@ ) -def _create_sample_cube(): +def _create_sample_cube(calendar='gregorian'): cube = Cube(np.arange(1, 25), var_name='co2', units='J') cube.add_dim_coord( iris.coords.DimCoord( np.arange(15., 720., 30.), standard_name='time', - units=Unit('days since 1950-01-01 00:00:00', calendar='gregorian'), + units=Unit('days since 1950-01-01 00:00:00', calendar=calendar), ), 0, ) @@ -831,6 +831,19 @@ def test_regrid_time_mon(self): newcube_1.coord('time').bounds[i], np.array([expected_minbound, expected_maxbound])) + def test_regrid_time_different_calendar_bounds(self): + """Test bounds in different calendars.""" + cube_360 = _create_sample_cube(calendar='360_day') + # Same cubes but differing time units + newcube_360 = regrid_time(cube_360, frequency='mon') + newcube_gregorian = regrid_time(self.cube_1, frequency='mon') + bounds_360 = newcube_360.coord('time').bounds + bounds_gregorian = newcube_gregorian.coord('time').bounds + # test value of the bounds is not the same + assert (bounds_360 != bounds_gregorian).any() + # assert length of the 360_day bounds interval is 30 days + assert (bounds_360[:, 1] - bounds_360[:, 0] == 30).all() + class TestRegridTimeDaily(tests.Test): """Tests for regrid_time with daily frequency.""" From fc31631cf3ad9ab1942f6a7a49d39c166b053825 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 1 Mar 2021 11:09:27 +0100 Subject: [PATCH 19/19] Move imports to right place --- esmvalcore/cmor/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 7eb5a1621e..2e10b970c6 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -9,6 +9,7 @@ import iris.util import numpy as np +from esmvalcore.preprocessor._time import _get_time_bounds from .table import CMOR_TABLES CheckLevels = IntEnum('CheckLevels', 'DEBUG STRICT DEFAULT RELAXED IGNORE') @@ -508,7 +509,6 @@ def _check_coord_bounds(self, cmor, coord, var_name): coord.var_name, var_name) def _check_time_bounds(self, freq, time): - from esmvalcore.preprocessor._time import _get_time_bounds times = {'time', 'time1', 'time2', 'time3'} key = times.intersection(self._cmor_var.coordinates) cmor = self._cmor_var.coordinates[" ".join(key)]