diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index e72877e27c..2e10b970c6 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -9,6 +9,7 @@ import iris.util import numpy as np +from esmvalcore.preprocessor._time import _get_time_bounds from .table import CMOR_TABLES CheckLevels = IntEnum('CheckLevels', 'DEBUG STRICT DEFAULT RELAXED IGNORE') @@ -507,6 +508,21 @@ def _check_coord_bounds(self, cmor, coord, var_name): 'Coordinate {0} from var {1} does not have bounds', coord.var_name, var_name) + def _check_time_bounds(self, freq, time): + times = {'time', 'time1', 'time2', 'time3'} + key = times.intersection(self._cmor_var.coordinates) + cmor = self._cmor_var.coordinates[" ".join(key)] + if cmor.must_have_bounds == 'yes' and not time.has_bounds(): + if self.automatic_fixes: + time.bounds = _get_time_bounds(time, freq) + self.report_warning( + 'Added guessed bounds to coordinate {0} from var {1}', + time.var_name, self._cmor_var.short_name) + else: + self.report_warning( + 'Coordinate {0} from var {1} does not have bounds', + time.var_name, self._cmor_var.short_name) + def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" if coord.ndim > 1: @@ -658,9 +674,7 @@ def _check_time_coord(self): calendar=coord.units.calendar)) simplified_cal = self._simplify_calendar(coord.units.calendar) coord.units = cf_units.Unit(coord.units.origin, simplified_cal) - attrs = self._cube.attributes - parent_time = 'parent_time_units' if parent_time in attrs: if attrs[parent_time] in 'no parent': @@ -720,6 +734,8 @@ def _check_time_coord(self): interval = intervals[freq] target_interval = (interval[0] - tol, interval[1] + tol) elif freq.endswith('hr'): + if freq == 'hr': + freq = '1hr' frequency = freq[:-2] if frequency == 'sub': frequency = 1.0 / 24 @@ -738,7 +754,7 @@ def _check_time_coord(self): msg = '{}: Frequency {} does not match input data' self.report_error(msg, var_name, freq) break - + self._check_time_bounds(freq, coord) # remove time_origin from attributes coord.attributes.pop('time_origin', None) @@ -782,7 +798,7 @@ def has_debug_messages(self): return len(self._debug_messages) > 0 def report(self, level, message, *args): - """Generic method to report a message from the checker. + """Report a message from the checker. Parameters ---------- diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 854ab35eea..06b810b13e 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -725,7 +725,7 @@ def regrid_time(cube, frequency): # uniformize bounds cube.coord('time').bounds = None - cube.coord('time').guess_bounds() + cube.coord('time').bounds = _get_time_bounds(cube.coord('time'), frequency) # remove aux coords that will differ reset_aux = ['day_of_month', 'day_of_year'] @@ -744,6 +744,46 @@ def regrid_time(cube, frequency): return cube +def _get_time_bounds(time, freq): + bounds = [] + for step, point in enumerate(time.points): + month = time.cell(step).point.month + year = time.cell(step).point.year + if freq in ['mon', 'mo']: + next_month, next_year = _get_next_month(month, year) + min_bound = time.units.date2num( + datetime.datetime(year, month, 1, 0, 0)) + max_bound = time.units.date2num( + datetime.datetime(next_year, next_month, 1, 0, 0)) + elif freq == 'yr': + min_bound = time.units.date2num( + datetime.datetime(year, 1, 1, 0, 0)) + max_bound = time.units.date2num( + datetime.datetime(year + 1, 1, 1, 0, 0)) + elif freq == 'dec': + min_bound = time.units.date2num( + datetime.datetime(year, 1, 1, 0, 0)) + max_bound = time.units.date2num( + datetime.datetime(year + 10, 1, 1, 0, 0)) + else: + delta = { + 'day': 12 / 24, + '6hr': 3 / 24, + '3hr': 1.5 / 24, + '1hr': 0.5 / 24, + } + min_bound = point - delta[freq] + max_bound = point + delta[freq] + bounds.append([min_bound, max_bound]) + return np.array(bounds) + + +def _get_next_month(month, year): + if month != 12: + return month + 1, year + return 1, year + 1 + + def low_pass_weights(window, cutoff): """Calculate weights for a low pass Lanczos filter. diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 3ea9241822..504eaf06f1 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -385,8 +385,9 @@ def test_check_missing_time_strict_flag(self): def test_check_missing_coord_strict_flag(self): """Test check fails for missing coord other than lat and lon with --cmor-check strict""" - self.var_info.coordinates = {'height2m': CoordinateInfoMock('height2m') - } + self.var_info.coordinates.update( + {'height2m': CoordinateInfoMock('height2m')} + ) self._check_fails_in_metadata(automatic_fixes=False) def test_check_bad_var_standard_name_relaxed_flag(self): @@ -461,8 +462,9 @@ def test_check_missing_time_relaxed_flag(self): def test_check_missing_coord_relaxed_flag(self): """Test check reports warning for missing coord other than lat and lon with --cmor-check relaxed""" - self.var_info.coordinates = {'height2m': CoordinateInfoMock('height2m') - } + self.var_info.coordinates.update( + {'height2m': CoordinateInfoMock('height2m')} + ) self._check_warnings_on_metadata(automatic_fixes=False, check_level=CheckLevels.RELAXED) @@ -541,8 +543,9 @@ def test_check_missing_time_none_flag(self): def test_check_missing_coord_none_flag(self): """Test check reports warning for missing coord other than lat, lon and time with --cmor-check ignore""" - self.var_info.coordinates = {'height2m': CoordinateInfoMock('height2m') - } + self.var_info.coordinates.update( + {'height2m': CoordinateInfoMock('height2m')} + ) self._check_warnings_on_metadata(automatic_fixes=False, check_level=CheckLevels.IGNORE) @@ -932,6 +935,29 @@ def test_frequency_not_supported(self): """Fail at metadata if frequency is not supported.""" self._check_fails_in_metadata(frequency='wrong_freq') + def test_time_bounds(self): + """Test time bounds are guessed for all frequencies""" + freqs = ['hr', '3hr', '6hr', 'day', 'mon', 'yr', 'dec'] + guessed = [] + for freq in freqs: + cube = self.get_cube(self.var_info, frequency=freq) + cube.coord('time').bounds = None + self.cube = cube + self._check_cube(automatic_fixes=True, frequency=freq) + if cube.coord('time').bounds is not None: + guessed.append(True) + assert len(guessed) == len(freqs) + + def test_no_time_bounds(self): + """Test time bounds are not guessed for instantaneous data""" + self.var_info.coordinates['time'].must_have_bounds = 'no' + self.var_info.coordinates['time1'] = ( + self.var_info.coordinates.pop('time')) + self.cube.coord('time').bounds = None + self._check_cube(automatic_fixes=True) + guessed_bounds = self.cube.coord('time').bounds + assert guessed_bounds is None + def _check_fails_on_data(self): checker = CMORCheck(self.cube, self.var_info) checker.check_metadata() @@ -1130,7 +1156,13 @@ def _get_time_values(dim_spec): delta = 30 elif frequency == 'day': delta = 1 - elif frequency.ends_with('hr'): + elif frequency == 'yr': + delta = 360 + elif frequency == 'dec': + delta = 3600 + elif frequency.endswith('hr'): + if frequency == 'hr': + frequency = '1hr' delta = float(frequency[:-2]) / 24 else: raise Exception('Frequency {} not supported'.format(frequency)) diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index d529eae94d..5096ae1fb4 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -9,6 +9,7 @@ import iris.exceptions import numpy as np import pytest +import datetime from cf_units import Unit from iris.cube import Cube from numpy.testing import ( @@ -39,13 +40,13 @@ ) -def _create_sample_cube(): +def _create_sample_cube(calendar='gregorian'): cube = Cube(np.arange(1, 25), var_name='co2', units='J') cube.add_dim_coord( iris.coords.DimCoord( np.arange(15., 720., 30.), standard_name='time', - units=Unit('days since 1950-01-01 00:00:00', calendar='gregorian'), + units=Unit('days since 1950-01-01 00:00:00', calendar=calendar), ), 0, ) @@ -798,6 +799,17 @@ def test_regrid_time_year(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 assert_array_equal(diff_cube.data, expected) + # test bounds are set at [01-01-YEAR 00:00, 01-01-NEXT_YEAR 00:00] + timeunit_1 = newcube_1.coord('time').units + for i, time in enumerate(newcube_1.coord('time').points): + year_1 = timeunit_1.num2date(time).year + expected_minbound = timeunit_1.date2num( + datetime.datetime(year_1, 1, 1)) + expected_maxbound = timeunit_1.date2num( + datetime.datetime(year_1+1, 1, 1)) + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTimeMonthly(tests.Test): @@ -834,6 +846,37 @@ def test_regrid_time_mon(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 assert_array_equal(diff_cube.data, expected) + # test bounds are set at + # [01-MONTH-YEAR 00:00, 01-NEXT_MONTH-YEAR 00:00] + timeunit_1 = newcube_1.coord('time').units + for i, time in enumerate(newcube_1.coord('time').points): + month_1 = timeunit_1.num2date(time).month + year_1 = timeunit_1.num2date(time).year + next_month = month_1 + 1 + next_year = year_1 + if month_1 == 12: + next_month = 1 + next_year += 1 + expected_minbound = timeunit_1.date2num( + datetime.datetime(year_1, month_1, 1)) + expected_maxbound = timeunit_1.date2num( + datetime.datetime(next_year, next_month, 1)) + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) + + def test_regrid_time_different_calendar_bounds(self): + """Test bounds in different calendars.""" + cube_360 = _create_sample_cube(calendar='360_day') + # Same cubes but differing time units + newcube_360 = regrid_time(cube_360, frequency='mon') + newcube_gregorian = regrid_time(self.cube_1, frequency='mon') + bounds_360 = newcube_360.coord('time').bounds + bounds_gregorian = newcube_gregorian.coord('time').bounds + # test value of the bounds is not the same + assert (bounds_360 != bounds_gregorian).any() + # assert length of the 360_day bounds interval is 30 days + assert (bounds_360[:, 1] - bounds_360[:, 0] == 30).all() class TestRegridTimeDaily(tests.Test): @@ -880,6 +923,13 @@ def test_regrid_time_day(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 12/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 12/24 + expected_maxbound = time + 12/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTime6Hourly(tests.Test): @@ -926,6 +976,13 @@ def test_regrid_time_6hour(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 3/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 3/24 + expected_maxbound = time + 3/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTime3Hourly(tests.Test): @@ -972,6 +1029,13 @@ def test_regrid_time_3hour(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 1.5/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 1.5/24 + expected_maxbound = time + 1.5/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestRegridTime1Hourly(tests.Test): @@ -1018,6 +1082,13 @@ def test_regrid_time_hour(self): expected = self.cube_1.data diff_cube = newcube_2 - newcube_1 self.assert_array_equal(diff_cube.data, expected) + # test bounds are set with a dt = 0.5/24 days + for i, time in enumerate(newcube_1.coord('time').points): + expected_minbound = time - 0.5/24 + expected_maxbound = time + 0.5/24 + assert_array_equal( + newcube_1.coord('time').bounds[i], + np.array([expected_minbound, expected_maxbound])) class TestTimeseriesFilter(tests.Test):