diff --git a/doc/esmvalcore/fixing_data.rst b/doc/esmvalcore/fixing_data.rst index 3448c16ff1..2b9f0f30bc 100644 --- a/doc/esmvalcore/fixing_data.rst +++ b/doc/esmvalcore/fixing_data.rst @@ -16,6 +16,15 @@ coordinate bounds like ''lat_bnds'') or problems with the actual data The ESMValTool can apply on the fly fixes to data sets that have known errors that can be fixed automatically. +.. note:: + **CMORization as a fix**. As of early 2020, we've started implementing CMORization as fixes for + observational datasets. Previously, CMORization was an additional function implemented in ESMValTool. + This meant that users always had to store 2 copies of their observational data: both raw and CMORized. + Implementing CMORization as a fix removes this redundancy, as the fixes are applied 'on the fly' when + running a recipe. **ERA5** is the first dataset for which this 'CMORization on the fly' is supported. + For more information about CMORization, see: + `Contributing a CMORizing script for an observational dataset `_. + Fix structure ============= @@ -44,7 +53,7 @@ Check the output Next to the error message, you should see some info about the iris cube: size, coordinates. In our example it looks like this: -.. code-block:: +.. code-block:: python air_temperature/ (K) (time: 312; altitude: 90; longitude: 180) Dimension coordinates: @@ -202,16 +211,28 @@ with the actual data units. .. code-block:: python - ... - def fix_metadata(self, cubes): - ... - cube.units = 'real_units' - ... + def fix_metadata(self, cubes): + cube.units = 'real_units' + Detecting this error can be tricky if the units are similar enough. It also has a good chance of going undetected until you notice strange results in your diagnostic. +For the above example, it can be useful to access the variable definition +and associated coordinate definitions as provided by the CMOR table. +For example: + +.. code-block:: python + + def fix_metadata(self, cubes): + cube.units = self.vardef.units + +To learn more about what is available in these definitions, see: +:class:`esmvalcore.cmor.table.VariableInfo` and +:class:`esmvalcore.cmor.table.CoordinateInfo`. + + Coordinates missing ------------------- diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index 7b39127995..7a27dee4f6 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -5,10 +5,11 @@ # Mattia Righi (DLR, Germany - mattia.righi@dlr.de) import fnmatch +import glob import logging import os import re -import glob +from pathlib import Path import iris @@ -32,56 +33,30 @@ def find_files(dirnames, filenames): def get_start_end_year(filename): - """Get the start and end year from a file name. - - This works for filenames matching - - *[-,_]YYYY*[-,_]YYYY*.* - or - *[-,_]YYYY*.* - or - YYYY*[-,_]*.* - or - YYYY*[-,_]YYYY*[-,_]*.* - or - YYYY*[-,_]*[-,_]YYYY*.* (Does this make sense? Is this worth catching?) - """ - name = os.path.splitext(filename)[0] - - name = name.split(os.sep)[-1] - filename_list = [elem.split('-') for elem in name.split('_')] - filename_list = [elem for sublist in filename_list for elem in sublist] - - pos_ydates = [elem.isdigit() and len(elem) >= 4 for elem in filename_list] - pos_ydates_l = list(pos_ydates) - pos_ydates_r = list(pos_ydates) - - for ind, _ in enumerate(pos_ydates_l): - if ind != 0: - pos_ydates_l[ind] = (pos_ydates_l[ind - 1] and pos_ydates_l[ind]) - - for ind, _ in enumerate(pos_ydates_r): - if ind != 0: - pos_ydates_r[-ind - 1] = (pos_ydates_r[-ind] - and pos_ydates_r[-ind - 1]) - - dates = [ - filename_list[ind] for ind, _ in enumerate(pos_ydates) - if pos_ydates_r[ind] or pos_ydates_l[ind] - ] - start_year = None - end_year = None - if len(dates) == 1: - start_year = int(dates[0][:4]) - end_year = start_year - elif len(dates) == 2: - start_year, end_year = int(dates[0][:4]), int(dates[1][:4]) + """Get the start and end year from a file name.""" + stem = Path(filename).stem + start_year = end_year = None + + # First check for a block of two potential dates separated by _ or - + daterange = re.findall(r'([0-9]{4,12}[-_][0-9]{4,12})', stem) + if daterange: + start_date, end_date = re.findall(r'([0-9]{4,12})', daterange[0]) + start_year = start_date[:4] + end_year = end_date[:4] else: - # Slower than just parsing the name - try: - cubes = iris.load(filename) - except OSError: - raise ValueError('File {0} can not be read'.format(filename)) + # Check for single dates in the filename + dates = re.findall(r'([0-9]{4,12})', stem) + if len(dates) == 1: + start_year = end_year = dates[0][:4] + elif len(dates) > 1: + # Check for dates at start or end of filename + outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$', stem) + if len(outerdates) == 1: + start_year = end_year = outerdates[0][:4] + + # As final resort, try to get the dates from the file contents + if start_year is None or end_year is None: + cubes = iris.load(filename) for cube in cubes: logger.debug(cube) @@ -94,11 +69,11 @@ def get_start_end_year(filename): break if start_year is None or end_year is None: - raise ValueError( - 'File {0} dates do not match a recognized pattern and time can ' - 'not be read from the file'.format(filename) - ) - return start_year, end_year + raise ValueError(f'File {filename} dates do not match a recognized' + 'pattern and time can not be read from the file') + + logger.debug("Found start_year %s and end_year %s", start_year, end_year) + return int(start_year), int(end_year) def select_files(filenames, start_year, end_year): diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 7f20c5ace1..2bcf759634 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -88,40 +88,26 @@ def _get_value(key, datasets): return value -def _update_from_others(variable, keys, datasets): - """Get values for keys by copying from the other datasets.""" - for key in keys: - if key not in variable: - value = _get_value(key, datasets) - if value is not None: - variable[key] = value - - def _add_cmor_info(variable, override=False): """Add information from CMOR tables to variable.""" logger.debug("If not present: adding keys from CMOR table to %s", variable) - - if 'cmor_table' not in variable or 'mip' not in variable: - logger.debug("Skipping because cmor_table or mip not specified") - return - - if variable['cmor_table'] not in CMOR_TABLES: - logger.warning("Unknown CMOR table %s", variable['cmor_table']) - - derive = variable.get('derive', False) # Copy the following keys from CMOR table cmor_keys = [ 'standard_name', 'long_name', 'units', 'modeling_realm', 'frequency' ] - cmor_table = variable['cmor_table'] + project = variable['project'] mip = variable['mip'] short_name = variable['short_name'] - table_entry = CMOR_TABLES[cmor_table].get_variable(mip, short_name, derive) - + derive = variable.get('derive', False) + table = CMOR_TABLES.get(project) + if table: + table_entry = table.get_variable(mip, short_name, derive) + else: + table_entry = None if table_entry is None: raise RecipeError( - "Unable to load CMOR table '{}' for variable '{}' with mip '{}'". - format(cmor_table, short_name, mip)) + f"Unable to load CMOR table (project) '{project}' for variable " + f"'{short_name}' with mip '{mip}'") for key in cmor_keys: if key not in variable or override: @@ -180,12 +166,17 @@ def _update_target_levels(variable, variables, settings, config_user): del settings['extract_levels'] else: variable_data = _get_dataset_info(dataset, variables) - filename = \ - _dataset_to_file(variable_data, config_user) + filename = _dataset_to_file(variable_data, config_user) settings['extract_levels']['levels'] = get_reference_levels( - filename, variable_data['project'], dataset, - variable_data['short_name'], - os.path.splitext(variable_data['filename'])[0] + '_fixed') + filename=filename, + project=variable_data['project'], + dataset=dataset, + short_name=variable_data['short_name'], + mip=variable_data['mip'], + frequency=variable_data['frequency'], + fix_dir=os.path.splitext( + variable_data['filename'])[0] + '_fixed', + ) def _update_target_grid(variable, variables, settings, config_user): @@ -300,19 +291,16 @@ def _get_default_settings(variable, config_user, derive=False): 'project': variable['project'], 'dataset': variable['dataset'], 'short_name': variable['short_name'], + 'mip': variable['mip'], } # File fixes fix_dir = os.path.splitext(variable['filename'])[0] + '_fixed' settings['fix_file'] = dict(fix) settings['fix_file']['output_dir'] = fix_dir # Cube fixes - # Only supply mip if the CMOR check fixes are implemented. - if variable.get('cmor_table'): - fix['cmor_table'] = variable['cmor_table'] - fix['mip'] = variable['mip'] - fix['frequency'] = variable['frequency'] - settings['fix_data'] = dict(fix) + fix['frequency'] = variable['frequency'] settings['fix_metadata'] = dict(fix) + settings['fix_data'] = dict(fix) # Configure time extraction if 'start_year' in variable and 'end_year' in variable \ @@ -335,21 +323,19 @@ def _get_default_settings(variable, config_user, derive=False): } # Configure CMOR metadata check - if variable.get('cmor_table'): - settings['cmor_check_metadata'] = { - 'cmor_table': variable['cmor_table'], - 'mip': variable['mip'], - 'short_name': variable['short_name'], - 'frequency': variable['frequency'], - } + settings['cmor_check_metadata'] = { + 'cmor_table': variable['project'], + 'mip': variable['mip'], + 'short_name': variable['short_name'], + 'frequency': variable['frequency'], + } # Configure final CMOR data check - if variable.get('cmor_table'): - settings['cmor_check_data'] = { - 'cmor_table': variable['cmor_table'], - 'mip': variable['mip'], - 'short_name': variable['short_name'], - 'frequency': variable['frequency'], - } + settings['cmor_check_data'] = { + 'cmor_table': variable['project'], + 'mip': variable['mip'], + 'short_name': variable['short_name'], + 'frequency': variable['frequency'], + } # Clean up fixed files if not config_user['save_intermediary_cubes']: @@ -1029,9 +1015,6 @@ def _initialize_variables(self, raw_variable, raw_datasets): variable.update(dataset) variable['recipe_dataset_index'] = index - if ('cmor_table' not in variable - and variable.get('project') in CMOR_TABLES): - variable['cmor_table'] = variable['project'] if 'end_year' in variable and 'max_years' in self._cfg: variable['end_year'] = min( variable['end_year'], @@ -1049,7 +1032,6 @@ def _initialize_variables(self, raw_variable, raw_datasets): if 'fx' not in raw_variable.get('mip', ''): required_keys.update({'start_year', 'end_year'}) for variable in variables: - _update_from_others(variable, ['cmor_table', 'mip'], datasets) if 'institute' not in variable: institute = get_institutes(variable) if institute: diff --git a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py index 9efeac6111..8d59f16510 100644 --- a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py +++ b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py @@ -121,7 +121,8 @@ def fix_metadata(self, cubes): ------- iris.cube.CubeList """ - cubes[0].standard_name = 'sea_ice_x_velocity' + cube = self.get_cube_from_list(cubes) + cube.standard_name = 'sea_ice_x_velocity' return cubes @@ -141,5 +142,6 @@ def fix_metadata(self, cubes): ------- iris.cube.CubeList """ - cubes[0].standard_name = 'sea_ice_y_velocity' + cube = self.get_cube_from_list(cubes) + cube.standard_name = 'sea_ice_y_velocity' return cubes diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 2e59000777..db7effa439 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -3,11 +3,23 @@ import os import inspect +from ..table import CMOR_TABLES -class Fix(object): + +class Fix: """ Base class for dataset fixes. """ + def __init__(self, vardef): + """Initialize fix object. + + Parameters + ---------- + vardef: basestring + CMOR table entry + + """ + self.vardef = vardef def fix_file(self, filepath, output_dir): """ @@ -109,7 +121,7 @@ def __ne__(self, other): return not self.__eq__(other) @staticmethod - def get_fixes(project, dataset, variable): + def get_fixes(project, dataset, mip, short_name): """ Get the fixes that must be applied for a given dataset. @@ -128,16 +140,20 @@ def get_fixes(project, dataset, variable): ---------- project: str dataset: str - variable: str + mip: str + short_name: str Returns ------- list(Fix) Fixes to apply for the given data """ + cmor_table = CMOR_TABLES[project] + vardef = cmor_table.get_variable(mip, short_name) + project = project.replace('-', '_').lower() dataset = dataset.replace('-', '_').lower() - variable = variable.replace('-', '_').lower() + short_name = short_name.replace('-', '_').lower() fixes = [] try: @@ -146,9 +162,9 @@ def get_fixes(project, dataset, variable): classes = inspect.getmembers(fixes_module, inspect.isclass) classes = dict((name.lower(), value) for name, value in classes) - for fix_name in ('allvars', variable): + for fix_name in (short_name, 'allvars'): try: - fixes.append(classes[fix_name]()) + fixes.append(classes[fix_name](vardef)) except KeyError: pass except ImportError: diff --git a/esmvalcore/cmor/_fixes/native6/__init__.py b/esmvalcore/cmor/_fixes/native6/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py new file mode 100644 index 0000000000..1f569db4dc --- /dev/null +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -0,0 +1,327 @@ +"""Fixes for ERA5.""" +import datetime +import logging + +import iris +import numpy as np + +from ..fix import Fix +from ..shared import add_scalar_height_coord + +logger = logging.getLogger(__name__) + + +def get_frequency(cube): + """Determine time frequency of input cube.""" + try: + time = cube.coord(axis='T') + except iris.exceptions.CoordinateNotFoundError: + return 'fx' + + time.convert_units('days since 1850-1-1 00:00:00.0') + if len(time.points) == 1: + if cube.long_name != 'Geopotential': + raise ValueError('Unable to infer frequency of cube ' + f'with length 1 time dimension: {cube}') + return 'fx' + + interval = time.points[1] - time.points[0] + if interval - 1 / 24 < 1e-4: + return 'hourly' + + return 'monthly' + + +def fix_hourly_time_coordinate(cube): + """Shift aggregated variables 30 minutes back in time.""" + if get_frequency(cube) == 'hourly': + time = cube.coord(axis='T') + time.points = time.points - 1 / 48 + return cube + + +def fix_accumulated_units(cube): + """Convert accumulations to fluxes.""" + if get_frequency(cube) == 'monthly': + cube.units = cube.units * 'd-1' + elif get_frequency(cube) == 'hourly': + cube.units = cube.units * 'h-1' + return cube + + +def multiply_with_density(cube, density=1000): + """Convert precipitatin from m to kg/m2.""" + cube.data = cube.core_data() * density + cube.units *= 'kg m**-3' + return cube + + +def remove_time_coordinate(cube): + """Remove time coordinate for invariant parameters.""" + cube = cube[0] + cube.remove_coord('time') + return cube + + +def divide_by_gravity(cube): + """Convert geopotential to height.""" + cube.units = cube.units / 'm s-2' + cube.data = cube.core_data() / 9.80665 + return cube + + +class Clt(Fix): + """Fixes for clt.""" + def fix_metadata(self, cubes): + for cube in cubes: + # Invalid input cube units (ignored on load) were '0-1' + cube.units = '%' + cube.data = cube.core_data()*100. + + return cubes + + +class Evspsbl(Fix): + """Fixes for evspsbl.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + # Set input cube units for invalid units were ignored on load + cube.units = 'm' + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + multiply_with_density(cube) + + return cubes + + +class Evspsblpot(Fix): + """Fixes for evspsblpot.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + # Set input cube units for invalid units were ignored on load + cube.units = 'm' + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + multiply_with_density(cube) + + return cubes + + +class Mrro(Fix): + """Fixes for mrro.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + multiply_with_density(cube) + + return cubes + + +class Orog(Fix): + """Fixes for orography.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + fixed_cubes = [] + for cube in cubes: + cube = remove_time_coordinate(cube) + divide_by_gravity(cube) + fixed_cubes.append(cube) + return iris.cube.CubeList(fixed_cubes) + + +class Pr(Fix): + """Fixes for pr.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + multiply_with_density(cube) + + return cubes + + +class Prsn(Fix): + """Fixes for prsn.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + # Set input cube units for invalid units were ignored on load + cube.units = 'm' + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + multiply_with_density(cube) + + return cubes + + +class Ptype(Fix): + """Fixes for ptype.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 1 + + return cubes + + +class Rlds(Fix): + """Fixes for Rlds.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + cube.attributes['positive'] = 'down' + + return cubes + + +class Rls(Fix): + """Fixes for Rls.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + fix_hourly_time_coordinate(cube) + cube.attributes['positive'] = 'down' + + return cubes + + +class Rsds(Fix): + """Fixes for Rsds.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + cube.attributes['positive'] = 'down' + + return cubes + + +class Rsdt(Fix): + """Fixes for Rsdt.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + cube.attributes['positive'] = 'down' + + return cubes + + +class Rss(Fix): + """Fixes for Rss.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + cube.attributes['positive'] = 'down' + + return cubes + + +class Tasmax(Fix): + """Fixes for tasmax.""" + def fix_metadata(self, cubes): + for cube in cubes: + fix_hourly_time_coordinate(cube) + return cubes + + +class Tasmin(Fix): + """Fixes for tasmin.""" + def fix_metadata(self, cubes): + for cube in cubes: + fix_hourly_time_coordinate(cube) + return cubes + + +class AllVars(Fix): + """Fixes for all variables.""" + def _fix_coordinates(self, cube): + """Fix coordinates.""" + # Fix coordinate increasing direction + slices = [] + for coord in cube.coords(): + if coord.var_name in ('latitude', 'pressure_level'): + slices.append(slice(None, None, -1)) + else: + slices.append(slice(None)) + cube = cube[tuple(slices)] + + # Add scalar height coordinates + if 'height2m' in self.vardef.dimensions: + add_scalar_height_coord(cube, 2.) + if 'height10m' in self.vardef.dimensions: + add_scalar_height_coord(cube, 10.) + + for coord_def in self.vardef.coordinates.values(): + axis = coord_def.axis + coord = cube.coord(axis=axis) + if axis == 'T': + coord.convert_units('days since 1850-1-1 00:00:00.0') + if axis == 'Z': + coord.convert_units(coord_def.units) + coord.standard_name = coord_def.standard_name + coord.var_name = coord_def.out_name + coord.long_name = coord_def.long_name + coord.points = coord.core_points().astype('float64') + if (coord.bounds is None and len(coord.points) > 1 + and coord_def.must_have_bounds == "yes"): + coord.guess_bounds() + + self._fix_monthly_time_coord(cube) + + return cube + + @staticmethod + def _fix_monthly_time_coord(cube): + """Set the monthly time coordinates to the middle of the month.""" + if get_frequency(cube) == 'monthly': + coord = cube.coord(axis='T') + end = [] + for cell in coord.cells(): + month = cell.point.month + 1 + year = cell.point.year + if month == 13: + month = 1 + year = year + 1 + end.append(cell.point.replace(month=month, year=year)) + end = coord.units.date2num(end) + start = coord.points + coord.points = 0.5 * (start + end) + coord.bounds = np.column_stack([start, end]) + + def _fix_units(self, cube): + """Fix units.""" + cube.convert_units(self.vardef.units) + + def fix_metadata(self, cubes): + """Fix metadata.""" + fixed_cubes = iris.cube.CubeList() + for cube in cubes: + cube.var_name = self.vardef.short_name + if self.vardef.standard_name: + cube.standard_name = self.vardef.standard_name + cube.long_name = self.vardef.long_name + + cube = self._fix_coordinates(cube) + self._fix_units(cube) + + cube.data = cube.core_data().astype('float32') + year = datetime.datetime.now().year + cube.attributes['comment'] = ( + 'Contains modified Copernicus Climate Change ' + f'Service Information {year}') + + fixed_cubes.append(cube) + + return fixed_cubes diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 2881d49aa7..6ddb13549d 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -def fix_file(file, short_name, project, dataset, output_dir): +def fix_file(file, short_name, project, dataset, mip, output_dir): """ Fix files before ESMValTool can load them. @@ -45,7 +45,7 @@ def fix_file(file, short_name, project, dataset, output_dir): """ for fix in Fix.get_fixes( - project=project, dataset=dataset, variable=short_name): + project=project, dataset=dataset, mip=mip, short_name=short_name): file = fix.fix_file(file, output_dir) return file @@ -54,8 +54,7 @@ def fix_metadata(cubes, short_name, project, dataset, - cmor_table=None, - mip=None, + mip, frequency=None): """ Fix cube metadata if fixes are required and check it anyway. @@ -75,11 +74,8 @@ def fix_metadata(cubes, dataset: str - cmor_table: str, optional - CMOR tables to use for the check, if available - - mip: str, optional - Variable's MIP, if available + mip: str + Variable's MIP frequency: str, optional Variable's data frequency, if available @@ -96,7 +92,7 @@ def fix_metadata(cubes, """ fixes = Fix.get_fixes( - project=project, dataset=dataset, variable=short_name) + project=project, dataset=dataset, mip=mip, short_name=short_name) fixed_cubes = [] by_file = defaultdict(list) for cube in cubes: @@ -134,15 +130,14 @@ def fix_metadata(cubes, else: cube = cube_list[0] - if cmor_table and mip: - checker = _get_cmor_checker( - frequency=frequency, - table=cmor_table, - mip=mip, - short_name=short_name, - fail_on_error=False, - automatic_fixes=True) - cube = checker(cube).check_metadata() + checker = _get_cmor_checker( + frequency=frequency, + table=project, + mip=mip, + short_name=short_name, + fail_on_error=False, + automatic_fixes=True) + cube = checker(cube).check_metadata() cube.attributes.pop('source_file', None) fixed_cubes.append(cube) return fixed_cubes @@ -152,8 +147,7 @@ def fix_data(cube, short_name, project, dataset, - cmor_table=None, - mip=None, + mip, frequency=None): """ Fix cube data if fixes add present and check it anyway. @@ -172,15 +166,9 @@ def fix_data(cube, short_name: str Variable's short name project: str - dataset: str - - cmor_table: str, optional - CMOR tables to use for the check, if available - - mip: str, optional - Variable's MIP, if available - + mip: str + Variable's MIP frequency: str, optional Variable's data frequency, if available @@ -196,15 +184,15 @@ def fix_data(cube, """ for fix in Fix.get_fixes( - project=project, dataset=dataset, variable=short_name): + project=project, dataset=dataset, mip=mip, short_name=short_name): cube = fix.fix_data(cube) - if cmor_table and mip: - checker = _get_cmor_checker( - frequency=frequency, - table=cmor_table, - mip=mip, - short_name=short_name, - fail_on_error=False, - automatic_fixes=True) - cube = checker(cube).check_data() + + checker = _get_cmor_checker( + frequency=frequency, + table=project, + mip=mip, + short_name=short_name, + fail_on_error=False, + automatic_fixes=True) + cube = checker(cube).check_data() return cube diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 34252edb4d..2cedc13a3e 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -11,6 +11,8 @@ import json import logging import os +from pathlib import Path +import yaml logger = logging.getLogger(__name__) @@ -18,7 +20,7 @@ """dict of str, obj: CMOR info objects.""" -def read_cmor_tables(cfg_developer): +def read_cmor_tables(cfg_developer=None): """Read cmor tables required in the configuration. Parameters @@ -27,7 +29,13 @@ def read_cmor_tables(cfg_developer): Parsed config-developer file """ + if cfg_developer is None: + cfg_file = Path(__file__).parent.parent / 'config-developer.yml' + with cfg_file.open() as file: + cfg_developer = yaml.safe_load(file) + custom = CustomInfo() + CMOR_TABLES.clear() CMOR_TABLES['custom'] = custom install_dir = os.path.dirname(os.path.realpath(__file__)) for table in cfg_developer: @@ -41,17 +49,22 @@ def read_cmor_tables(cfg_developer): if cmor_type == 'CMIP3': CMOR_TABLES[table] = CMIP3Info( - table_path, default=custom, strict=cmor_strict, + table_path, + default=custom, + strict=cmor_strict, ) elif cmor_type == 'CMIP5': CMOR_TABLES[table] = CMIP5Info( - table_path, default=custom, strict=cmor_strict, + table_path, + default=custom, + strict=cmor_strict, ) elif cmor_type == 'CMIP6': CMOR_TABLES[table] = CMIP6Info( - table_path, default=custom, strict=cmor_strict, - default_table_prefix=default_table_prefix - ) + table_path, + default=custom, + strict=cmor_strict, + default_table_prefix=default_table_prefix) class CMIP6Info(object): @@ -82,7 +95,10 @@ class CMIP6Info(object): 'vsi': 'siv', } - def __init__(self, cmor_tables_path, default=None, strict=True, + def __init__(self, + cmor_tables_path, + default=None, + strict=True, default_table_prefix=''): cmor_tables_path = self._get_cmor_path(cmor_tables_path) @@ -121,8 +137,7 @@ def _get_cmor_path(cmor_tables_path): if os.path.isdir(cmor_tables_path): return cmor_tables_path raise ValueError( - 'CMOR tables not found in {}'.format(cmor_tables_path) - ) + 'CMOR tables not found in {}'.format(cmor_tables_path)) def _load_table(self, json_file): with open(json_file) as inf: @@ -163,9 +178,8 @@ def _assign_dimensions(self, var, generic_levels): coord = self.coords[dimension] except KeyError: logger.exception( - 'Can not find dimension %s for variable %s', - dimension, var - ) + 'Can not find dimension %s for variable %s', dimension, + var) raise var.coordinates[dimension] = coord @@ -184,8 +198,8 @@ def _load_coordinates(self): def _load_controlled_vocabulary(self): self.activities = {} self.institutes = {} - for json_file in glob.glob( - os.path.join(self._cmor_folder, '*_CV.json')): + for json_file in glob.glob(os.path.join(self._cmor_folder, + '*_CV.json')): with open(json_file) as inf: table_data = json.loads(inf.read()) try: @@ -285,7 +299,6 @@ def _is_table(table_data): @total_ordering class TableInfo(dict): """Container class for storing a CMOR table.""" - def __init__(self, *args, **kwargs): """Create a new TableInfo object for storing VariableInfo objects.""" super(TableInfo, self).__init__(*args, **kwargs) @@ -312,7 +325,6 @@ class JsonInfo(object): Provides common utility methods to read json variables """ - def __init__(self): self._json_data = {} @@ -357,7 +369,6 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" - def __init__(self, table_type, short_name): """ Class to read and store variable information. @@ -444,7 +455,6 @@ def read_json(self, json_data, default_freq): class CoordinateInfo(JsonInfo): """Class to read and store coordinate information.""" - def __init__(self, name): """ Class to read and store coordinate information. @@ -485,7 +495,8 @@ def __init__(self, name): """Minimum allowed value""" self.valid_max = "" """Maximum allowed value""" - + self.must_have_bounds = "" + """Whether bounds are required on this dimension""" def read_json(self, json_data): """ Read coordinate information from json. @@ -512,6 +523,7 @@ def read_json(self, json_data): self.valid_min = self._read_json_variable('valid_min') self.valid_max = self._read_json_variable('valid_max') self.requested = self._read_json_list_variable('requested') + self.must_have_bounds = self._read_json_variable('must_have_bounds') class CMIP5Info(object): @@ -531,7 +543,6 @@ class CMIP5Info(object): found in the requested one """ - def __init__(self, cmor_tables_path, default=None, strict=True): cmor_tables_path = self._get_cmor_path(cmor_tables_path) @@ -634,8 +645,7 @@ def _read_coordinate(self, value): return coord if key == 'requested': coord.requested.extend( - (val for val in value.split(' ') if val) - ) + (val for val in value.split(' ') if val)) continue if hasattr(coord, key): setattr(coord, key, value) @@ -730,9 +740,8 @@ class CMIP3Info(CMIP5Info): found in the requested one """ - def _read_table_file(self, table_file, table=None): - for dim in ('zlevel',): + for dim in ('zlevel', ): coord = CoordinateInfo(dim) coord.generic_level = True coord.axis = 'Z' @@ -764,7 +773,6 @@ class CustomInfo(CMIP5Info): ESMValTool repository """ - def __init__(self, cmor_tables_path=None): cwd = os.path.dirname(os.path.realpath(__file__)) self._cmor_folder = os.path.join(cwd, 'tables', 'custom') @@ -852,3 +860,6 @@ def _read_table_file(self, table_file, table=None): continue if not self._read_line(): return + + +read_cmor_tables() diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index b03e947a43..1cd99565fb 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -188,6 +188,18 @@ OBS6: output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' cmor_type: 'CMIP6' +native6: + cmor_strict: false + input_dir: + default: 'Tier{tier}/{dataset}' + era5cli: 'Tier{tier}/{dataset}' + input_file: + default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc' + era5cli: 'era5_{era5_name}*{era5_freq}.nc' + output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' + cmor_type: 'CMIP6' + cmor_default_table_prefix: 'CMIP6_' + obs4mips: cmor_strict: false input_dir: diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 6a6a74de97..aed7a8d1fe 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -2,6 +2,7 @@ import copy import inspect import logging +from pprint import pformat from iris.cube import Cube @@ -418,7 +419,8 @@ def __str__(self): step for step in self.order if any(step in product.settings for product in self.products) ] - products = '\n\n'.join(str(p) for p in self.products) + products = '\n\n'.join('\n'.join([str(p), pformat(p.settings)]) + for p in self.products) txt = "{}:\norder: {}\n{}\n{}".format( self.__class__.__name__, order, diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 22dc6b1c58..3d393e2488 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -61,6 +61,13 @@ def load(file, callback=None): category=UserWarning, module='iris', ) + filterwarnings( + 'ignore', + message="Ignoring netCDF variable '.*' invalid units '.*'", + category=UserWarning, + module='iris', + ) + raw_cubes = iris.load_raw(file, callback=callback) if not raw_cubes: raise Exception('Can not load cubes from {0}'.format(file)) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index bf24c4c738..2842fad760 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -498,8 +498,10 @@ def get_reference_levels(filename, project, dataset, short_name, + mip, + frequency, fix_dir): - """Get level definition from a CMOR coordinate. + """Get level definition from a reference dataset. Parameters ---------- @@ -517,9 +519,23 @@ def get_reference_levels(filename, levels or the string is badly formatted. """ - filename = fix_file(filename, short_name, project, dataset, fix_dir) + filename = fix_file( + file=filename, + short_name=short_name, + project=project, + dataset=dataset, + mip=mip, + output_dir=fix_dir, + ) cubes = load(filename, callback=concatenate_callback) - cubes = fix_metadata(cubes, short_name, project, dataset) + cubes = fix_metadata( + cubes=cubes, + short_name=short_name, + project=project, + dataset=dataset, + mip=mip, + frequency=frequency, + ) cube = cubes[0] try: coord = cube.coord(axis='Z') diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py index 59784dba6e..3669a2b380 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py @@ -11,22 +11,18 @@ class TestAllVars(unittest.TestCase): """Test all vars fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.cube.add_aux_coord(AuxCoord( - 0, - 'time', - 'time', - 'time', - Unit('days since 1850-01-01', 'julian') - )) - self.fix = AllVars() + self.cube.add_aux_coord( + AuxCoord(0, 'time', 'time', 'time', + Unit('days since 1850-01-01', 'julian'))) + self.fix = AllVars(None) def test_get(self): self.assertListEqual( - Fix.get_fixes('CMIP5', 'ACCESS1-0', 'tas'), [AllVars()]) + Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas'), + [AllVars(None)]) def test_fix_metadata(self): """Test fix for bad calendar.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py index 56caf4c0b5..a09b79f266 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py @@ -11,22 +11,18 @@ class TestAllVars(unittest.TestCase): """Test fixes for all vars.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.cube.add_aux_coord(AuxCoord( - 0, - 'time', - 'time', - 'time', - Unit('days since 1850-01-01', 'julian') - )) - self.fix = AllVars() + self.cube.add_aux_coord( + AuxCoord(0, 'time', 'time', 'time', + Unit('days since 1850-01-01', 'julian'))) + self.fix = AllVars(None) def test_get(self): self.assertListEqual( - Fix.get_fixes('CMIP5', 'ACCESS1-3', 'tas'), [AllVars()]) + Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas'), + [AllVars(None)]) def test_fix_metadata(self): """Test calendar fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py index a7bf80ee1a..30c0c44466 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py @@ -7,8 +7,7 @@ class TestTos(unittest.TestCase): """Test tos fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'BCC-CSM1-1', 'tos'), [Tos()]) + Fix.get_fixes('CMIP5', 'BCC-CSM1-1', 'Amon', 'tos'), [Tos(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py index 1e25b77243..01f427f478 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py @@ -7,8 +7,7 @@ class TestTos(unittest.TestCase): """Test tos fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'BCC-CSM1-1-M', 'tos'), [Tos()]) + Fix.get_fixes('CMIP5', 'BCC-CSM1-1-M', 'Amon', 'tos'), [Tos(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py index 9c3edd574b..6fd29d47a2 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py @@ -12,16 +12,15 @@ class TestCo2(unittest.TestCase): """Test fixes for CO2.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.fix = Co2() + self.fix = Co2(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'BNU-ESM', 'co2'), [Co2()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'co2'), + [Co2(None)]) def test_fix_metadata(self): """Test unit change.""" @@ -38,16 +37,15 @@ def test_fix_data(self): class Testfgco2(unittest.TestCase): """Test fixes for FgCO2.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='fgco2', units='J') - self.fix = FgCo2() + self.fix = FgCo2(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'BNU-ESM', 'fgco2'), [FgCo2()]) + Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'fgco2'), [FgCo2(None)]) def test_fix_metadata(self): """Test unit fix.""" @@ -64,16 +62,15 @@ def test_fix_data(self): class TestCh4(unittest.TestCase): """Test fixes for ch4.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='ch4', units='J') - self.fix = Ch4() + self.fix = Ch4(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'BNU-ESM', 'ch4'), [Ch4()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), + [Ch4(None)]) def test_fix_metadata(self): """Test unit fix.""" @@ -90,16 +87,15 @@ def test_fix_data(self): class Testspco2(unittest.TestCase): """Test fixes for SpCO2.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='spco2', units='J') - self.fix = SpCo2() + self.fix = SpCo2(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'BNU-ESM', 'spco2'), [SpCo2()]) + Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'spco2'), [SpCo2(None)]) def test_fix_metadata(self): """Test fix.""" @@ -116,18 +112,19 @@ def test_fix_data(self): class TestOd550Aer(unittest.TestCase): """Test fixes for SpCO2.""" - def setUp(self): """Prepare tests.""" self.cube = Cube( - ma.MaskedArray([1.e36], mask=(False,)), - var_name='od550aer',) - self.fix = Od550Aer() + ma.MaskedArray([1.e36], mask=(False, )), + var_name='od550aer', + ) + self.fix = Od550Aer(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'BNU-ESM', 'od550aer'), [Od550Aer()]) + Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'od550aer'), + [Od550Aer(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py index 29ebaaef33..ff9d75082e 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py @@ -10,16 +10,15 @@ class TestCanESM2Fgco2(unittest.TestCase): """Test fgc02 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='fgco2', units='J') - self.fix = FgCo2() + self.fix = FgCo2(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'CANESM2', 'fgco2'), [FgCo2()]) + Fix.get_fixes('CMIP5', 'CANESM2', 'Amon', 'fgco2'), [FgCo2(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py index fc5b59f46f..1d326c5a03 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py @@ -11,26 +11,22 @@ class TestsRlut(unittest.TestCase): """Test for rlut fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0, 2.0], var_name='rlut') self.cube.add_dim_coord( - DimCoord( - [0.50001, 1.499999], - standard_name='latitude', - bounds=[ - [0.00001, 0.999999], - [1.00001, 1.999999], - ]), - 0 - ) - self.fix = Rlut() + DimCoord([0.50001, 1.499999], + standard_name='latitude', + bounds=[ + [0.00001, 0.999999], + [1.00001, 1.999999], + ]), 0) + self.fix = Rlut(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'CCSM4', 'rlut'), [Rlut()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'rlut'), + [Rlut(None)]) def test_fix_metadata(self): """Check that latitudes values are rounded.""" @@ -45,26 +41,22 @@ def test_fix_metadata(self): class TestsRlutcs(unittest.TestCase): """Test for rlutcs fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0, 2.0], var_name='rlutcs') self.cube.add_dim_coord( - DimCoord( - [0.50001, 1.499999], - standard_name='latitude', - bounds=[ - [0.00001, 0.999999], - [1.00001, 1.999999], - ]), - 0 - ) - self.fix = Rlutcs() + DimCoord([0.50001, 1.499999], + standard_name='latitude', + bounds=[ + [0.00001, 0.999999], + [1.00001, 1.999999], + ]), 0) + self.fix = Rlutcs(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'CCSM4', 'rlutcs'), [Rlutcs()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'rlutcs'), + [Rlutcs(None)]) def test_fix_metadata(self): """Check that latitudes values are rounded.""" @@ -79,16 +71,15 @@ def test_fix_metadata(self): class TestSo(unittest.TestCase): """Tests for so fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0, 2.0], var_name='so', units='1.0') - self.fix = So() + self.fix = So(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'CCSM4', 'so'), [So()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'so'), + [So(None)]) def test_fix_metadata(self): """Checks that units are changed to the correct value.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py b/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py index a41e759fd0..917a0ded4a 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py @@ -14,12 +14,12 @@ class TestCo2(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.fix = Co2() + self.fix = Co2(None) def test_get(self): """Test fix get""" - self.assertListEqual(Fix.get_fixes('CMIP5', 'CESM1-BGC', 'co2'), - [Co2()]) + self.assertListEqual( + Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'co2'), [Co2(None)]) def test_fix_data(self): """Test fix to set units correctly.""" @@ -33,12 +33,12 @@ class TestGpp(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0, 1.0e33, 2.0]) - self.fix = Gpp() + self.fix = Gpp(None) def test_get(self): """Test fix get""" - self.assertListEqual(Fix.get_fixes('CMIP5', 'CESM1-BGC', 'gpp'), - [Gpp()]) + self.assertListEqual( + Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'gpp'), [Gpp(None)]) def test_fix_data(self): """Test fix to set missing values correctly.""" @@ -55,9 +55,9 @@ class TestNbp(TestGpp): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0, 1.0e33, 2.0]) - self.fix = Nbp() + self.fix = Nbp(None) def test_get(self): """Test fix get""" - self.assertListEqual(Fix.get_fixes('CMIP5', 'CESM1-BGC', 'nbp'), - [Nbp()]) + self.assertListEqual( + Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'nbp'), [Nbp(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py index e431ca0dec..ae9da2ef04 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py @@ -10,16 +10,16 @@ class TestMsftmyz(unittest.TestCase): """Test msftmyz fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='msftmyz', units='J') - self.fix = Msftmyz() + self.fix = Msftmyz(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'CNRM-CM5', 'msftmyz'), [Msftmyz()]) + Fix.get_fixes('CMIP5', 'CNRM-CM5', 'Amon', 'msftmyz'), + [Msftmyz(None)]) def test_fix_data(self): """Test data fix.""" @@ -30,16 +30,16 @@ def test_fix_data(self): class TestMsftmyzba(unittest.TestCase): """Test msftmyzba fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='msftmyzba', units='J') - self.fix = Msftmyzba() + self.fix = Msftmyzba(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'CNRM-CM5', 'msftmyzba'), [Msftmyzba()]) + Fix.get_fixes('CMIP5', 'CNRM-CM5', 'Amon', 'msftmyzba'), + [Msftmyzba(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py index 6daaacf8d6..2ed2d1bd02 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py @@ -11,16 +11,15 @@ class TestSic(unittest.TestCase): """Test sic fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='sic', units='J') - self.fix = Sic() + self.fix = Sic(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'EC-EARTH', 'sic'), [Sic()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'sic'), + [Sic(None)]) def test_fix_data(self): """Test data fix.""" @@ -31,16 +30,15 @@ def test_fix_data(self): class TestSftlf(unittest.TestCase): """Test sftlf fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='sftlf', units='J') - self.fix = Sftlf() + self.fix = Sftlf(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'EC-EARTH', 'sftlf'), [Sftlf()]) + Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'sftlf'), [Sftlf(None)]) def test_fix_data(self): """Test data fix.""" @@ -51,26 +49,23 @@ def test_fix_data(self): class TestTas(unittest.TestCase): """Test tas fixes.""" - def setUp(self): """Prepare tests.""" - height_coord = DimCoord( - 2., - standard_name='height', - long_name='height', - var_name='height', - units='m', - bounds=None, - attributes={'positive': 'up'} - ) + height_coord = DimCoord(2., + standard_name='height', + long_name='height', + var_name='height', + units='m', + bounds=None, + attributes={'positive': 'up'}) time_coord = DimCoord( 1., standard_name='time', var_name='time', units=Unit('days since 2070-01-01 00:00:00', calendar='gregorian'), - ) + ) self.height_coord = height_coord @@ -82,12 +77,12 @@ def setUp(self): self.cube_with[0].add_aux_coord(time_coord, 0) self.cube_with[0].coord('time').long_name = 'time' - self.fix = Tas() + self.fix = Tas(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'EC-EARTH', 'tas'), [Tas()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'tas'), + [Tas(None)]) def test_tas_fix_metadata(self): """Test metadata fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py index a17eeeb29d..8488632db6 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py @@ -11,22 +11,21 @@ class TestAll(unittest.TestCase): """Test fixes for all vars.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0, 2.0], var_name='co2', units='J') self.cube.add_dim_coord( - DimCoord( - [0.0, 1.0], - standard_name='time', - units=Unit('days since 0001-01', calendar='gregorian')), + DimCoord([0.0, 1.0], + standard_name='time', + units=Unit('days since 0001-01', calendar='gregorian')), 0) - self.fix = AllVars() + self.fix = AllVars(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'FGOALS-G2', 'tas'), [AllVars()]) + Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas'), + [AllVars(None)]) def test_fix_metadata(self): """Test calendar fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py index 9f1b237edf..709f78ce24 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py @@ -10,16 +10,15 @@ class TestCh4(unittest.TestCase): """Test ch4 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='ch4', units='J') - self.fix = Ch4() + self.fix = Ch4(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'FIO-ESM', 'ch4'), [Ch4()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'FIO-ESM', 'Amon', 'ch4'), + [Ch4(None)]) def test_fix_data(self): """Test data fix.""" @@ -30,16 +29,15 @@ def test_fix_data(self): class TestCo2(unittest.TestCase): """Test co2 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.fix = Co2() + self.fix = Co2(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'FIO-ESM', 'co2'), [Co2()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'FIO-ESM', 'Amon', 'co2'), + [Co2(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py index 862f74803f..4540645c98 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py @@ -10,18 +10,16 @@ class TestSftof(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='sftof', units='J') - self.fix = Sftof() + self.fix = Sftof(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'sftof'), - [AllVars(), Sftof()] - ) + Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'sftof'), + [Sftof(None), AllVars(None)]) def test_fix_data(self): """Test data fix.""" @@ -32,27 +30,26 @@ def test_fix_data(self): class TestAreacello(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello() + self.fix = Areacello(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'areacello'), - [AllVars(), Areacello()]) + Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'areacello'), + [Areacello(None), AllVars(None)]) def test_fix_metadata(self): """Test data fix.""" - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) def test_fix_data(self): """Test data fix.""" self.cube.units = 'm2' - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py index 8644cf15f1..fdd2f4c4c2 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py @@ -10,16 +10,16 @@ class TestSftof(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='sftof', units='J') - self.fix = Sftof() + self.fix = Sftof(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM3', 'sftof'), [AllVars(), Sftof()]) + Fix.get_fixes('CMIP5', 'GFDL-CM3', 'Amon', 'sftof'), + [Sftof(None), AllVars(None)]) def test_fix_data(self): """Test data fix.""" @@ -30,27 +30,26 @@ def test_fix_data(self): class TestAreacello(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello() + self.fix = Areacello(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM3', 'areacello'), - [AllVars(), Areacello()]) + Fix.get_fixes('CMIP5', 'GFDL-CM3', 'Amon', 'areacello'), + [Areacello(None), AllVars(None)]) def test_fix_metadata(self): """Test data fix.""" - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) def test_fix_data(self): """Test data fix.""" self.cube.units = 'm2' - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py index 6332ad9312..8c24e35077 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py @@ -32,11 +32,11 @@ def test_get_and_remove(cubes_in, cubes_out): CUBES = iris.cube.CubeList([CUBE_1, CUBE_2]) -@mock.patch( - 'esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove', autospec=True) +@mock.patch('esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove', + autospec=True) def test_allvars(mock_get_and_remove): """Test fixes for all vars.""" - fix = AllVars() + fix = AllVars(None) fix.fix_metadata(CUBES) assert mock_get_and_remove.call_count == 3 assert mock_get_and_remove.call_args_list == [ @@ -46,11 +46,11 @@ def test_allvars(mock_get_and_remove): ] -@mock.patch( - 'esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove', autospec=True) +@mock.patch('esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove', + autospec=True) def test_fgco2(mock_get_and_remove): """Test fgco2 fixes.""" - fix = FgCo2() + fix = FgCo2(None) fix.fix_metadata(CUBES) assert mock_get_and_remove.call_count == 2 assert mock_get_and_remove.call_args_list == [ @@ -61,16 +61,16 @@ def test_fgco2(mock_get_and_remove): class TestCo2(unittest.TestCase): """Test co2 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = iris.cube.Cube([1.0], var_name='co2', units='J') - self.fix = Co2() + self.fix = Co2(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'co2'), [AllVars(), Co2()]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'co2'), + [Co2(None), AllVars(None)]) def test_fix_data(self): """Test data fix.""" @@ -81,16 +81,16 @@ def test_fix_data(self): class TestUsi(unittest.TestCase): """Test usi fixes.""" - def setUp(self): """Prepare tests.""" self.cube = iris.cube.Cube([1.0], var_name='usi', units='J') - self.fix = Usi() + self.fix = Usi(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'usi'), [AllVars(), Usi()]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'usi'), + [Usi(None), AllVars(None)]) def test_fix_data(self): """Test metadata fix.""" @@ -100,16 +100,16 @@ def test_fix_data(self): class TestVsi(unittest.TestCase): """Test vsi fixes.""" - def setUp(self): """Prepare tests.""" self.cube = iris.cube.Cube([1.0], var_name='vsi', units='J') - self.fix = Vsi() + self.fix = Vsi(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'vsi'), [AllVars(), Vsi()]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'vsi'), + [Vsi(None), AllVars(None)]) def test_fix_data(self): """Test metadata fix.""" @@ -119,27 +119,26 @@ def test_fix_data(self): class TestAreacello(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello() + self.fix = Areacello(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'areacello'), - [AllVars(), Areacello()]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'areacello'), + [Areacello(None), AllVars(None)]) def test_fix_metadata(self): """Test data fix.""" - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) def test_fix_data(self): """Test data fix.""" self.cube.units = 'm2' - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py index 5126d8959c..900da08a97 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py @@ -11,18 +11,16 @@ class TestSftof(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='sftof', units='J') - self.fix = Sftof() + self.fix = Sftof(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'sftof'), - [AllVars(), Sftof()] - ) + Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'sftof'), + [Sftof(None), AllVars(None)]) def test_fix_data(self): """Test data fix.""" @@ -33,16 +31,16 @@ def test_fix_data(self): class TestCo2(unittest.TestCase): """Test co2 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.fix = Co2() + self.fix = Co2(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'co2'), [AllVars(), Co2()]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'co2'), + [Co2(None), AllVars(None)]) def test_fix_data(self): """Test data fix.""" @@ -53,27 +51,26 @@ def test_fix_data(self): class TestAreacello(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello() + self.fix = Areacello(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'areacello'), - [AllVars(), Areacello()]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'areacello'), + [Areacello(None), AllVars(None)]) def test_fix_metadata(self): """Test data fix.""" - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) def test_fix_data(self): """Test data fix.""" self.cube.units = 'm2' - cube = self.fix.fix_metadata((self.cube,))[0] + cube = self.fix.fix_metadata((self.cube, ))[0] self.assertEqual(cube.data[0], 1.0) self.assertEqual(cube.units, Unit('m2')) diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py index 1bfc2a0f54..ed2f7c0d1f 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py @@ -7,17 +7,17 @@ class TestAllVars(unittest.TestCase): """Test allvars fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'tas'), [AllVars()]) + Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'Amon', 'tas'), + [AllVars(None)]) class TestO2(unittest.TestCase): """Test o2 fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'o2'), [AllVars(), O2()]) + Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'Amon', 'o2'), + [O2(None), AllVars(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py index 46343f4b7b..b6eea1cd25 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py @@ -7,17 +7,17 @@ class TestAllVars(unittest.TestCase): """Test allvars fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'tas'), [AllVars()]) + Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'Amon', 'tas'), + [AllVars(None)]) class TestO2(unittest.TestCase): """Test o2 fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'o2'), [AllVars(), O2()]) + Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'Amon', 'o2'), + [O2(None), AllVars(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py index 65094c567e..9da7f16103 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py +++ b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py @@ -10,16 +10,15 @@ class TestGpp(unittest.TestCase): """Test gpp fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='gpp', units='J') - self.fix = Gpp() + self.fix = Gpp(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'INMCM4', 'gpp'), [Gpp()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'gpp'), + [Gpp(None)]) def test_fix_data(self): """Test data fox.""" @@ -30,16 +29,15 @@ def test_fix_data(self): class TestLai(unittest.TestCase): """Test lai fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='lai', units='J') - self.fix = Lai() + self.fix = Lai(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'INMCM4', 'lai'), [Lai()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'lai'), + [Lai(None)]) def test_fix_data(self): """Test data fix.""" @@ -50,16 +48,15 @@ def test_fix_data(self): class TestNbp(unittest.TestCase): """Tests for nbp.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='nbp') - self.fix = Nbp() + self.fix = Nbp(None) def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'INMCM4', 'nbp'), [Nbp()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'nbp'), + [Nbp(None)]) def test_fix_metadata(self): """Test fix on nbp files to set standard_name.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc5.py b/tests/integration/cmor/_fixes/cmip5/test_miroc5.py index d35ac414e9..d44093c405 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc5.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc5.py @@ -11,16 +11,15 @@ class TestSftof(unittest.TestCase): """Test sftof fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='sftof', units='J') - self.fix = Sftof() + self.fix = Sftof(None) def test_get(self): """Test fix get""" - self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'sftof'), - [Sftof()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'sftof'), + [Sftof(None)]) def test_fix_data(self): """Test data fix.""" @@ -31,7 +30,6 @@ def test_fix_data(self): class TestTas(unittest.TestCase): """Test tas fixes.""" - def setUp(self): """Prepare tests.""" self.coord_name = 'latitude' @@ -39,11 +37,12 @@ def setUp(self): bounds=[[1.23, 4.5678910]], standard_name=self.coord_name) self.cube = Cube([1.0], dim_coords_and_dims=[(self.coord, 0)]) - self.fix = Tas() + self.fix = Tas(None) def test_get(self): """Test fix get""" - self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'tas'), [Tas()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'tas'), + [Tas(None)]) def test_fix_metadata(self): """Test metadata fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py index 4660444cd5..51d887c597 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py @@ -12,16 +12,16 @@ class TestCo2(unittest.TestCase): """Test c02 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.fix = Co2() + self.fix = Co2(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MIROC-ESM', 'co2'), [AllVars(), Co2()]) + Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'co2'), + [Co2(None), AllVars(None)]) def test_fix_metadata(self): """Test unit fix.""" @@ -32,16 +32,16 @@ def test_fix_metadata(self): class TestTro3(unittest.TestCase): """Test tro3 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='tro3', units='J') - self.fix = Tro3() + self.fix = Tro3(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MIROC-ESM', 'tro3'), [AllVars(), Tro3()]) + Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'tro3'), + [Tro3(None), AllVars(None)]) def test_fix_data(self): """Test data fix.""" @@ -52,25 +52,23 @@ def test_fix_data(self): class TestAll(unittest.TestCase): """Test fixes for allvars.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([[1.0, 2.0], [3.0, 4.0]], var_name='co2', units='J') self.cube.add_dim_coord( - DimCoord( - [0, 1], - standard_name='time', - units=Unit( - 'days since 0000-01-01 00:00:00', calendar='gregorian')), - 0) + DimCoord([0, 1], + standard_name='time', + units=Unit('days since 0000-01-01 00:00:00', + calendar='gregorian')), 0) self.cube.add_dim_coord(DimCoord([0, 1], long_name='AR5PL35'), 1) - self.fix = AllVars() + self.fix = AllVars(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MIROC-ESM', 'tos'), [AllVars()]) + Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'tos'), + [AllVars(None)]) def test_fix_metadata_plev(self): """Test plev fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py index 276df42a2e..186f321710 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py @@ -11,16 +11,16 @@ class TestTro3(unittest.TestCase): """Test tro3 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='tro3', units='J') - self.fix = Tro3() + self.fix = Tro3(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MIROC-ESM-CHEM', 'tro3'), [Tro3()]) + Fix.get_fixes('CMIP5', 'MIROC-ESM-CHEM', 'Amon', 'tro3'), + [Tro3(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py index f20ecd3f09..f8c90ec3e2 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py @@ -10,16 +10,16 @@ class TestPctisccp2(unittest.TestCase): """Test Pctisccp2 fixes.""" - def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='pctisccp', units='J') - self.fix = Pctisccp() + self.fix = Pctisccp(None) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MPI-ESM-LR', 'pctisccp'), [Pctisccp()]) + Fix.get_fixes('CMIP5', 'MPI-ESM-LR', 'Amon', 'pctisccp'), + [Pctisccp(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py index 926484f300..1b93fb3004 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py @@ -7,19 +7,17 @@ class TestMsftmyz(unittest.TestCase): """Test msftmyz fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'msftmyz'), [Msftmyz()] - ) + Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'Amon', 'msftmyz'), + [Msftmyz(None)]) class TestThetao(unittest.TestCase): """Test thetao fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'thetao'), [ThetaO()] - ) + Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'Amon', 'thetao'), + [ThetaO(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py index a9e664381a..c00af1d6fe 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py @@ -7,9 +7,8 @@ class TestMsftmyz(unittest.TestCase): """Test msftmyz fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'MRI-ESM1', 'msftmyz'), [Msftmyz()] - ) + Fix.get_fixes('CMIP5', 'MRI-ESM1', 'Amon', 'msftmyz'), + [Msftmyz(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py index a7a285a1ed..ab286d78bd 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py +++ b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py @@ -53,15 +53,15 @@ CUBES_TO_FIX = [ (CubeList([CUBE_IN_SHORT]), CubeList([CUBE_IN_SHORT])), (CubeList([CUBE_IN_LONG]), CubeList([CUBE_OUT_LONG])), - (CubeList([CUBE_IN_LONG, CUBE_IN_SHORT]), - CubeList([CUBE_OUT_LONG, CUBE_IN_SHORT])), + (CubeList([CUBE_IN_LONG, + CUBE_IN_SHORT]), CubeList([CUBE_OUT_LONG, CUBE_IN_SHORT])), ] @pytest.mark.parametrize('cubes_in,cubes_out', CUBES_TO_FIX) def test_tas(cubes_in, cubes_out): """Test tas fixes.""" - fix = Tas() + fix = Tas(None) new_cubes = fix.fix_metadata(cubes_in) assert new_cubes is cubes_in assert new_cubes == cubes_out @@ -69,4 +69,4 @@ def test_tas(cubes_in, cubes_out): def test_get(): """Test fix get""" - assert Fix.get_fixes('CMIP5', 'NORESM1-ME', 'tas') == [Tas()] + assert Fix.get_fixes('CMIP5', 'NORESM1-ME', 'Amon', 'tas') == [Tas(None)] diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index fb753713f2..826d2f6118 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -15,8 +15,8 @@ def tas_cubes(): def test_get_tas_fix(): - fix = Fix.get_fixes('CMIP6', 'CESM2', 'tas') - assert fix == [Tas()] + fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'tas') + assert fix == [Tas(None)] def test_tas_fix_metadata(tas_cubes): @@ -29,7 +29,7 @@ def test_tas_fix_metadata(tas_cubes): long_name='height', units=Unit('m'), attributes={'positive': 'up'}) - fix = Tas() + fix = Tas(None) out_cubes = fix.fix_metadata(tas_cubes) assert out_cubes is tas_cubes for cube in out_cubes: diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index 75553adccb..f5410a8393 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -15,8 +15,8 @@ def tas_cubes(): def test_get_tas_fix(): - fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'tas') - assert fix == [Tas()] + fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'Amon', 'tas') + assert fix == [Tas(None)] def test_tas_fix_metadata(tas_cubes): @@ -29,7 +29,7 @@ def test_tas_fix_metadata(tas_cubes): long_name='height', units=Unit('m'), attributes={'positive': 'up'}) - fix = Tas() + fix = Tas(None) out_cubes = fix.fix_metadata(tas_cubes) assert out_cubes is tas_cubes for cube in out_cubes: diff --git a/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py b/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py index 52635a6ff6..3fe21b3d29 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py +++ b/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py @@ -14,14 +14,14 @@ def sample_cubes(): def test_get_tas_fix(): - fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'tas') - assert fix == [AllVars()] + fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'Amon', 'tas') + assert fix == [AllVars(None)] def test_allvars_fix_metadata(sample_cubes): for cube in sample_cubes: cube.attributes['parent_time_units'] = 'days since 1850-01-01' - out_cubes = AllVars().fix_metadata(sample_cubes) + out_cubes = AllVars(None).fix_metadata(sample_cubes) assert out_cubes is sample_cubes for cube in out_cubes: assert cube.attributes['parent_time_units'] == 'days since 1850-01-01' @@ -30,7 +30,7 @@ def test_allvars_fix_metadata(sample_cubes): def test_allvars_no_need_tofix_metadata(sample_cubes): for cube in sample_cubes: cube.attributes['parent_time_units'] = 'days since 1850-01-01' - out_cubes = AllVars().fix_metadata(sample_cubes) + out_cubes = AllVars(None).fix_metadata(sample_cubes) assert out_cubes is sample_cubes for cube in out_cubes: assert cube.attributes['parent_time_units'] == 'days since 1850-01-01' diff --git a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py index 82416a77c8..5edc8b8554 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py @@ -11,24 +11,16 @@ class TestAllVars(unittest.TestCase): def setUp(self): - self.fix = AllVars() + self.fix = AllVars(None) self.cube = Cube(np.random.rand(2, 2, 2), var_name='ch4') self.cube.add_aux_coord( - AuxCoord( - np.random.rand(2, 2), - var_name='nav_lat', - standard_name='latitude' - ), - (1, 2) - ) + AuxCoord(np.random.rand(2, 2), + var_name='nav_lat', + standard_name='latitude'), (1, 2)) self.cube.add_aux_coord( - AuxCoord( - np.random.rand(2, 2), - var_name='nav_lon', - standard_name='longitude' - ), - (1, 2) - ) + AuxCoord(np.random.rand(2, 2), + var_name='nav_lon', + standard_name='longitude'), (1, 2)) def test_fix_metadata_ocean_var(self): cell_area = Cube(np.random.rand(2, 2), standard_name='cell_area') diff --git a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py index 8723a8bbac..2239576ec5 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py @@ -9,13 +9,17 @@ @pytest.fixture def cubes(): - correct_lat_coord = iris.coords.DimCoord([0.0], var_name='lat', + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', standard_name='latitude') - wrong_lat_coord = iris.coords.DimCoord([0.0], var_name='latitudeCoord', + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', standard_name='latitude') - correct_lon_coord = iris.coords.DimCoord([0.0], var_name='lon', + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', standard_name='longitude') - wrong_lon_coord = iris.coords.DimCoord([0.0], var_name='longitudeCoord', + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', standard_name='longitude') correct_cube = iris.cube.Cube( [[10.0]], @@ -33,17 +37,17 @@ def cubes(): def test_get_allvars_fix(): - fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'arbitrary_var_name') - assert fix == [AllVars()] + fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Amon', 'arbitrary_var_name') + assert fix == [AllVars(None)] def test_get_tas_fix(): - fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'tas') - assert fix == [AllVars(), Tas()] + fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Amon', 'tas') + assert fix == [Tas(None), AllVars(None)] def test_allvars_fix_metadata(cubes): - fix = AllVars() + fix = AllVars(None) out_cubes = fix.fix_metadata(cubes) assert cubes is out_cubes for cube in out_cubes: @@ -76,7 +80,7 @@ def test_tas_fix_metadata(cubes): long_name='height', units=Unit('m'), attributes={'positive': 'up'}) - fix = Tas() + fix = Tas(None) # Check fix out_cubes = fix.fix_metadata(cubes) diff --git a/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py b/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py index b4f6d1aecf..fa09029520 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py @@ -14,14 +14,14 @@ def sample_cubes(): def test_get_tas_fix(): - fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'tas') - assert fix == [AllVars()] + fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'Amon', 'tas') + assert fix == [AllVars(None)] def test_allvars_fix_metadata(sample_cubes): for cube in sample_cubes: cube.attributes['parent_time_units'] = 'days since 1850-01-01' - out_cubes = AllVars().fix_metadata(sample_cubes) + out_cubes = AllVars(None).fix_metadata(sample_cubes) assert out_cubes is sample_cubes for cube in out_cubes: assert cube.attributes['parent_time_units'] == 'days since 1850-01-01' @@ -30,7 +30,7 @@ def test_allvars_fix_metadata(sample_cubes): def test_allvars_no_need_tofix_metadata(sample_cubes): for cube in sample_cubes: cube.attributes['parent_time_units'] = 'days since 1850-01-01' - out_cubes = AllVars().fix_metadata(sample_cubes) + out_cubes = AllVars(None).fix_metadata(sample_cubes) assert out_cubes is sample_cubes for cube in out_cubes: assert cube.attributes['parent_time_units'] == 'days since 1850-01-01' diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py new file mode 100644 index 0000000000..1134a4733e --- /dev/null +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -0,0 +1,882 @@ +"""Tests for the fixes of ERA5.""" +import iris +import numpy as np +import pytest +from cf_units import Unit + +from esmvalcore.cmor._fixes.native6.era5 import AllVars, Evspsbl, get_frequency +from esmvalcore.cmor.fix import Fix, fix_metadata +from esmvalcore.cmor.table import CMOR_TABLES + + +def test_get_evspsbl_fix(): + """Test whether the right fixes are gathered for a single variable.""" + fix = Fix.get_fixes('native6', 'ERA5', 'E1hr', 'evspsbl') + assert fix == [Evspsbl(None), AllVars(None)] + + +def test_get_frequency_hourly(): + """Test cubes with hourly frequency.""" + time = iris.coords.DimCoord([0, 1, 2], + standard_name='time', + units=Unit('hours since 1900-01-01')) + cube = iris.cube.Cube([1, 6, 3], + var_name='random_var', + dim_coords_and_dims=[(time, 0)]) + assert get_frequency(cube) == 'hourly' + cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0') + assert get_frequency(cube) == 'hourly' + + +def test_get_frequency_monthly(): + """Test cubes with monthly frequency.""" + time = iris.coords.DimCoord([0, 31, 59], + standard_name='time', + units=Unit('hours since 1900-01-01')) + cube = iris.cube.Cube([1, 6, 3], + var_name='random_var', + dim_coords_and_dims=[(time, 0)]) + assert get_frequency(cube) == 'monthly' + cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0') + assert get_frequency(cube) == 'monthly' + + +def test_get_frequency_fx(): + """Test cubes with time invariant frequency.""" + cube = iris.cube.Cube(1., long_name='Cube without time coordinate') + assert get_frequency(cube) == 'fx' + time = iris.coords.DimCoord(0, + standard_name='time', + units=Unit('hours since 1900-01-01')) + cube = iris.cube.Cube([1], + var_name='cube_with_length_1_time_coord', + long_name='Geopotential', + dim_coords_and_dims=[(time, 0)]) + assert get_frequency(cube) == 'fx' + cube.long_name = 'Not geopotential' + with pytest.raises(ValueError): + get_frequency(cube) + + +def _era5_latitude(): + return iris.coords.DimCoord(np.array([90., 0., -90.]), + standard_name='latitude', + long_name='latitude', + var_name='latitude', + units=Unit('degrees')) + + +def _era5_longitude(): + return iris.coords.DimCoord(np.array([0, 180, 359.75]), + standard_name='longitude', + long_name='longitude', + var_name='longitude', + units=Unit('degrees'), + circular=True) + + +def _era5_time(frequency): + if frequency == 'invariant': + timestamps = [788928] # hours since 1900 at 1 january 1990 + elif frequency == 'hourly': + timestamps = [788928, 788929, 788930] + elif frequency == 'monthly': + timestamps = [788928, 789672, 790344] + return iris.coords.DimCoord(np.array(timestamps, dtype='int32'), + standard_name='time', + long_name='time', + var_name='time', + units=Unit( + 'hours since 1900-01-01' + '00:00:00.0', + calendar='gregorian')) + + +def _era5_data(frequency): + if frequency == 'invariant': + return np.arange(9).reshape(1, 3, 3) + return np.arange(27).reshape(3, 3, 3) + + +def _cmor_latitude(): + return iris.coords.DimCoord(np.array([-90., 0., 90.]), + standard_name='latitude', + long_name='Latitude', + var_name='lat', + units=Unit('degrees_north'), + bounds=np.array([[-90., -45.], [-45., 45.], + [45., 90.]])) + + +def _cmor_longitude(): + return iris.coords.DimCoord(np.array([0, 180, 359.75]), + standard_name='longitude', + long_name='Longitude', + var_name='lon', + units=Unit('degrees_east'), + bounds=np.array([[-0.125, 90.], [90., 269.875], + [269.875, 359.875]]), + circular=True) + + +def _cmor_time(mip, bounds=None, shifted=False): + """Provide expected time coordinate after fixes.""" + if mip == 'E1hr': + offset = 51134 # days since 1850 at 1 january 1990 + timestamps = offset + np.arange(3) / 24 + if shifted: + timestamps -= 1 / 48 + if bounds is not None: + bounds = [[t - 1 / 48, t + 1 / 48] for t in timestamps] + elif mip == 'Amon': + timestamps = np.array([51149.5, 51179., 51208.5]) + if bounds is not None: + bounds = np.array([[51134., 51165.], [51165., 51193.], + [51193., 51224.]]) + + return iris.coords.DimCoord(np.array(timestamps, dtype=float), + standard_name='time', + long_name='time', + var_name='time', + units=Unit('days since 1850-1-1 00:00:00', + calendar='gregorian'), + bounds=bounds) + + +def _cmor_aux_height(value): + return iris.coords.AuxCoord(value, + long_name="height", + standard_name="height", + units=Unit('m'), + var_name="height", + attributes={'positive': 'up'}) + + +def _cmor_data(mip): + if mip == 'fx': + return np.arange(9).reshape(3, 3)[::-1, :] + return np.arange(27).reshape(3, 3, 3)[:, ::-1, :] + + +def clt_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='cloud cover fraction', + var_name='cloud_cover', + units='unknown', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def clt_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'clt') + time = _cmor_time('E1hr', bounds=True) + data = _cmor_data('E1hr') * 100 + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def evspsbl_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='total evapotranspiration', + var_name='e', + units='unknown', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def evspsbl_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'evspsbl') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') * 1000 / 3600. + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def evspsblpot_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='potential evapotranspiration', + var_name='epot', + units='unknown', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def evspsblpot_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'evspsblpot') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') * 1000 / 3600. + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def mrro_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='runoff', + var_name='runoff', + units='m', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def mrro_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'mrro') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') * 1000 / 3600. + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def orog_era5_hourly(): + time = _era5_time('invariant') + cube = iris.cube.Cube( + _era5_data('invariant'), + long_name='geopotential height', + var_name='zg', + units='m**2 s**-2', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def orog_cmor_fx(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('fx', 'orog') + data = _cmor_data('fx') / 9.80665 + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(_cmor_latitude(), 0), + (_cmor_longitude(), 1)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def pr_era5_monthly(): + time = _era5_time('monthly') + cube = iris.cube.Cube( + _era5_data('monthly'), + long_name='total_precipitation', + var_name='tp', + units='m', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def pr_cmor_amon(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('Amon', 'pr') + time = _cmor_time('Amon', bounds=True) + data = _cmor_data('Amon') * 1000. / 3600. / 24. + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def pr_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='total_precipitation', + var_name='tp', + units='m', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def pr_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'pr') + time = _cmor_time('E1hr', bounds=True, shifted=True) + data = _cmor_data('E1hr') * 1000. / 3600. + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def prsn_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='snow', + var_name='snow', + units='unknown', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def prsn_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'prsn') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') * 1000 / 3600. + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + return iris.cube.CubeList([cube]) + + +def ptype_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='snow', + var_name='snow', + units='unknown', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def ptype_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'ptype') + time = _cmor_time('E1hr', shifted=False, bounds=True) + data = _cmor_data('E1hr') + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + units=1, + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + cube.coord('latitude').long_name = 'latitude' + cube.coord('longitude').long_name = 'longitude' + return iris.cube.CubeList([cube]) + + +def rlds_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='surface thermal radiation downwards', + var_name='ssrd', + units='J m**-2', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def rlds_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'rlds') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') / 3600 + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020', + 'positive': + 'down' + }) + return iris.cube.CubeList([cube]) + + +def rls_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='runoff', + var_name='runoff', + units='W m-2', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def rls_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'rls') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020', + 'positive': + 'down' + }) + return iris.cube.CubeList([cube]) + + +def rsds_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='solar_radiation_downwards', + var_name='rlwd', + units='J m**-2', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def rsds_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'rsds') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') / 3600 + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020', + 'positive': + 'down' + }) + return iris.cube.CubeList([cube]) + + +def rsdt_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='thermal_radiation_downwards', + var_name='strd', + units='J m**-2', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def rsdt_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'rsdt') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') / 3600 + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020', + 'positive': + 'down' + }) + return iris.cube.CubeList([cube]) + + +def rss_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='net_solar_radiation', + var_name='ssr', + units='J m**-2', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def rss_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'rss') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') / 3600 + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020', + 'positive': + 'down' + }) + return iris.cube.CubeList([cube]) + + +def tas_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='2m_temperature', + var_name='t2m', + units='K', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def tas_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'tas') + time = _cmor_time('E1hr') + data = _cmor_data('E1hr') + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + cube.add_aux_coord(_cmor_aux_height(2.)) + return iris.cube.CubeList([cube]) + + +def tas_era5_monthly(): + time = _era5_time('monthly') + cube = iris.cube.Cube( + _era5_data('monthly'), + long_name='2m_temperature', + var_name='t2m', + units='K', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def tas_cmor_amon(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('Amon', 'tas') + time = _cmor_time('Amon', bounds=True) + data = _cmor_data('Amon') + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + cube.add_aux_coord(_cmor_aux_height(2.)) + return iris.cube.CubeList([cube]) + + +def tasmax_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='maximum 2m temperature', + var_name='mx2t', + units='K', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def tasmax_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'tasmax') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + cube.add_aux_coord(_cmor_aux_height(2.)) + return iris.cube.CubeList([cube]) + + +def tasmin_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='minimum 2m temperature', + var_name='mn2t', + units='K', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def tasmin_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'tasmin') + time = _cmor_time('E1hr', shifted=True, bounds=True) + data = _cmor_data('E1hr') + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + cube.add_aux_coord(_cmor_aux_height(2.)) + return iris.cube.CubeList([cube]) + + +def uas_era5_hourly(): + time = _era5_time('hourly') + cube = iris.cube.Cube( + _era5_data('hourly'), + long_name='10m_u_component_of_wind', + var_name='u10', + units='m s-1', + dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), + (_era5_longitude(), 2)], + ) + return iris.cube.CubeList([cube]) + + +def uas_cmor_e1hr(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('E1hr', 'uas') + time = _cmor_time('E1hr') + data = _cmor_data('E1hr') + cube = iris.cube.Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={ + 'comment': + 'Contains modified ' + 'Copernicus Climate Change Service ' + 'Information 2020' + }) + cube.add_aux_coord(_cmor_aux_height(10.)) + return iris.cube.CubeList([cube]) + + +VARIABLES = [ + pytest.param(a, b, c, d, id=c + '_' + d) for (a, b, c, d) in [ + (clt_era5_hourly(), clt_cmor_e1hr(), 'clt', 'E1hr'), + (evspsbl_era5_hourly(), evspsbl_cmor_e1hr(), 'evspsbl', 'E1hr'), + (evspsblpot_era5_hourly(), evspsblpot_cmor_e1hr(), 'evspsblpot', + 'E1hr'), + (mrro_era5_hourly(), mrro_cmor_e1hr(), 'mrro', 'E1hr'), + (orog_era5_hourly(), orog_cmor_fx(), 'orog', 'fx'), + (pr_era5_monthly(), pr_cmor_amon(), 'pr', 'Amon'), + (pr_era5_hourly(), pr_cmor_e1hr(), 'pr', 'E1hr'), + (prsn_era5_hourly(), prsn_cmor_e1hr(), 'prsn', 'E1hr'), + (ptype_era5_hourly(), ptype_cmor_e1hr(), 'ptype', 'E1hr'), + (rlds_era5_hourly(), rlds_cmor_e1hr(), 'rlds', 'E1hr'), + (rls_era5_hourly(), rls_cmor_e1hr(), 'rls', 'E1hr'), + (rsds_era5_hourly(), rsds_cmor_e1hr(), 'rsds', 'E1hr'), + (rsdt_era5_hourly(), rsdt_cmor_e1hr(), 'rsdt', 'E1hr'), + (rss_era5_hourly(), rss_cmor_e1hr(), 'rss', 'E1hr'), + (tas_era5_hourly(), tas_cmor_e1hr(), 'tas', 'E1hr'), + (tas_era5_monthly(), tas_cmor_amon(), 'tas', 'Amon'), + (tasmax_era5_hourly(), tasmax_cmor_e1hr(), 'tasmax', 'E1hr'), + (tasmin_era5_hourly(), tasmin_cmor_e1hr(), 'tasmin', 'E1hr'), + (uas_era5_hourly(), uas_cmor_e1hr(), 'uas', 'E1hr'), + ] +] + + +@pytest.mark.parametrize('era5_cubes, cmor_cubes, var, mip', VARIABLES) +def test_cmorization(era5_cubes, cmor_cubes, var, mip): + """Verify that cmorization results in the expected target cube.""" + fixed_cubes = fix_metadata(era5_cubes, var, 'native6', 'era5', mip) + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + cmor_cube = cmor_cubes[0] + if fixed_cube.coords('time'): + for cube in [fixed_cube, cmor_cube]: + coord = cube.coord('time') + coord.points = np.round(coord.points, decimals=7) + if coord.bounds is not None: + coord.bounds = np.round(coord.bounds, decimals=7) + print('cmor_cube:', cmor_cube.xml()) + print('fixed_cube:', fixed_cube.xml()) + assert fixed_cube.xml() == cmor_cube.xml() + assert fixed_cube == cmor_cube diff --git a/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py b/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py index 87bb3d3cf2..62be67de55 100644 --- a/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py +++ b/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py @@ -7,9 +7,7 @@ class TestPrw(unittest.TestCase): """Test prw fixes.""" - def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('OBS4MIPS', 'SSMI', 'prw'), [Prw()] - ) + self.assertListEqual(Fix.get_fixes('obs4mips', 'SSMI', 'Amon', 'prw'), + [Prw(None)]) diff --git a/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py b/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py index cd130f5f05..03b0dac3ab 100644 --- a/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py +++ b/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py @@ -7,9 +7,8 @@ class TestPrw(unittest.TestCase): """Test prw fixes.""" - def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('OBS4MIPS', 'SSMI-MERIS', 'prw'), [Prw()] - ) + Fix.get_fixes('obs4mips', 'SSMI-MERIS', 'Amon', 'prw'), + [Prw(None)]) diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py index 30ecd23c84..3a54e24127 100644 --- a/tests/integration/cmor/_fixes/test_fix.py +++ b/tests/integration/cmor/_fixes/test_fix.py @@ -3,6 +3,7 @@ import tempfile import unittest +import pytest from iris.cube import Cube from esmvalcore.cmor.fix import Fix @@ -20,51 +21,54 @@ def tearDown(self): def test_get_fix(self): from esmvalcore.cmor._fixes.cmip5.canesm2 import FgCo2 self.assertListEqual( - Fix.get_fixes('CMIP5', 'CanESM2', 'fgco2'), [FgCo2()]) + Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgco2'), [FgCo2(None)]) def test_get_fix_case_insensitive(self): from esmvalcore.cmor._fixes.cmip5.canesm2 import FgCo2 self.assertListEqual( - Fix.get_fixes('CMIP5', 'CanESM2', 'fgCo2'), [FgCo2()]) + Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgCo2'), [FgCo2(None)]) def test_get_fixes_with_replace(self): from esmvalcore.cmor._fixes.cmip5.bnu_esm import Ch4 - self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'ch4'), [Ch4()]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), + [Ch4(None)]) def test_get_fixes_with_generic(self): from esmvalcore.cmor._fixes.cmip5.cesm1_bgc import Co2 self.assertListEqual( - Fix.get_fixes('CMIP5', 'CESM1-BGC', 'co2'), [Co2()]) + Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'co2'), [Co2(None)]) def test_get_fix_no_project(self): - self.assertListEqual( - Fix.get_fixes('BAD_PROJECT', 'BNU-ESM', 'ch4'), []) + with pytest.raises(KeyError): + Fix.get_fixes('BAD_PROJECT', 'BNU-ESM', 'Amon', 'ch4') def test_get_fix_no_model(self): - self.assertListEqual(Fix.get_fixes('CMIP5', 'BAD_MODEL', 'ch4'), []) + self.assertListEqual( + Fix.get_fixes('CMIP5', 'BAD_MODEL', 'Amon', 'ch4'), []) def test_get_fix_no_var(self): - self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'BAD_VAR'), []) + self.assertListEqual( + Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'BAD_VAR'), []) def test_fix_metadata(self): cube = Cube([0]) reference = Cube([0]) - self.assertEqual(Fix().fix_metadata(cube), reference) + self.assertEqual(Fix(None).fix_metadata(cube), reference) def test_fix_data(self): cube = Cube([0]) reference = Cube([0]) - self.assertEqual(Fix().fix_data(cube), reference) + self.assertEqual(Fix(None).fix_data(cube), reference) def test_fix_file(self): filepath = 'sample_filepath' - self.assertEqual(Fix().fix_file(filepath, 'preproc'), filepath) + self.assertEqual(Fix(None).fix_file(filepath, 'preproc'), filepath) def test_fixed_filenam(self): filepath = os.path.join(self.temp_folder, 'file.nc') output_dir = os.path.join(self.temp_folder, 'fixed') os.makedirs(output_dir) - fixed_filepath = Fix().get_fixed_filepath(output_dir, filepath) + fixed_filepath = Fix(None).get_fixed_filepath(output_dir, filepath) self.assertTrue(fixed_filepath, os.path.join(output_dir, 'file.nc')) diff --git a/tests/integration/preprocessor/_regrid/test_get_file_levels.py b/tests/integration/preprocessor/_regrid/test_get_file_levels.py index 2c92a1d2cb..128a074453 100644 --- a/tests/integration/preprocessor/_regrid/test_get_file_levels.py +++ b/tests/integration/preprocessor/_regrid/test_get_file_levels.py @@ -36,8 +36,22 @@ def tearDown(self): os.remove(self.path) def test_get_coord(self): - self.assertListEqual( - _regrid.get_reference_levels( - self.path, 'project', 'dataset', 'short_name', 'output_dir'), - [0., 1] - ) + fix_file = unittest.mock.create_autospec(_regrid.fix_file) + fix_file.side_effect = lambda file, **_: file + fix_metadata = unittest.mock.create_autospec(_regrid.fix_metadata) + fix_metadata.side_effect = lambda cubes, **_: cubes + with unittest.mock.patch('esmvalcore.preprocessor._regrid.fix_file', + fix_file): + with unittest.mock.patch( + 'esmvalcore.preprocessor._regrid.fix_metadata', + fix_metadata): + reference_levels = _regrid.get_reference_levels( + filename=self.path, + project='CMIP6', + dataset='dataset', + short_name='short_name', + mip='mip', + frequency='mon', + fix_dir='output_dir', + ) + self.assertListEqual(reference_levels, [0., 1]) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index cff40fcef8..864c091c09 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -19,7 +19,6 @@ from .test_provenance import check_provenance MANDATORY_DATASET_KEYS = ( - 'cmor_table', 'dataset', 'diagnostic', 'end_year', @@ -348,13 +347,13 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'project': 'CMIP5', 'dataset': 'CanESM2', 'short_name': 'chl', + 'mip': 'Oyr', 'output_dir': fix_dir, }, 'fix_data': { 'project': 'CMIP5', 'dataset': 'CanESM2', 'short_name': 'chl', - 'cmor_table': 'CMIP5', 'mip': 'Oyr', 'frequency': 'yr', }, @@ -362,7 +361,6 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'project': 'CMIP5', 'dataset': 'CanESM2', 'short_name': 'chl', - 'cmor_table': 'CMIP5', 'mip': 'Oyr', 'frequency': 'yr', }, @@ -435,13 +433,13 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'project': 'CMIP5', 'dataset': 'CanESM2', 'short_name': 'sftlf', + 'mip': 'fx', 'output_dir': fix_dir, }, 'fix_data': { 'project': 'CMIP5', 'dataset': 'CanESM2', 'short_name': 'sftlf', - 'cmor_table': 'CMIP5', 'mip': 'fx', 'frequency': 'fx', }, @@ -449,7 +447,6 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'project': 'CMIP5', 'dataset': 'CanESM2', 'short_name': 'sftlf', - 'cmor_table': 'CMIP5', 'mip': 'fx', 'frequency': 'fx', }, @@ -671,7 +668,6 @@ def test_simple_cordex_recipe(tmp_path, patched_datafinder, 'tas_MOHC-HadGEM3-RA_evaluation_r1i1p1_v1_mon_1991-1993.nc') reference = { 'alias': 'MOHC-HadGEM3-RA', - 'cmor_table': 'CORDEX', 'dataset': 'MOHC-HadGEM3-RA', 'diagnostic': 'test', 'domain': 'AFR-44', @@ -768,11 +764,13 @@ def test_reference_dataset(tmp_path, patched_datafinder, config_user, fix_dir = os.path.splitext(reference.filename)[0] + '_fixed' get_reference_levels.assert_called_once_with( - reference.files[0], - 'CMIP5', - 'MPI-ESM-LR', - 'ta', - fix_dir, + filename=reference.files[0], + project='CMIP5', + dataset='MPI-ESM-LR', + short_name='ta', + mip='Amon', + frequency='mon', + fix_dir=fix_dir, ) assert 'regrid' not in reference.settings @@ -2040,9 +2038,11 @@ def test_wrong_project(tmp_path, patched_datafinder, config_user): - {dataset: CanESM2} scripts: null """) - with pytest.raises(ValueError) as wrong_proj: + with pytest.raises(RecipeError) as wrong_proj: get_recipe(tmp_path, content, config_user) - assert wrong_proj == "Project CMIP7 not in config-developer" + assert str(wrong_proj.value) == ( + "Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " + "with mip 'Omon'") def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user): diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index a51f23b8ce..814e810dd4 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -1,50 +1,59 @@ """Unit tests for the variable_info module.""" -import unittest -from unittest import mock +from unittest import TestCase +from unittest.mock import Mock, patch from esmvalcore.cmor.fix import Fix, fix_data, fix_file, fix_metadata -class TestFixFile(unittest.TestCase): +class TestFixFile(TestCase): """Fix file tests.""" - def setUp(self): """Prepare for testing.""" self.filename = 'filename' - self.mock_fix = mock.Mock() + self.mock_fix = Mock() self.mock_fix.fix_file.return_value = 'new_filename' def test_fix(self): """Check that the returned fix is applied.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', - return_value=[self.mock_fix]): - file_returned = fix_file('filename', 'short_name', 'project', - 'model', 'output_dir') + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[self.mock_fix]): + file_returned = fix_file( + file='filename', + short_name='short_name', + project='project', + dataset='model', + mip='mip', + output_dir='output_dir', + ) self.assertNotEqual(file_returned, self.filename) self.assertEqual(file_returned, 'new_filename') def test_nofix(self): """Check that the same file is returned if no fix is available.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - file_returned = fix_file('filename', 'short_name', 'project', - 'model', 'output_dir') + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[]): + file_returned = fix_file( + file='filename', + short_name='short_name', + project='project', + dataset='model', + mip='mip', + output_dir='output_dir', + ) self.assertEqual(file_returned, self.filename) -class TestGetCube(unittest.TestCase): +class TestGetCube(TestCase): """Test get cube by var_name method.""" - def setUp(self): """Prepare for testing.""" - self.cube_1 = mock.Mock() + self.cube_1 = Mock() self.cube_1.var_name = 'cube1' - self.cube_2 = mock.Mock() + self.cube_2 = Mock() self.cube_2.var_name = 'cube2' self.cubes = [self.cube_1, self.cube_2] - self.fix = Fix() + self.fix = Fix(None) def test_get_first_cube(self): """Test selecting first cube.""" @@ -67,138 +76,198 @@ def test_get_default(self): self.assertIs(self.cube_1, self.fix.get_cube_from_list(self.cubes)) -class TestFixMetadata(unittest.TestCase): +class TestFixMetadata(TestCase): """Fix metadata tests.""" - def setUp(self): """Prepare for testing.""" self.cube = self._create_mock_cube() + self.intermediate_cube = self._create_mock_cube() self.fixed_cube = self._create_mock_cube() - self.mock_fix = mock.Mock() - self.mock_fix.fix_metadata.return_value = [self.fixed_cube] + self.mock_fix = Mock() + self.mock_fix.fix_metadata.return_value = [self.intermediate_cube] + self.checker = Mock() + self.check_metadata = self.checker.return_value.check_metadata @staticmethod def _create_mock_cube(var_name='short_name'): - cube = mock.Mock() + cube = Mock() cube.var_name = var_name cube.attributes = {'source_file': 'source_file'} return cube def test_fix(self): """Check that the returned fix is applied.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', - return_value=[self.mock_fix]): - cube_returned = fix_metadata([self.cube], 'short_name', 'project', - 'model')[0] - self.assertTrue(cube_returned is not self.cube) - self.assertTrue(cube_returned is self.fixed_cube) + self.check_metadata.side_effect = lambda: self.fixed_cube + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[self.mock_fix]): + with patch('esmvalcore.cmor.fix._get_cmor_checker', + return_value=self.checker): + cube_returned = fix_metadata( + cubes=[self.cube], + short_name='short_name', + project='project', + dataset='model', + mip='mip', + )[0] + self.checker.assert_called_once_with(self.intermediate_cube) + self.check_metadata.assert_called_once_with() + assert cube_returned is not self.cube + assert cube_returned is not self.intermediate_cube + assert cube_returned is self.fixed_cube def test_nofix(self): """Check that the same cube is returned if no fix is available.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - cube_returned = fix_metadata([self.cube], 'short_name', 'project', - 'model')[0] - self.assertTrue(cube_returned is self.cube) - self.assertTrue(cube_returned is not self.fixed_cube) + self.check_metadata.side_effect = lambda: self.cube + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[]): + with patch('esmvalcore.cmor.fix._get_cmor_checker', + return_value=self.checker): + cube_returned = fix_metadata( + cubes=[self.cube], + short_name='short_name', + project='project', + dataset='model', + mip='mip', + )[0] + self.checker.assert_called_once_with(self.cube) + self.check_metadata.assert_called_once_with() + assert cube_returned is self.cube + assert cube_returned is not self.intermediate_cube + assert cube_returned is not self.fixed_cube def test_select_var(self): """Check that the same cube is returned if no fix is available.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - cube_returned = fix_metadata( - [self.cube, self._create_mock_cube('extra')], - 'short_name', - 'project', - 'model' - )[0] - self.assertTrue(cube_returned is self.cube) + self.check_metadata.side_effect = lambda: self.cube + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[]): + with patch('esmvalcore.cmor.fix._get_cmor_checker', + return_value=self.checker): + cube_returned = fix_metadata( + cubes=[self.cube, + self._create_mock_cube('extra')], + short_name='short_name', + project='CMIP6', + dataset='model', + mip='mip', + )[0] + self.checker.assert_called_once_with(self.cube) + self.check_metadata.assert_called_once_with() + assert cube_returned is self.cube def test_select_var_failed_if_bad_var_name(self): """Check that the same cube is returned if no fix is available.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[]): with self.assertRaises(ValueError): fix_metadata( - [ + cubes=[ self._create_mock_cube('not_me'), self._create_mock_cube('me_neither') ], - 'short_name', - 'project', - 'model' + short_name='short_name', + project='CMIP6', + dataset='model', + mip='mip', ) def test_cmor_checker_called(self): """Check that the cmor check is done.""" - checker = mock.Mock() - checker.return_value = mock.Mock() - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - with mock.patch( - 'esmvalcore.cmor.fix._get_cmor_checker', - return_value=checker) as get_mock: - fix_metadata([self.cube], 'short_name', 'project', 'model', - 'cmor_table', 'mip', 'frequency') + checker = Mock() + checker.return_value = Mock() + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[]): + with patch('esmvalcore.cmor.fix._get_cmor_checker', + return_value=checker) as get_mock: + fix_metadata( + cubes=[self.cube], + short_name='short_name', + project='CMIP6', + dataset='dataset', + mip='mip', + frequency='frequency', + ) get_mock.assert_called_once_with( automatic_fixes=True, fail_on_error=False, frequency='frequency', mip='mip', short_name='short_name', - table='cmor_table') + table='CMIP6', + ) checker.assert_called_once_with(self.cube) checker.return_value.check_metadata.assert_called_once_with() -class TestFixData(unittest.TestCase): +class TestFixData(TestCase): """Fix data tests.""" - def setUp(self): """Prepare for testing.""" - self.cube = mock.Mock() - self.fixed_cube = mock.Mock() - self.mock_fix = mock.Mock() - self.mock_fix.fix_data.return_value = self.fixed_cube + self.cube = Mock() + self.intermediate_cube = Mock() + self.fixed_cube = Mock() + self.mock_fix = Mock() + self.mock_fix.fix_data.return_value = self.intermediate_cube + self.checker = Mock() + self.check_data = self.checker.return_value.check_data def test_fix(self): """Check that the returned fix is applied.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', - return_value=[self.mock_fix]): - cube_returned = fix_data(self.cube, 'short_name', 'project', - 'model') - self.assertTrue(cube_returned is not self.cube) - self.assertTrue(cube_returned is self.fixed_cube) + self.check_data.side_effect = lambda: self.fixed_cube + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[self.mock_fix]): + with patch('esmvalcore.cmor.fix._get_cmor_checker', + return_value=self.checker): + cube_returned = fix_data( + self.cube, + short_name='short_name', + project='project', + dataset='model', + mip='mip', + ) + self.checker.assert_called_once_with(self.intermediate_cube) + self.check_data.assert_called_once_with() + assert cube_returned is not self.cube + assert cube_returned is not self.intermediate_cube + assert cube_returned is self.fixed_cube def test_nofix(self): """Check that the same cube is returned if no fix is available.""" - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - cube_returned = fix_data(self.cube, 'short_name', 'project', - 'model') - self.assertTrue(cube_returned is self.cube) - self.assertTrue(cube_returned is not self.fixed_cube) + self.check_data.side_effect = lambda: self.cube + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[]): + with patch('esmvalcore.cmor.fix._get_cmor_checker', + return_value=self.checker): + cube_returned = fix_data( + self.cube, + short_name='short_name', + project='CMIP6', + dataset='model', + mip='mip', + ) + self.checker.assert_called_once_with(self.cube) + self.check_data.assert_called_once_with() + assert cube_returned is self.cube + assert cube_returned is not self.intermediate_cube + assert cube_returned is not self.fixed_cube def test_cmor_checker_called(self): """Check that the cmor check is done.""" - checker = mock.Mock() - checker.return_value = mock.Mock() - with mock.patch( - 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - with mock.patch( - 'esmvalcore.cmor.fix._get_cmor_checker', - return_value=checker) as get_mock: - fix_data(self.cube, 'short_name', 'project', 'model', - 'cmor_table', 'mip', 'frequency') + checker = Mock() + checker.return_value = Mock() + with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', + return_value=[]): + with patch('esmvalcore.cmor.fix._get_cmor_checker', + return_value=checker) as get_mock: + fix_data(self.cube, 'short_name', 'CMIP6', 'model', 'mip', + 'frequency') get_mock.assert_called_once_with( + table='CMIP6', automatic_fixes=True, fail_on_error=False, frequency='frequency', mip='mip', short_name='short_name', - table='cmor_table') + ) checker.assert_called_once_with(self.cube) checker.return_value.check_data.assert_called_once_with() diff --git a/tests/unit/data_finder/test_get_start_end_year.py b/tests/unit/data_finder/test_get_start_end_year.py index 2982ae7f03..136e4e62dc 100644 --- a/tests/unit/data_finder/test_get_start_end_year.py +++ b/tests/unit/data_finder/test_get_start_end_year.py @@ -1,99 +1,51 @@ """Unit tests for :func:`esmvalcore._data_finder.regrid._stock_cube`""" -import unittest -import os import tempfile +import iris +import pytest from esmvalcore._data_finder import get_start_end_year - -class TestGetStartEndYear(unittest.TestCase): +FILENAME_CASES = [ + ['var_whatever_1980-1981', 1980, 1981], + ['var_whatever_1980.nc', 1980, 1980], + ['var_whatever_19800101-19811231.nc1', 1980, 1981], + ['var_whatever_19800101.nc', 1980, 1980], + ['1980-1981_var_whatever.nc', 1980, 1981], + ['1980_var_whatever.nc', 1980, 1980], + ['var_control-1980_whatever.nc', 1980, 1980], + ['19800101-19811231_var_whatever.nc', 1980, 1981], + ['19800101_var_whatever.nc', 1980, 1980], + ['var_control-19800101_whatever.nc', 1980, 1980], + ['19800101_var_control-1950_whatever.nc', 1980, 1980], + ['var_control-1950_whatever_19800101.nc', 1980, 1980], +] + + +@pytest.mark.parametrize('case', FILENAME_CASES) +def test_get_start_end_year(case): """Tests for get_start_end_year function""" - - def setUp(self): - descriptor, self.temp_file = tempfile.mkstemp(suffix='.nc') - os.close(descriptor) - - def tearDown(self): - if os.path.isfile(self.temp_file): - os.remove(self.temp_file) - - def test_years_at_the_end(self): - """Test parse files with two years at the end""" - start, end = get_start_end_year('var_whatever_1980-1981') - self.assertEqual(1980, start) - self.assertEqual(1981, end) - - def test_one_year_at_the_end(self): - """Test parse files with one year at the end""" - start, end = get_start_end_year('var_whatever_1980.nc') - self.assertEqual(1980, start) - self.assertEqual(1980, end) - - def test_full_dates_at_the_end(self): - """Test parse files with two dates at the end""" - start, end = get_start_end_year('var_whatever_19800101-19811231.nc') - self.assertEqual(1980, start) - self.assertEqual(1981, end) - - def test_one_fulldate_at_the_end(self): - """Test parse files with one date at the end""" - start, end = get_start_end_year('var_whatever_19800101.nc') - self.assertEqual(1980, start) - self.assertEqual(1980, end) - - def test_years_at_the_start(self): - """Test parse files with two years at the start""" - start, end = get_start_end_year('1980-1981_var_whatever.nc') - self.assertEqual(1980, start) - self.assertEqual(1981, end) - - def test_one_year_at_the_start(self): - """Test parse files with one year at the start""" - start, end = get_start_end_year('1980_var_whatever.nc') - self.assertEqual(1980, start) - self.assertEqual(1980, end) - - def test_full_dates_at_the_start(self): - """Test parse files with two dates at the start""" - start, end = get_start_end_year('19800101-19811231_var_whatever.nc') - self.assertEqual(1980, start) - self.assertEqual(1981, end) - - def test_one_fulldate_at_the_start(self): - """Test parse files with one date at the start""" - start, end = get_start_end_year('19800101_var_whatever.nc') - self.assertEqual(1980, start) - self.assertEqual(1980, end) - - def test_start_and_date_in_name(self): - """Test parse one date at the start and one in experiment's name""" - start, end = get_start_end_year( - '19800101_var_control-1950_whatever.nc') - self.assertEqual(1980, start) - self.assertEqual(1980, end) - - def test_end_and_date_in_name(self): - """Test parse one date at the end and one in experiment's name""" - start, end = get_start_end_year( - 'var_control-1950_whatever_19800101.nc') - self.assertEqual(1980, start) - self.assertEqual(1980, end) - - def test_read_file_if_no_date_present(self): - """Test raises if no date is present""" - import iris - from iris.cube import Cube - from iris.coords import DimCoord - cube = Cube([0, 0], var_name='var') - time = DimCoord([0, 366], 'time', units='days since 1990-01-01') - cube.add_dim_coord(time, 0) - iris.save(cube, self.temp_file) - start, end = get_start_end_year(self.temp_file) - self.assertEqual(1990, start) - self.assertEqual(1991, end) - - def test_fails_if_no_date_present(self): - """Test raises if no date is present""" - with self.assertRaises(ValueError): - get_start_end_year('var_whatever') + filename, case_start, case_end = case + start, end = get_start_end_year(filename) + assert case_start == start + assert case_end == end + + +def test_read_time_from_cube(): + """Try to get time from cube if no date in filename""" + temp_file = tempfile.NamedTemporaryFile(suffix='.nc') + cube = iris.cube.Cube([0, 0], var_name='var') + time = iris.coords.DimCoord([0, 366], + 'time', + units='days since 1990-01-01') + cube.add_dim_coord(time, 0) + iris.save(cube, temp_file.name) + start, end = get_start_end_year(temp_file.name) + assert start == 1990 + assert end == 1991 + + +def test_fails_if_no_date_present(): + """Test raises if no date is present""" + with pytest.raises((ValueError, OSError)): + get_start_end_year('var_whatever')