From d9d43e84cce019d56b70aae9bf3c59e34e5ae72a Mon Sep 17 00:00:00 2001
From: Bouwe Andela <b.andela@esciencecenter.nl>
Date: Fri, 1 Nov 2019 17:32:53 +0100
Subject: [PATCH 01/40] Initial idea for using ERA5 data in native format

---
 esmvalcore/_recipe.py                      |   7 +-
 esmvalcore/cmor/_fixes/fix.py              |  28 ++++--
 esmvalcore/cmor/_fixes/native6/__init__.py |   0
 esmvalcore/cmor/_fixes/native6/era5.py     | 112 +++++++++++++++++++++
 esmvalcore/cmor/fix.py                     |  23 ++---
 esmvalcore/config-developer.yml            |   9 ++
 esmvalcore/preprocessor/__init__.py        |   4 +-
 7 files changed, 157 insertions(+), 26 deletions(-)
 create mode 100644 esmvalcore/cmor/_fixes/native6/__init__.py
 create mode 100644 esmvalcore/cmor/_fixes/native6/era5.py

diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py
index c75af61145..6a379eff42 100644
--- a/esmvalcore/_recipe.py
+++ b/esmvalcore/_recipe.py
@@ -304,11 +304,8 @@ def _get_default_settings(variable, config_user, derive=False):
     settings['fix_file'] = dict(fix)
     settings['fix_file']['output_dir'] = fix_dir
     # Cube fixes
-    # Only supply mip if the CMOR check fixes are implemented.
-    if variable.get('cmor_table'):
-        fix['cmor_table'] = variable['cmor_table']
-        fix['mip'] = variable['mip']
-        fix['frequency'] = variable['frequency']
+    fix['mip'] = variable['mip']
+    fix['frequency'] = variable['frequency']
     settings['fix_data'] = dict(fix)
     settings['fix_metadata'] = dict(fix)
 
diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py
index 2e59000777..2358e02224 100644
--- a/esmvalcore/cmor/_fixes/fix.py
+++ b/esmvalcore/cmor/_fixes/fix.py
@@ -3,11 +3,23 @@
 import os
 import inspect
 
+from ..table import CMOR_TABLES
 
-class Fix(object):
+
+class Fix:
     """
     Base class for dataset fixes.
     """
+    def __init__(self, vardef):
+        """Initialize fix object.
+
+        Parameters
+        ----------
+        vardef: basestring
+            CMOR table entry
+
+        """
+        self.vardef = vardef
 
     def fix_file(self, filepath, output_dir):
         """
@@ -109,7 +121,7 @@ def __ne__(self, other):
         return not self.__eq__(other)
 
     @staticmethod
-    def get_fixes(project, dataset, variable):
+    def get_fixes(project, dataset, mip, short_name):
         """
         Get the fixes that must be applied for a given dataset.
 
@@ -128,16 +140,20 @@ def get_fixes(project, dataset, variable):
         ----------
         project: str
         dataset: str
-        variable: str
+        mip: str
+        short_name: str
 
         Returns
         -------
         list(Fix)
             Fixes to apply for the given data
         """
+        cmor_table = CMOR_TABLES[project]
+        vardef = cmor_table.get_variable(mip, short_name)
+
         project = project.replace('-', '_').lower()
         dataset = dataset.replace('-', '_').lower()
-        variable = variable.replace('-', '_').lower()
+        short_name = short_name.replace('-', '_').lower()
 
         fixes = []
         try:
@@ -146,9 +162,9 @@ def get_fixes(project, dataset, variable):
 
             classes = inspect.getmembers(fixes_module, inspect.isclass)
             classes = dict((name.lower(), value) for name, value in classes)
-            for fix_name in ('allvars', variable):
+            for fix_name in ('allvars', short_name):
                 try:
-                    fixes.append(classes[fix_name]())
+                    fixes.append(classes[fix_name](vardef))
                 except KeyError:
                     pass
         except ImportError:
diff --git a/esmvalcore/cmor/_fixes/native6/__init__.py b/esmvalcore/cmor/_fixes/native6/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
new file mode 100644
index 0000000000..f11529671e
--- /dev/null
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -0,0 +1,112 @@
+"""Fixes for ERA5."""
+import numpy as np
+from iris.cube import CubeList
+
+from ..fix import Fix
+from ..shared import add_scalar_height_coord
+
+
+class AllVars(Fix):
+    """Fixes for all variables."""
+
+    def _fix_coordinates(self, cube):
+        """Fix coordinates."""
+        # Make latitude increasing
+        cube = cube[..., ::-1, :]
+
+        # Make pressure_levels decreasing
+        if cube.coords('pressure_level'):
+            cube = cube[:, ::-1, ...]
+
+        # Add scalar height coordinates
+        if 'height2m' in self.vardef.dimensions:
+            add_scalar_height_coord(cube, 2.)
+        if 'height10m' in self.vardef.dimensions:
+            add_scalar_height_coord(cube, 10.)
+
+        for axis in 'T', 'X', 'Y', 'Z':
+            coord_def = self.vardef.coordinates.get(axis)
+            if coord_def:
+                coord = cube.coord(axis=axis)
+                if axis == 'T':
+                    coord.convert_units('days since 1850-1-1 00:00:00.0')
+                if axis == 'Z':
+                    coord.convert_units(coord_def.units)
+                coord.standard_name = coord_def.standard_name
+                coord.var_name = coord_def.out_name
+                coord.long_name = coord_def.long_name
+                coord.points = coord.core_points().astype('float64')
+                if len(coord.points) > 1:
+                    coord.guess_bounds()
+
+        self._fix_monthly_time_coord(cube)
+
+        return cube
+
+    @staticmethod
+    def _frequency(cube):
+        if not cube.coords(axis='T'):
+            return 'fx'
+        coord = cube.coord(axis='T')
+        if 27 < coord.points[1] - coord.points[0] < 32:
+            return 'monthly'
+        return 'hourly'
+
+    def _fix_monthly_time_coord(self, cube):
+        """Set the monthly time coordinates to the middle of the month."""
+        if self._frequency(cube) == 'mon':
+            coord = cube.coord(axis='T')
+            end = []
+            for cell in coord.cells():
+                month = cell.point.month + 1
+                year = cell.point.year
+                if month == 13:
+                    month = 1
+                    year = year + 1
+                end.append(cell.point.replace(month=month, year=year))
+            end = coord.units.date2num(end)
+            start = coord.points
+            coord.points = 0.5 * (start + end)
+            coord.bounds = np.column_stack([start, end])
+
+    def _fix_units(self, cube):
+        """Fix units."""
+        if cube.var_name == 'clt':
+            # Correct cloud cover units from (0 - 1) to 1
+            cube.units = 1
+        if cube.var_name in {'evspsbl', 'mrro', 'prsn', 'pr', 'evspsblpot'}:
+            # Correct units from m of water to kg of water per m2
+            cube.units = 'kg m-2'
+            cube.data = cube.core_data() * 1000.
+
+        if cube.var_name in {
+                'rss', 'rsds', 'rsdt', 'evspsbl', 'mrro', 'prsn', 'pr',
+                'evspsblpot'
+        }:
+            if self._frequency(cube) == 'monthly':
+                cube.units = cube.units * 'd-1'
+            elif self._frequency(cube) == 'hourly':
+                cube.units = cube.units * 'h-1'
+
+        if cube.var_name in {'rls', 'rsds', 'rsdt', 'rss'}:
+            # Radiation fluxes are positive in downward direction
+            cube.attributes['positive'] = 'down'
+
+        cube.convert_units(self.vardef.units)
+
+    def fix_metadata(self, cubes):
+        """Fix metadata."""
+        fixed_cubes = CubeList()
+        for cube in cubes:
+            cube.var_name = self.vardef.short_name
+            cube.standard_name = self.vardef.standard_name
+            cube.long_name = self.vardef.long_name
+
+            cube = self._fix_coordinates(cube)
+            self._fix_units(cube)
+
+            cube.data = cube.core_data().astype('float32')
+
+            fixed_cubes.append(cube)
+
+        return fixed_cubes
diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py
index 2881d49aa7..69054ed141 100644
--- a/esmvalcore/cmor/fix.py
+++ b/esmvalcore/cmor/fix.py
@@ -45,7 +45,7 @@ def fix_file(file, short_name, project, dataset, output_dir):
 
     """
     for fix in Fix.get_fixes(
-            project=project, dataset=dataset, variable=short_name):
+            project=project, dataset=dataset, mip='', short_name=short_name):
         file = fix.fix_file(file, output_dir)
     return file
 
@@ -54,8 +54,7 @@ def fix_metadata(cubes,
                  short_name,
                  project,
                  dataset,
-                 cmor_table=None,
-                 mip=None,
+                 mip='',
                  frequency=None):
     """
     Fix cube metadata if fixes are required and check it anyway.
@@ -75,9 +74,6 @@ def fix_metadata(cubes,
 
     dataset: str
 
-    cmor_table: str, optional
-        CMOR tables to use for the check, if available
-
     mip: str, optional
         Variable's MIP, if available
 
@@ -96,7 +92,7 @@ def fix_metadata(cubes,
 
     """
     fixes = Fix.get_fixes(
-        project=project, dataset=dataset, variable=short_name)
+        project=project, dataset=dataset, mip=mip, short_name=short_name)
     fixed_cubes = []
     by_file = defaultdict(list)
     for cube in cubes:
@@ -134,10 +130,10 @@ def fix_metadata(cubes,
         else:
             cube = cube_list[0]
 
-        if cmor_table and mip:
+        if mip:
             checker = _get_cmor_checker(
                 frequency=frequency,
-                table=cmor_table,
+                table=project,
                 mip=mip,
                 short_name=short_name,
                 fail_on_error=False,
@@ -152,8 +148,7 @@ def fix_data(cube,
              short_name,
              project,
              dataset,
-             cmor_table=None,
-             mip=None,
+             mip='',
              frequency=None):
     """
     Fix cube data if fixes add present and check it anyway.
@@ -196,12 +191,12 @@ def fix_data(cube,
 
     """
     for fix in Fix.get_fixes(
-            project=project, dataset=dataset, variable=short_name):
+            project=project, dataset=dataset, mip=mip, short_name=short_name):
         cube = fix.fix_data(cube)
-    if cmor_table and mip:
+    if mip:
         checker = _get_cmor_checker(
             frequency=frequency,
-            table=cmor_table,
+            table=project,
             mip=mip,
             short_name=short_name,
             fail_on_error=False,
diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml
index bf1b689280..c7b171b22c 100644
--- a/esmvalcore/config-developer.yml
+++ b/esmvalcore/config-developer.yml
@@ -129,6 +129,15 @@ OBS6:
   output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}'
   cmor_type: 'CMIP6'
 
+Native6:
+  cmor_strict: false
+  input_dir:
+    default: 'Tier{tier}/{dataset}'
+  input_file:
+    default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc'
+  output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}'
+  cmor_type: 'CMIP6'
+
 obs4mips:
   cmor_strict: false
   input_dir:
diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py
index d24dd5d686..2d46bfe5ba 100644
--- a/esmvalcore/preprocessor/__init__.py
+++ b/esmvalcore/preprocessor/__init__.py
@@ -2,6 +2,7 @@
 import copy
 import inspect
 import logging
+from pprint import pformat
 
 from iris.cube import Cube
 
@@ -416,7 +417,8 @@ def __str__(self):
             step for step in self.order
             if any(step in product.settings for product in self.products)
         ]
-        products = '\n\n'.join(str(p) for p in self.products)
+        products = '\n\n'.join('\n'.join([str(p), pformat(p.settings)])
+                               for p in self.products)
         txt = "{}:\norder: {}\n{}\n{}".format(
             self.__class__.__name__,
             order,

From eac9142e7a32ea913d13e616a8bca957083443e7 Mon Sep 17 00:00:00 2001
From: Jaro Camphuijsen <jjecamphuijsen@gmail.com>
Date: Tue, 24 Dec 2019 16:36:03 +0100
Subject: [PATCH 02/40] Refactor into variable specific fixes

---
 esmvalcore/cmor/_fixes/native6/era5.py | 119 +++++++++++++++++++------
 1 file changed, 92 insertions(+), 27 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index f11529671e..485050b072 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -6,7 +6,94 @@
 from ..shared import add_scalar_height_coord
 
 
-class AllVars(Fix):
+class FixEra5(Fix):
+    """Fixes for ERA5 variables"""
+
+    @staticmethod
+    def _frequency(cube):
+        if not cube.coords(axis='T'):
+            return 'fx'
+        coord = cube.coord(axis='T')
+        if 27 < coord.points[1] - coord.points[0] < 32:
+            return 'monthly'
+        return 'hourly'
+
+class Accumulated(FixEra5):
+    """Fixes for accumulated variables."""
+
+    def _fix_frequency(self, cube):
+        if self._frequency(cube) == 'monthly':
+            cube.units = cube.units * 'd-1'
+        elif self._frequency(cube) == 'hourly':
+            cube.units = cube.units * 'h-1'
+        return cube
+
+class Hydrological(Accumulated):
+    """Fixes for accumulated hydrological variables."""
+
+    def _fix_units(self, cube):
+        cube.units = cube.units * 'kg m-3'
+        cube.data = cube.core_data() * 1000.
+
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            self._fix_frequency(cube)
+            self._fix_units(cube)
+        return cubes
+
+# radiation fixes: 'rls', 'rsds', 'rsdt', 'rss'
+
+class Radiation(Accumulated):
+    """Fixes for accumulated radiation variables."""
+
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            cube.attributes['positive'] = 'down'
+            self._fix_frequency(cube)
+        return cubes
+
+class Evspsbl(Hydrological):
+    """Fixes for evspsbl."""
+
+class Mrro(Hydrological):
+    """Fixes for evspsbl."""
+
+class Prsn(Hydrological):
+    """Fixes for evspsbl."""
+
+class Pr(Hydrological):
+    """Fixes for evspsbl."""
+
+class Evspsblpot(Hydrological):
+    """Fixes for evspsbl."""
+
+class Rss(Radiation):
+    """Fixes for Rss."""
+
+class Rsds(Radiation):
+    """Fixes for Rsds."""
+
+class Rsdt(Radiation):
+    """Fixes for Rsdt."""
+
+class Rls(Radiation):
+    """Fixes for Rls."""
+
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            cube.attributes['positive'] = 'down'
+        return cubes
+
+class Clt(Fix):
+    """Fixes for clt."""
+
+    def fix_metadata(self, cubes):
+        """Fix units."""
+        for cube in cubes:
+            cube.units = 1
+        return cubes   
+
+class AllVars(FixEra5):
     """Fixes for all variables."""
 
     def _fix_coordinates(self, cube):
@@ -43,18 +130,9 @@ def _fix_coordinates(self, cube):
 
         return cube
 
-    @staticmethod
-    def _frequency(cube):
-        if not cube.coords(axis='T'):
-            return 'fx'
-        coord = cube.coord(axis='T')
-        if 27 < coord.points[1] - coord.points[0] < 32:
-            return 'monthly'
-        return 'hourly'
-
     def _fix_monthly_time_coord(self, cube):
         """Set the monthly time coordinates to the middle of the month."""
-        if self._frequency(cube) == 'mon':
+        if self._frequency(cube) == 'monthly':
             coord = cube.coord(axis='T')
             end = []
             for cell in coord.cells():
@@ -71,27 +149,14 @@ def _fix_monthly_time_coord(self, cube):
 
     def _fix_units(self, cube):
         """Fix units."""
-        if cube.var_name == 'clt':
-            # Correct cloud cover units from (0 - 1) to 1
+        if cube.units == '(0 - 1)':
+            # Correct dimensionless units to 1 from ecmwf format '(0 - 1)'
             cube.units = 1
-        if cube.var_name in {'evspsbl', 'mrro', 'prsn', 'pr', 'evspsblpot'}:
+        if cube.units == 'm of water equivalent':
             # Correct units from m of water to kg of water per m2
             cube.units = 'kg m-2'
             cube.data = cube.core_data() * 1000.
 
-        if cube.var_name in {
-                'rss', 'rsds', 'rsdt', 'evspsbl', 'mrro', 'prsn', 'pr',
-                'evspsblpot'
-        }:
-            if self._frequency(cube) == 'monthly':
-                cube.units = cube.units * 'd-1'
-            elif self._frequency(cube) == 'hourly':
-                cube.units = cube.units * 'h-1'
-
-        if cube.var_name in {'rls', 'rsds', 'rsdt', 'rss'}:
-            # Radiation fluxes are positive in downward direction
-            cube.attributes['positive'] = 'down'
-
         cube.convert_units(self.vardef.units)
 
     def fix_metadata(self, cubes):

From 87eb08ba8170b4558f283d9caf880ceae1ae8619 Mon Sep 17 00:00:00 2001
From: Jaro Camphuijsen <jjecamphuijsen@gmail.com>
Date: Thu, 2 Jan 2020 10:32:21 +0100
Subject: [PATCH 03/40] clean up and improve class inheritance

---
 esmvalcore/cmor/_fixes/native6/era5.py | 51 +++++++++++---------------
 1 file changed, 22 insertions(+), 29 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 485050b072..d5a1b6f850 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -28,71 +28,64 @@ def _fix_frequency(self, cube):
             cube.units = cube.units * 'h-1'
         return cube
 
-class Hydrological(Accumulated):
+    def fix_metadata(self, cubes):
+        super.fix_metadata(cubes)
+        for cube in cubes:
+            self._fix_frequency(cube)
+        return cubes
+
+class Hydrological(FixEra5):
     """Fixes for accumulated hydrological variables."""
 
-    def _fix_units(self, cube):
+    @staticmethod
+    def _fix_units(cube):
         cube.units = cube.units * 'kg m-3'
         cube.data = cube.core_data() * 1000.
+        return cube
 
     def fix_metadata(self, cubes):
+        super.fix_metadata(cubes)
         for cube in cubes:
-            self._fix_frequency(cube)
             self._fix_units(cube)
         return cubes
 
-# radiation fixes: 'rls', 'rsds', 'rsdt', 'rss'
-
-class Radiation(Accumulated):
+class Radiation(FixEra5):
     """Fixes for accumulated radiation variables."""
 
     def fix_metadata(self, cubes):
+        super.fix_metadata(cubes)
         for cube in cubes:
             cube.attributes['positive'] = 'down'
-            self._fix_frequency(cube)
         return cubes
 
-class Evspsbl(Hydrological):
+
+class Evspsbl(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
-class Mrro(Hydrological):
+class Mrro(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
-class Prsn(Hydrological):
+class Prsn(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
-class Pr(Hydrological):
+class Pr(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
-class Evspsblpot(Hydrological):
+class Evspsblpot(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
-class Rss(Radiation):
+class Rss(Radiation, Accumulated):
     """Fixes for Rss."""
 
-class Rsds(Radiation):
+class Rsds(Radiation, Accumulated):
     """Fixes for Rsds."""
 
-class Rsdt(Radiation):
+class Rsdt(Radiation, Accumulated):
     """Fixes for Rsdt."""
 
 class Rls(Radiation):
     """Fixes for Rls."""
 
-    def fix_metadata(self, cubes):
-        for cube in cubes:
-            cube.attributes['positive'] = 'down'
-        return cubes
-
-class Clt(Fix):
-    """Fixes for clt."""
-
-    def fix_metadata(self, cubes):
-        """Fix units."""
-        for cube in cubes:
-            cube.units = 1
-        return cubes   
-
 class AllVars(FixEra5):
     """Fixes for all variables."""
 

From c8c21647dcc8bff7fbbde953c297bc29e6ce9474 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 8 Jan 2020 13:24:36 +0100
Subject: [PATCH 04/40] some minor fixes and cleanup

---
 esmvalcore/cmor/_fixes/native6/era5.py | 52 +++++++++++++-------------
 esmvalcore/config-developer.yml        |  2 +-
 2 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index d5a1b6f850..220532f555 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -29,13 +29,13 @@ def _fix_frequency(self, cube):
         return cube
 
     def fix_metadata(self, cubes):
-        super.fix_metadata(cubes)
+        super().fix_metadata(cubes)
         for cube in cubes:
             self._fix_frequency(cube)
         return cubes
 
 class Hydrological(FixEra5):
-    """Fixes for accumulated hydrological variables."""
+    """Fixes for hydrological variables."""
 
     @staticmethod
     def _fix_units(cube):
@@ -44,7 +44,7 @@ def _fix_units(cube):
         return cube
 
     def fix_metadata(self, cubes):
-        super.fix_metadata(cubes)
+        super().fix_metadata(cubes)
         for cube in cubes:
             self._fix_units(cube)
         return cubes
@@ -53,7 +53,7 @@ class Radiation(FixEra5):
     """Fixes for accumulated radiation variables."""
 
     def fix_metadata(self, cubes):
-        super.fix_metadata(cubes)
+        super().fix_metadata(cubes)
         for cube in cubes:
             cube.attributes['positive'] = 'down'
         return cubes
@@ -91,12 +91,14 @@ class AllVars(FixEra5):
 
     def _fix_coordinates(self, cube):
         """Fix coordinates."""
-        # Make latitude increasing
-        cube = cube[..., ::-1, :]
-
-        # Make pressure_levels decreasing
-        if cube.coords('pressure_level'):
-            cube = cube[:, ::-1, ...]
+        # Fix coordinate increasing direction
+        slices = []
+        for coord in cube.coords():
+            if coord.var_name in ('latitude', 'pressure_level'):
+                slices.append(slice(None, None, -1))
+            else:
+                slices.append(slice(None))
+        cube = cube[tuple(slices)]
 
         # Add scalar height coordinates
         if 'height2m' in self.vardef.dimensions:
@@ -104,20 +106,19 @@ def _fix_coordinates(self, cube):
         if 'height10m' in self.vardef.dimensions:
             add_scalar_height_coord(cube, 10.)
 
-        for axis in 'T', 'X', 'Y', 'Z':
-            coord_def = self.vardef.coordinates.get(axis)
-            if coord_def:
-                coord = cube.coord(axis=axis)
-                if axis == 'T':
-                    coord.convert_units('days since 1850-1-1 00:00:00.0')
-                if axis == 'Z':
-                    coord.convert_units(coord_def.units)
-                coord.standard_name = coord_def.standard_name
-                coord.var_name = coord_def.out_name
-                coord.long_name = coord_def.long_name
-                coord.points = coord.core_points().astype('float64')
-                if len(coord.points) > 1:
-                    coord.guess_bounds()
+        # Fix coordinate units, dtypes and names
+        for coord_name, coord_def in self.vardef.coordinates.items():
+            coord = cube.coord(coord_name)
+            if coord_name == 'T':
+                coord.convert_units('days since 1850-1-1 00:00:00.0')
+            if coord_name == 'Z':
+                coord.convert_units(coord_def.units)
+            coord.standard_name = coord_def.standard_name
+            coord.var_name = coord_def.out_name
+            coord.long_name = coord_def.long_name
+            coord.points = coord.core_points().astype('float64')
+            if len(coord.points) > 1:
+                coord.guess_bounds()
 
         self._fix_monthly_time_coord(cube)
 
@@ -142,9 +143,6 @@ def _fix_monthly_time_coord(self, cube):
 
     def _fix_units(self, cube):
         """Fix units."""
-        if cube.units == '(0 - 1)':
-            # Correct dimensionless units to 1 from ecmwf format '(0 - 1)'
-            cube.units = 1
         if cube.units == 'm of water equivalent':
             # Correct units from m of water to kg of water per m2
             cube.units = 'kg m-2'
diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml
index 7b31a5de40..0ba57f78c3 100644
--- a/esmvalcore/config-developer.yml
+++ b/esmvalcore/config-developer.yml
@@ -129,7 +129,7 @@ OBS6:
   output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}'
   cmor_type: 'CMIP6'
 
-Native6:
+native6:
   cmor_strict: false
   input_dir:
     default: 'Tier{tier}/{dataset}'

From cc99f1c2f71fa3cefad9c1a048d368882b2102a8 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 8 Jan 2020 13:41:14 +0100
Subject: [PATCH 05/40] Fix units of hydrological variables

---
 esmvalcore/cmor/_fixes/native6/era5.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 220532f555..6d7e76143b 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -39,7 +39,7 @@ class Hydrological(FixEra5):
 
     @staticmethod
     def _fix_units(cube):
-        cube.units = cube.units * 'kg m-3'
+        cube.units = 'kg m-2'
         cube.data = cube.core_data() * 1000.
         return cube
 
@@ -58,7 +58,6 @@ def fix_metadata(self, cubes):
             cube.attributes['positive'] = 'down'
         return cubes
 
-
 class Evspsbl(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
@@ -143,11 +142,6 @@ def _fix_monthly_time_coord(self, cube):
 
     def _fix_units(self, cube):
         """Fix units."""
-        if cube.units == 'm of water equivalent':
-            # Correct units from m of water to kg of water per m2
-            cube.units = 'kg m-2'
-            cube.data = cube.core_data() * 1000.
-
         cube.convert_units(self.vardef.units)
 
     def fix_metadata(self, cubes):

From d5c7ec66bae65fc97f5f2b02618c4374d6ea648d Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 8 Jan 2020 17:07:49 +0100
Subject: [PATCH 06/40] Update get_start_end_year

---
 esmvalcore/_data_finder.py      | 75 ++++++++++++++-------------------
 esmvalcore/config-developer.yml |  2 +
 2 files changed, 34 insertions(+), 43 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index 49603f22d8..988281ad26 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -32,52 +32,41 @@ def find_files(dirnames, filenames):
 def get_start_end_year(filename):
     """Get the start and end year from a file name.
 
-    This works for filenames matching
-
-    *[-,_]YYYY*[-,_]YYYY*.*
-      or
-    *[-,_]YYYY*.*
-      or
-    YYYY*[-,_]*.*
-      or
-    YYYY*[-,_]YYYY*[-,_]*.*
-      or
-    YYYY*[-,_]*[-,_]YYYY*.* (Does this make sense? Is this worth catching?)
+    We assume that the filename *must* contain at least 1 year/date.
+    We look for four-to-eight-digit numbers.
+    If two potential dates are separated by either - or _,
+    we interpret this as the _daterange_ (startdate_enddate).
+    If no date range is present, but multiple potential dates are
+    present (e.g. including a version number), we assume that the last
+    number represents the date.
     """
-    name = os.path.splitext(filename)[0]
-
-    filename = name.split(os.sep)[-1]
-    filename_list = [elem.split('-') for elem in filename.split('_')]
-    filename_list = [elem for sublist in filename_list for elem in sublist]
-
-    pos_ydates = [elem.isdigit() and len(elem) >= 4 for elem in filename_list]
-    pos_ydates_l = list(pos_ydates)
-    pos_ydates_r = list(pos_ydates)
-
-    for ind, _ in enumerate(pos_ydates_l):
-        if ind != 0:
-            pos_ydates_l[ind] = (pos_ydates_l[ind - 1] and pos_ydates_l[ind])
-
-    for ind, _ in enumerate(pos_ydates_r):
-        if ind != 0:
-            pos_ydates_r[-ind - 1] = (pos_ydates_r[-ind]
-                                      and pos_ydates_r[-ind - 1])
-
-    dates = [
-        filename_list[ind] for ind, _ in enumerate(pos_ydates)
-        if pos_ydates_r[ind] or pos_ydates_l[ind]
-    ]
-
-    if len(dates) == 1:
-        start_year = int(dates[0][:4])
-        end_year = start_year
-    elif len(dates) == 2:
-        start_year, end_year = int(dates[0][:4]), int(dates[1][:4])
+    # Check for a block of two potential dates separated by _ or -
+    daterange = re.findall(r'([0-9]{4,8}[-_][0-9]{4,8})', filename)
+    if daterange:
+        start_date, end_date = re.findall(r'([0-9]{4,8})', daterange[0])
+        start_year = start_date[:4]
+        end_year = end_date[:4]
     else:
-        raise ValueError('Name {0} dates do not match a recognized '
-                         'pattern'.format(name))
+        dates = re.findall(r'([0-9]{4,8})', filename)
+        if not dates:
+            raise ValueError('Name {0} does not match a recognized '
+                             'pattern'.format(filename))
+        elif len(dates) == 1:
+            start_year = end_year = dates[0][:4]
+        else:
+            # Check for dates at start or end of filename
+            outerdates = re.findall(r'^[0-9]{4,8}|[0-9]{4,8}(?=\.)', filename)
+            if len(outerdates) == 1:
+                start_year = end_year = outerdates[0][:4]
+            else:
+                raise ValueError('Name {0} does not match a recognized '
+                                 'pattern'.format(filename))
+
+        # Interpret the first number as the single year
+        start_year = end_year = dates[0][:4]
 
-    return start_year, end_year
+    logger.debug("Found start_year %s and end_year %s", start_year, end_year)
+    return int(start_year), int(end_year)
 
 
 def select_files(filenames, start_year, end_year):
diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml
index 0ba57f78c3..1f47788e6a 100644
--- a/esmvalcore/config-developer.yml
+++ b/esmvalcore/config-developer.yml
@@ -133,8 +133,10 @@ native6:
   cmor_strict: false
   input_dir:
     default: 'Tier{tier}/{dataset}'
+    era5cli: 'Tier{tier}/{dataset}'
   input_file:
     default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc'
+    era5cli: 'era5_{era5_name}*{era5_freq}.nc'
   output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}'
   cmor_type: 'CMIP6'
 

From f3c512a8dedeae904b914a7235710e996b91eb3e Mon Sep 17 00:00:00 2001
From: Jaro Camphuijsen <jjecamphuijsen@gmail.com>
Date: Wed, 8 Jan 2020 17:11:27 +0100
Subject: [PATCH 07/40] add new tests for era5cli format and remove extra years
 in output file

---
 esmvalcore/config-developer.yml                   |  2 +-
 tests/unit/data_finder/test_get_start_end_year.py | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml
index 1f47788e6a..02e1b32d7c 100644
--- a/esmvalcore/config-developer.yml
+++ b/esmvalcore/config-developer.yml
@@ -137,7 +137,7 @@ native6:
   input_file:
     default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc'
     era5cli: 'era5_{era5_name}*{era5_freq}.nc'
-  output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}'
+  output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}'
   cmor_type: 'CMIP6'
 
 obs4mips:
diff --git a/tests/unit/data_finder/test_get_start_end_year.py b/tests/unit/data_finder/test_get_start_end_year.py
index 72004daf15..3ec73e7315 100644
--- a/tests/unit/data_finder/test_get_start_end_year.py
+++ b/tests/unit/data_finder/test_get_start_end_year.py
@@ -44,6 +44,13 @@ def test_one_year_at_the_start(self):
         self.assertEqual(1980, start)
         self.assertEqual(1980, end)
 
+    def test_one_year_in_the_middle(self):
+        """Test parse files with one year in the middle"""
+        start, end = get_start_end_year(
+            'var_control-1950_whatever.nc')
+        self.assertEqual(1950, start)
+        self.assertEqual(1950, end)
+
     def test_full_dates_at_the_start(self):
         """Test parse files with two dates at the start"""
         start, end = get_start_end_year('19800101-19811231_var_whatever.nc')
@@ -55,6 +62,13 @@ def test_one_fulldate_at_the_start(self):
         start, end = get_start_end_year('19800101_var_whatever.nc')
         self.assertEqual(1980, start)
         self.assertEqual(1980, end)
+        
+    def test_one_fulldate_in_the_middle(self):
+        """Test parse files with one date in the middle"""
+        start, end = get_start_end_year(
+            'var_control-19800101_whatever.nc')
+        self.assertEqual(1980, start)
+        self.assertEqual(1980, end)
 
     def test_start_and_date_in_name(self):
         """Test parse one date at the start and one in experiment's name"""

From 55083488a5dbd629c4273f6f20f3fc8b5c810e5e Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 8 Jan 2020 18:15:07 +0100
Subject: [PATCH 08/40] few more fixes in datafinder

---
 esmvalcore/_data_finder.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index 988281ad26..ecf8cf59fd 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -40,6 +40,10 @@ def get_start_end_year(filename):
     present (e.g. including a version number), we assume that the last
     number represents the date.
     """
+    # Strip only filename from full path (and discard extension?)
+    # filename = os.path.splitext(filename)[0]
+    filename = filename.split(os.sep)[-1]
+
     # Check for a block of two potential dates separated by _ or -
     daterange = re.findall(r'([0-9]{4,8}[-_][0-9]{4,8})', filename)
     if daterange:
@@ -48,6 +52,7 @@ def get_start_end_year(filename):
         end_year = end_date[:4]
     else:
         dates = re.findall(r'([0-9]{4,8})', filename)
+        print(dates)
         if not dates:
             raise ValueError('Name {0} does not match a recognized '
                              'pattern'.format(filename))
@@ -62,9 +67,6 @@ def get_start_end_year(filename):
                 raise ValueError('Name {0} does not match a recognized '
                                  'pattern'.format(filename))
 
-        # Interpret the first number as the single year
-        start_year = end_year = dates[0][:4]
-
     logger.debug("Found start_year %s and end_year %s", start_year, end_year)
     return int(start_year), int(end_year)
 

From 5e30684e5b9120263c9f91ce1394267f03b609f7 Mon Sep 17 00:00:00 2001
From: Jaro Camphuijsen <jjecamphuijsen@gmail.com>
Date: Thu, 9 Jan 2020 09:31:44 +0100
Subject: [PATCH 09/40] support long date format and remove extention from
 filename

---
 esmvalcore/_data_finder.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index ecf8cf59fd..f035df05b6 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -41,17 +41,17 @@ def get_start_end_year(filename):
     number represents the date.
     """
     # Strip only filename from full path (and discard extension?)
-    # filename = os.path.splitext(filename)[0]
+    filename = os.path.splitext(filename)[0]
     filename = filename.split(os.sep)[-1]
 
     # Check for a block of two potential dates separated by _ or -
-    daterange = re.findall(r'([0-9]{4,8}[-_][0-9]{4,8})', filename)
+    daterange = re.findall(r'([0-9]{4,12}[-_][0-9]{4,12})', filename)
     if daterange:
-        start_date, end_date = re.findall(r'([0-9]{4,8})', daterange[0])
+        start_date, end_date = re.findall(r'([0-9]{4,12})', daterange[0])
         start_year = start_date[:4]
         end_year = end_date[:4]
     else:
-        dates = re.findall(r'([0-9]{4,8})', filename)
+        dates = re.findall(r'([0-9]{4,12})', filename)
         print(dates)
         if not dates:
             raise ValueError('Name {0} does not match a recognized '
@@ -60,7 +60,7 @@ def get_start_end_year(filename):
             start_year = end_year = dates[0][:4]
         else:
             # Check for dates at start or end of filename
-            outerdates = re.findall(r'^[0-9]{4,8}|[0-9]{4,8}(?=\.)', filename)
+            outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$', filename)
             if len(outerdates) == 1:
                 start_year = end_year = outerdates[0][:4]
             else:

From ea721a8947c3214927418df159352d3370470e3e Mon Sep 17 00:00:00 2001
From: Jaro Camphuijsen <jjecamphuijsen@gmail.com>
Date: Thu, 9 Jan 2020 10:18:33 +0100
Subject: [PATCH 10/40] rewrite start_end_year test with pytest

---
 .../data_finder/test_get_start_end_year.py    | 111 +++++-------------
 1 file changed, 27 insertions(+), 84 deletions(-)

diff --git a/tests/unit/data_finder/test_get_start_end_year.py b/tests/unit/data_finder/test_get_start_end_year.py
index 3ec73e7315..cc4ee5e4a5 100644
--- a/tests/unit/data_finder/test_get_start_end_year.py
+++ b/tests/unit/data_finder/test_get_start_end_year.py
@@ -1,90 +1,33 @@
 """Unit tests for :func:`esmvalcore._data_finder.regrid._stock_cube`"""
 
-import unittest
+import pytest
 
 from esmvalcore._data_finder import get_start_end_year
 
-
-class TestGetStartEndYear(unittest.TestCase):
+FILENAME_CASES = [
+    ['var_whatever_1980-1981', 1980, 1981],
+    ['var_whatever_1980.nc', 1980, 1980],
+    ['var_whatever_19800101-19811231.nc1', 1980, 1981],
+    ['var_whatever_19800101.nc', 1980, 1980],
+    ['1980-1981_var_whatever.nc', 1980, 1981],
+    ['1980_var_whatever.nc', 1980, 1980],
+    ['var_control-1980_whatever.nc', 1980, 1980],
+    ['19800101-19811231_var_whatever.nc', 1980, 1981],
+    ['19800101_var_whatever.nc', 1980, 1980],
+    ['var_control-19800101_whatever.nc', 1980, 1980],
+    ['19800101_var_control-1950_whatever.nc', 1980, 1980],
+    ['var_control-1950_whatever_19800101.nc', 1980, 1980],
+]
+
+@pytest.mark.parametrize('case', FILENAME_CASES)
+def test_get_start_end_year(case):
     """Tests for get_start_end_year function"""
-
-    def test_years_at_the_end(self):
-        """Test parse files with two years at the end"""
-        start, end = get_start_end_year('var_whatever_1980-1981')
-        self.assertEqual(1980, start)
-        self.assertEqual(1981, end)
-
-    def test_one_year_at_the_end(self):
-        """Test parse files with one year at the end"""
-        start, end = get_start_end_year('var_whatever_1980.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1980, end)
-
-    def test_full_dates_at_the_end(self):
-        """Test parse files with two dates at the end"""
-        start, end = get_start_end_year('var_whatever_19800101-19811231.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1981, end)
-
-    def test_one_fulldate_at_the_end(self):
-        """Test parse files with one date at the end"""
-        start, end = get_start_end_year('var_whatever_19800101.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1980, end)
-
-    def test_years_at_the_start(self):
-        """Test parse files with two years at the start"""
-        start, end = get_start_end_year('1980-1981_var_whatever.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1981, end)
-
-    def test_one_year_at_the_start(self):
-        """Test parse files with one year at the start"""
-        start, end = get_start_end_year('1980_var_whatever.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1980, end)
-
-    def test_one_year_in_the_middle(self):
-        """Test parse files with one year in the middle"""
-        start, end = get_start_end_year(
-            'var_control-1950_whatever.nc')
-        self.assertEqual(1950, start)
-        self.assertEqual(1950, end)
-
-    def test_full_dates_at_the_start(self):
-        """Test parse files with two dates at the start"""
-        start, end = get_start_end_year('19800101-19811231_var_whatever.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1981, end)
-
-    def test_one_fulldate_at_the_start(self):
-        """Test parse files with one date at the start"""
-        start, end = get_start_end_year('19800101_var_whatever.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1980, end)
-        
-    def test_one_fulldate_in_the_middle(self):
-        """Test parse files with one date in the middle"""
-        start, end = get_start_end_year(
-            'var_control-19800101_whatever.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1980, end)
-
-    def test_start_and_date_in_name(self):
-        """Test parse one date at the start and one in experiment's name"""
-        start, end = get_start_end_year(
-            '19800101_var_control-1950_whatever.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1980, end)
-
-    def test_end_and_date_in_name(self):
-        """Test parse one date at the end and one in experiment's name"""
-        start, end = get_start_end_year(
-            'var_control-1950_whatever_19800101.nc')
-        self.assertEqual(1980, start)
-        self.assertEqual(1980, end)
-
-    def test_fails_if_no_date_present(self):
-        """Test raises if no date is present"""
-        with self.assertRaises(ValueError):
-            get_start_end_year('var_whatever')
+    filename, case_start, case_end = case
+    start, end = get_start_end_year(filename)
+    assert case_start == start
+    assert case_end == end
+
+def test_fails_if_no_date_present():
+    """Test raises if no date is present"""
+    with pytest.raises(ValueError):
+        get_start_end_year('var_whatever')

From 35caf44b5db21e4b7fcbbc14bc15c25a8964a3a2 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 9 Jan 2020 15:49:48 +0100
Subject: [PATCH 11/40] Resolve key/value confusion: vardef.coordinates has
 axis as key

---
 esmvalcore/cmor/_fixes/native6/era5.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 6d7e76143b..7c21a8bb18 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -106,11 +106,12 @@ def _fix_coordinates(self, cube):
             add_scalar_height_coord(cube, 10.)
 
         # Fix coordinate units, dtypes and names
-        for coord_name, coord_def in self.vardef.coordinates.items():
+        for coord_def in self.vardef.coordinates.values():
+            coord_name = coord_def.standard_name
             coord = cube.coord(coord_name)
-            if coord_name == 'T':
+            if coord_def.axis == 'T':
                 coord.convert_units('days since 1850-1-1 00:00:00.0')
-            if coord_name == 'Z':
+            if coord_def.axis == 'Z':
                 coord.convert_units(coord_def.units)
             coord.standard_name = coord_def.standard_name
             coord.var_name = coord_def.out_name

From db2a6389d1c17fc452007267588e6f5aa29f18d8 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Fri, 10 Jan 2020 11:32:02 +0100
Subject: [PATCH 12/40] Add note/todo on #413

---
 esmvalcore/cmor/_fixes/native6/era5.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 7c21a8bb18..2187029124 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -108,6 +108,11 @@ def _fix_coordinates(self, cube):
         # Fix coordinate units, dtypes and names
         for coord_def in self.vardef.coordinates.values():
             coord_name = coord_def.standard_name
+        # TODO: After merging #413
+        # the 2 lines above simplify to:
+        # for coord_name, coord_def in self.vardef.coordinates.items():
+        # and coord_def.axis == 'T' changes to coord_name == 'time'
+        # NOTE: perhaps we need a mapping between ERA5 and cmor coordinate names
             coord = cube.coord(coord_name)
             if coord_def.axis == 'T':
                 coord.convert_units('days since 1850-1-1 00:00:00.0')

From 9d3cf466e78b065a138568a2789b11943fd1b14f Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Fri, 10 Jan 2020 17:03:35 +0100
Subject: [PATCH 13/40] Add tests and resolve keyerror in CMOR_TABLES

---
 esmvalcore/cmor/table.py                      | 15 ++-
 .../cmor/_fixes/cmip6/test_cesm2.py           |  2 +-
 .../cmor/_fixes/native6/test_era5.py          | 97 +++++++++++++++++++
 3 files changed, 112 insertions(+), 2 deletions(-)
 create mode 100644 tests/integration/cmor/_fixes/native6/test_era5.py

diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py
index d876f9d062..31ea2bad55 100644
--- a/esmvalcore/cmor/table.py
+++ b/esmvalcore/cmor/table.py
@@ -11,6 +11,8 @@
 import json
 import logging
 import os
+import yaml
+from pathlib import Path
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +20,7 @@
 """dict of str, obj: CMOR info objects."""
 
 
-def read_cmor_tables(cfg_developer):
+def read_cmor_tables(cfg_developer=None):
     """Read cmor tables required in the configuration.
 
     Parameters
@@ -27,7 +29,13 @@ def read_cmor_tables(cfg_developer):
         Parsed config-developer file
 
     """
+    if cfg_developer is None:
+        cfg_file = Path(__file__).parent.parent / 'config-developer.yml'
+        with cfg_file.open() as file:
+            cfg_developer = yaml.safe_load(file)
+
     custom = CustomInfo()
+    CMOR_TABLES.clear()
     CMOR_TABLES['custom'] = custom
 
     for table in cfg_developer:
@@ -50,6 +58,8 @@ def read_cmor_tables(cfg_developer):
             )
 
 
+
+
 class CMIP6Info(object):
     """
     Class to read CMIP6-like data request.
@@ -170,6 +180,7 @@ def _assign_dimensions(self, var, generic_levels):
 
             var.coordinates[axis] = coord
 
+
     def _load_coordinates(self):
         self.coords = {}
         for json_file in glob.glob(
@@ -806,3 +817,5 @@ def _read_table_file(self, table_file, table=None):
                     continue
                 if not self._read_line():
                     return
+
+read_cmor_tables()
diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py
index fb753713f2..e7793ecabc 100644
--- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py
+++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py
@@ -15,7 +15,7 @@ def tas_cubes():
 
 
 def test_get_tas_fix():
-    fix = Fix.get_fixes('CMIP6', 'CESM2', 'tas')
+    fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'tas')
     assert fix == [Tas()]
 
 
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
new file mode 100644
index 0000000000..bb41467ec2
--- /dev/null
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -0,0 +1,97 @@
+"""Tests for the fixes of ERA5."""
+import iris
+import numpy as np
+import pytest
+from cf_units import Unit
+
+from esmvalcore.cmor._fixes.native6.era5 import Evspsbl, Pr, AllVars
+from esmvalcore.cmor.fix import Fix
+from esmvalcore.cmor.fix import fix_file
+
+def test_get_evspsbl_fix():
+    """Test whether the right fixes are gathered for a single variable."""
+    fix = Fix.get_fixes('native6', 'ERA5', 'E1hr', 'evspsbl')
+    assert fix == [AllVars(None), Evspsbl(None)]
+
+
+@pytest.fixture
+def pr_src_cube():
+    """Make dummy cube that looks like the ERA5 data."""
+    latitude = iris.coords.DimCoord(np.array([90., 0., -90.]),
+                                    standard_name='latitude',
+                                    long_name='latitude',
+                                    var_name='latitude',
+                                    units=Unit('degrees'))
+    longitude = iris.coords.DimCoord(np.array([0, 180, 359.75]),
+                                     standard_name='longitude',
+                                     long_name='longitude',
+                                     var_name='longitude',
+                                     units=Unit('degrees'),
+                                     circular=True)
+    time = iris.coords.DimCoord(np.arange(788928, 788931, dtype='int32'),
+                                standard_name='time',
+                                long_name='time',
+                                var_name='time',
+                                units=Unit('hours since 1900-01-01 00:00:00.0', calendar='gregorian'))
+    pr = iris.cube.Cube(np.zeros((3, 3, 3)),
+                        long_name='Total precipitation',
+                        var_name='tp',
+                        units=Unit('m'),
+                        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
+                        attributes= {'Conventions': 'CF-1.6',
+                                     'History': 'uninteresting info'})
+    return iris.cube.CubeList([pr])
+
+
+@pytest.fixture
+def pr_target_cube():
+    """Make dummy cube that looks like the CMORized data."""
+    # TODO: Verify that all cmor standards are accounted for in this
+    # cube. I addressed: lat/lon var_name; latitude direction;
+    # coordinate bounds; variable metadata from cmor table; ...
+    latitude = iris.coords.DimCoord(np.array([-90., 0., 90.]),
+                                    standard_name='latitude',
+                                    long_name='latitude',
+                                    var_name='lat',
+                                    units=Unit('degrees'),
+                                    bounds=-np.array([[-89.875, -90],
+                                                      [-0.125, 0.125],
+                                                      [90, 89.875]]))
+    longitude = iris.coords.DimCoord(np.array([0, 180, 359.75]),
+                                     standard_name='longitude',
+                                     long_name='longitude',
+                                     var_name='lon',
+                                     units=Unit('degrees'),
+                                     bounds=np.array([[-0.125, 0.125],
+                                                      [179.875, 180.125],
+                                                      [359.625, 359.875]]),
+                                     circular=True)
+    time = iris.coords.DimCoord(np.arange(788928, 788931, dtype='int32'),
+                                standard_name='time',
+                                long_name='time',
+                                var_name='time',
+                                units=Unit('hours since 1900-01-01 00:00:00.0', calendar='gregorian'),
+                                bounds=np.array([[788927.5, 788928.5],
+                                                 [788928.5, 788929.5],
+                                                 [788929.5, 788930.5]]))
+    pr = iris.cube.Cube(np.zeros((3, 3, 3)),
+                        long_name='Precipitation',
+                        var_name='pr',
+                        standard_name='precipitation_flux',
+                        units=Unit('kg m-2 s-1'),
+                        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
+                        attributes= {'Conventions': 'CF-1.5',
+                                     'History': 'uninteresting info',
+                                     'More': 'uninteresting stuff'})
+    return iris.cube.CubeList([pr])
+
+def test_cmorization(pr_src_cubes,  pr_target_cubes):
+    """Verify that cmorization results in the expected target cube."""
+    fix = Pr()
+    out_cubes = fix.fix_metadata(pr_src_cubes)
+    assert out_cubes == target_cubes
+
+# TODO:
+# - Make the dummy cube function more generic, such that it can be reused for multiple variables
+# - Make a pytest parameterize function to test for multiple variables, make sure to include all variables that have a specific class and 1 general variable.
+# - Include tests for monthly data and 3D variables (1 variable is enough)

From 812a6b87ca570a27bc4521dee8162d14083ba1e5 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 15 Jan 2020 13:21:32 +0100
Subject: [PATCH 14/40] Add fix for fx/orography

---
 esmvalcore/cmor/_fixes/native6/era5.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 2187029124..f86fb76be4 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -58,6 +58,20 @@ def fix_metadata(self, cubes):
             cube.attributes['positive'] = 'down'
         return cubes
 
+class Fx(FixERA5):
+    """Fixes for time invariant variables."""
+
+    def fix_metadata(self, cubes):
+        super().fix_metadata(cubes)
+        for cube in cubes:
+            if cube.var_name in ['zg', 'orog']:
+                # Divide by acceleration of gravity [m s-2],
+                # required for geopotential height, see:
+                # https://apps.ecmwf.int/codes/grib/param-db?id=129
+                cube.units = cube.units / 'm s-2'
+                cube.data = cube.core_data() / 9.80665
+        return cubes
+
 class Evspsbl(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
@@ -85,6 +99,9 @@ class Rsdt(Radiation, Accumulated):
 class Rls(Radiation):
     """Fixes for Rls."""
 
+class Orog(Fx):
+    """Fixes for orography"""
+
 class AllVars(FixEra5):
     """Fixes for all variables."""
 

From 39cfd59464645758067ef6b0109248e71738b41d Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 15 Jan 2020 13:51:08 +0100
Subject: [PATCH 15/40] AllVars after individual fixes

---
 esmvalcore/cmor/_fixes/fix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py
index 2358e02224..db7effa439 100644
--- a/esmvalcore/cmor/_fixes/fix.py
+++ b/esmvalcore/cmor/_fixes/fix.py
@@ -162,7 +162,7 @@ def get_fixes(project, dataset, mip, short_name):
 
             classes = inspect.getmembers(fixes_module, inspect.isclass)
             classes = dict((name.lower(), value) for name, value in classes)
-            for fix_name in ('allvars', short_name):
+            for fix_name in (short_name, 'allvars'):
                 try:
                     fixes.append(classes[fix_name](vardef))
                 except KeyError:

From f2c754e7259da307b424c1620f8002227a3afd04 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 15 Jan 2020 13:51:51 +0100
Subject: [PATCH 16/40] Extend test

---
 .../cmor/_fixes/native6/test_era5.py          | 100 ++++++++++++------
 1 file changed, 67 insertions(+), 33 deletions(-)

diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index bb41467ec2..7eb0251a96 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -4,19 +4,20 @@
 import pytest
 from cf_units import Unit
 
-from esmvalcore.cmor._fixes.native6.era5 import Evspsbl, Pr, AllVars
-from esmvalcore.cmor.fix import Fix
-from esmvalcore.cmor.fix import fix_file
+from esmvalcore.cmor._fixes.native6.era5 import AllVars, Evspsbl
+from esmvalcore.cmor.fix import Fix, fix_metadata
+from esmvalcore.cmor.table import CMOR_TABLES
+
 
 def test_get_evspsbl_fix():
     """Test whether the right fixes are gathered for a single variable."""
     fix = Fix.get_fixes('native6', 'ERA5', 'E1hr', 'evspsbl')
-    assert fix == [AllVars(None), Evspsbl(None)]
+    assert fix == [Evspsbl(None), AllVars(None)]
 
 
-@pytest.fixture
-def pr_src_cube():
+def make_src_cubes(long_name, var_name, units, ndims=3, mip='E1hr'):
     """Make dummy cube that looks like the ERA5 data."""
+    # TODO: Make 2d and 4d dimensions possible
     latitude = iris.coords.DimCoord(np.array([90., 0., -90.]),
                                     standard_name='latitude',
                                     long_name='latitude',
@@ -33,22 +34,23 @@ def pr_src_cube():
                                 long_name='time',
                                 var_name='time',
                                 units=Unit('hours since 1900-01-01 00:00:00.0', calendar='gregorian'))
-    pr = iris.cube.Cube(np.zeros((3, 3, 3)),
-                        long_name='Total precipitation',
-                        var_name='tp',
-                        units=Unit('m'),
-                        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
-                        attributes= {'Conventions': 'CF-1.6',
-                                     'History': 'uninteresting info'})
-    return iris.cube.CubeList([pr])
+
+    cube = iris.cube.Cube(np.zeros((3, 3, 3)),
+                          long_name=long_name,
+                          var_name=var_name,
+                          units=Unit(units),
+                          dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
+                          attributes={'Conventions': 'CF-1.6',
+                                      'History': 'uninteresting info'})
+    return iris.cube.CubeList([cube])
 
 
-@pytest.fixture
-def pr_target_cube():
+def make_target_cubes(long_name, var_name, standard_name, units, ndims=3, mip='E1hr'):
     """Make dummy cube that looks like the CMORized data."""
     # TODO: Verify that all cmor standards are accounted for in this
     # cube. I addressed: lat/lon var_name; latitude direction;
     # coordinate bounds; variable metadata from cmor table; ...
+    # TODO: Make 2d and 4d dimensions possible
     latitude = iris.coords.DimCoord(np.array([-90., 0., 90.]),
                                     standard_name='latitude',
                                     long_name='latitude',
@@ -74,24 +76,56 @@ def pr_target_cube():
                                 bounds=np.array([[788927.5, 788928.5],
                                                  [788928.5, 788929.5],
                                                  [788929.5, 788930.5]]))
-    pr = iris.cube.Cube(np.zeros((3, 3, 3)),
-                        long_name='Precipitation',
-                        var_name='pr',
-                        standard_name='precipitation_flux',
-                        units=Unit('kg m-2 s-1'),
-                        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
-                        attributes= {'Conventions': 'CF-1.5',
-                                     'History': 'uninteresting info',
-                                     'More': 'uninteresting stuff'})
-    return iris.cube.CubeList([pr])
+    cube = iris.cube.Cube(np.zeros((3, 3, 3)),
+                          long_name=long_name,
+                          var_name=var_name,
+                          standard_name=standard_name,
+                          units=Unit(units),
+                          dim_coords_and_dims=[(time, 0), (latitude, 1),   (longitude, 2)],
+                          attributes={'Conventions': 'CF-1.5',
+                                      'History': 'uninteresting info',
+                                      'More': 'uninteresting stuff'})
+    return [cube]
+
+
+variables = [
+    # short_name, mip, era5_units, ndims
+    ['pr', 'E1hr', 'm', 3],
+    ['evspsbl', 'E1hr', 'm', 3],
+    # ['mrro', 'E1hr', 'm', 3],
+    # ['prsn', 'E1hr', 'm of water equivalent', 3],
+    # ['evspsblpot', 'E1hr', 'm', 3],
+    # ['rss', 'E1hr', 'J m**-2', 3],
+    # ['rsds', 'E1hr', 'J m**-2', 3],
+    # ['rsdt', 'E1hr', 'J m**-2', 3],
+    # ['rls', 'E1hr', 'W m**-2', 3], # variables with explicit fixes
+    # ['uas', 'E1hr', 'm s**-1', 3], # a variable without explicit fixes
+    # ['pr', 'Amon', 'm', 3], # a monthly variable
+    # ['ua', 'E1hr', 'm s**-1', 4], # a 4d variable
+    # ['orog', 'Fx', 'm**2 s**-2'] # ?? # a funky 2D variable
+]
 
-def test_cmorization(pr_src_cubes,  pr_target_cubes):
+
+@pytest.mark.parametrize('variable', variables)
+def test_cmorization(variable):
     """Verify that cmorization results in the expected target cube."""
-    fix = Pr()
-    out_cubes = fix.fix_metadata(pr_src_cubes)
-    assert out_cubes == target_cubes
+    short_name, mip, era5_units, ndims = variable
+    project, dataset = 'native6', 'era5'
+
+    # Look up variable definition in CMOR table
+    cmor_table = CMOR_TABLES[project]
+    vardef = cmor_table.get_variable(mip, short_name)
+    long_name = vardef.long_name
+    var_name = short_name # vardef.cmor_name after #391?
+    standard_name = vardef.standard_name
+    units = vardef.units
+
+    src_cubes = make_src_cubes('era5_long_name', 'era5_var_name', era5_units)
+    target_cubes = make_target_cubes(long_name, var_name, standard_name, units)
+    out_cubes = fix_metadata(src_cubes, short_name, project, dataset, mip)
+    print(out_cubes[0])
+    print(target_cubes[0])
+    assert out_cubes[0] == target_cubes[0]
 
 # TODO:
-# - Make the dummy cube function more generic, such that it can be reused for multiple variables
-# - Make a pytest parameterize function to test for multiple variables, make sure to include all variables that have a specific class and 1 general variable.
-# - Include tests for monthly data and 3D variables (1 variable is enough)
+# The test now fails because AllVars is fixed before FixUnits.

From b5ac5f7b34ed321dff16554c8a37155673504720 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 15 Jan 2020 13:52:07 +0100
Subject: [PATCH 17/40] fix typo

---
 esmvalcore/cmor/_fixes/native6/era5.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index f86fb76be4..936080a1bb 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -39,7 +39,7 @@ class Hydrological(FixEra5):
 
     @staticmethod
     def _fix_units(cube):
-        cube.units = 'kg m-2'
+        cube.units = 'kg m-2 s-1'
         cube.data = cube.core_data() * 1000.
         return cube
 
@@ -58,7 +58,7 @@ def fix_metadata(self, cubes):
             cube.attributes['positive'] = 'down'
         return cubes
 
-class Fx(FixERA5):
+class Fx(FixEra5):
     """Fixes for time invariant variables."""
 
     def fix_metadata(self, cubes):

From 2c291f6b9c767272b36dcc1ee79e47579225282d Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 15 Jan 2020 16:57:20 +0100
Subject: [PATCH 18/40] Extend tests

---
 esmvalcore/cmor/_fixes/native6/era5.py        |  19 ++-
 esmvalcore/cmor/table.py                      |   3 +
 .../cmor/_fixes/native6/test_era5.py          | 135 ++++++++++--------
 3 files changed, 84 insertions(+), 73 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 936080a1bb..a0d8853c33 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -122,26 +122,23 @@ def _fix_coordinates(self, cube):
         if 'height10m' in self.vardef.dimensions:
             add_scalar_height_coord(cube, 10.)
 
-        # Fix coordinate units, dtypes and names
+
         for coord_def in self.vardef.coordinates.values():
-            coord_name = coord_def.standard_name
-        # TODO: After merging #413
-        # the 2 lines above simplify to:
-        # for coord_name, coord_def in self.vardef.coordinates.items():
-        # and coord_def.axis == 'T' changes to coord_name == 'time'
-        # NOTE: perhaps we need a mapping between ERA5 and cmor coordinate names
-            coord = cube.coord(coord_name)
-            if coord_def.axis == 'T':
+            axis = coord_def.axis
+            coord = cube.coord(axis=axis)
+            if axis == 'T':
                 coord.convert_units('days since 1850-1-1 00:00:00.0')
-            if coord_def.axis == 'Z':
+            if axis == 'Z':
                 coord.convert_units(coord_def.units)
             coord.standard_name = coord_def.standard_name
             coord.var_name = coord_def.out_name
             coord.long_name = coord_def.long_name
             coord.points = coord.core_points().astype('float64')
-            if len(coord.points) > 1:
+            print(coord_def)
+            if len(coord.points) > 1 and coord_def.must_have_bounds == "yes":
                 coord.guess_bounds()
 
+
         self._fix_monthly_time_coord(cube)
 
         return cube
diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py
index ee9ceaacda..51fbfed2ec 100644
--- a/esmvalcore/cmor/table.py
+++ b/esmvalcore/cmor/table.py
@@ -496,6 +496,8 @@ def __init__(self, name):
         """Minimum allowed value"""
         self.valid_max = ""
         """Maximum allowed value"""
+        self.must_have_bounds = ""
+        """Whether bounds are required on this dimension"""
 
     def read_json(self, json_data):
         """
@@ -523,6 +525,7 @@ def read_json(self, json_data):
         self.valid_min = self._read_json_variable('valid_min')
         self.valid_max = self._read_json_variable('valid_max')
         self.requested = self._read_json_list_variable('requested')
+        self.must_have_bounds = self._read_json_variable('must_have_bounds')
 
 
 class CMIP5Info(object):
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index 7eb0251a96..411352759d 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -15,7 +15,7 @@ def test_get_evspsbl_fix():
     assert fix == [Evspsbl(None), AllVars(None)]
 
 
-def make_src_cubes(long_name, var_name, units, ndims=3, mip='E1hr'):
+def make_src_cubes(units, ndims=3, mip='E1hr'):
     """Make dummy cube that looks like the ERA5 data."""
     # TODO: Make 2d and 4d dimensions possible
     latitude = iris.coords.DimCoord(np.array([90., 0., -90.]),
@@ -33,58 +33,80 @@ def make_src_cubes(long_name, var_name, units, ndims=3, mip='E1hr'):
                                 standard_name='time',
                                 long_name='time',
                                 var_name='time',
-                                units=Unit('hours since 1900-01-01 00:00:00.0', calendar='gregorian'))
+                                units=Unit('hours since 1900-01-01'
+                                           '00:00:00.0', calendar='gregorian'))
 
     cube = iris.cube.Cube(np.zeros((3, 3, 3)),
-                          long_name=long_name,
-                          var_name=var_name,
+                          long_name='random_long_name',
+                          var_name='random_var_name',
                           units=Unit(units),
                           dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
-                          attributes={'Conventions': 'CF-1.6',
-                                      'History': 'uninteresting info'})
+                        )
     return iris.cube.CubeList([cube])
 
 
-def make_target_cubes(long_name, var_name, standard_name, units, ndims=3, mip='E1hr'):
+def make_target_cubes(project, mip, short_name):
     """Make dummy cube that looks like the CMORized data."""
-    # TODO: Verify that all cmor standards are accounted for in this
-    # cube. I addressed: lat/lon var_name; latitude direction;
-    # coordinate bounds; variable metadata from cmor table; ...
-    # TODO: Make 2d and 4d dimensions possible
+    # Look up variable definition in CMOR table
+    cmor_table = CMOR_TABLES[project]
+    vardef = cmor_table.get_variable(mip, short_name)
+
+    # Make lat/lon/time coordinates
     latitude = iris.coords.DimCoord(np.array([-90., 0., 90.]),
                                     standard_name='latitude',
-                                    long_name='latitude',
+                                    long_name='Latitude',
                                     var_name='lat',
-                                    units=Unit('degrees'),
-                                    bounds=-np.array([[-89.875, -90],
-                                                      [-0.125, 0.125],
-                                                      [90, 89.875]]))
+                                    units=Unit('degrees_north'),
+                                    bounds=np.array([[-90., -45.],
+                                                     [-45., 45.],
+                                                     [45., 90.]]))
     longitude = iris.coords.DimCoord(np.array([0, 180, 359.75]),
                                      standard_name='longitude',
-                                     long_name='longitude',
+                                     long_name='Longitude',
                                      var_name='lon',
-                                     units=Unit('degrees'),
-                                     bounds=np.array([[-0.125, 0.125],
-                                                      [179.875, 180.125],
-                                                      [359.625, 359.875]]),
+                                     units=Unit('degrees_east'),
+                                     bounds=np.array([[-0.125, 90.],
+                                                      [90., 269.875],
+                                                      [269.875, 359.875]]),
                                      circular=True)
-    time = iris.coords.DimCoord(np.arange(788928, 788931, dtype='int32'),
+    bounds = None if 'time1' in vardef.dimensions else np.array([
+        [51133.9791667, 51134.0208333],
+        [51134.0208333, 51134.0625],
+        [51134.0625, 51134.1041667]])
+    time = iris.coords.DimCoord(np.array([51134.0, 51134.0416667,
+                                          51134.0833333], dtype=float),
                                 standard_name='time',
                                 long_name='time',
                                 var_name='time',
-                                units=Unit('hours since 1900-01-01 00:00:00.0', calendar='gregorian'),
-                                bounds=np.array([[788927.5, 788928.5],
-                                                 [788928.5, 788929.5],
-                                                 [788929.5, 788930.5]]))
-    cube = iris.cube.Cube(np.zeros((3, 3, 3)),
-                          long_name=long_name,
-                          var_name=var_name,
-                          standard_name=standard_name,
-                          units=Unit(units),
-                          dim_coords_and_dims=[(time, 0), (latitude, 1),   (longitude, 2)],
-                          attributes={'Conventions': 'CF-1.5',
-                                      'History': 'uninteresting info',
-                                      'More': 'uninteresting stuff'})
+                                units=Unit('days since 1850-1-1 00:00:00',
+                                           calendar='gregorian'),
+                                bounds=bounds)
+
+    # Make dummy cube that's the cmor equivalent of the era5 dummy cube.
+    attributes = {}
+    if vardef.positive:
+        attributes['positive'] = vardef.positive
+
+    cube = iris.cube.Cube(np.zeros((3, 3, 3), dtype='float32'),
+                          long_name=vardef.long_name,
+                          var_name = short_name, # vardef.cmor_name after #391?,
+                          standard_name = vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0), (latitude, 1),
+                                               (longitude, 2)],
+                          attributes = attributes)
+
+    # Add auxiliary height coordinate for certain variables
+    if short_name in ['uas', 'vas', 'tas', 'tasmin', 'tasmax']:
+        value = 10. if short_name in ['uas', 'vas'] else 2.
+        aux_coord = iris.coords.AuxCoord([value],
+                                         long_name="height",
+                                         standard_name="height",
+                                         units=Unit('m'),
+                                         var_name="height",
+                                         attributes={'positive': 'up'})
+        cube.add_aux_coord(aux_coord, ())
+
     return [cube]
 
 
@@ -92,16 +114,16 @@ def make_target_cubes(long_name, var_name, standard_name, units, ndims=3, mip='E
     # short_name, mip, era5_units, ndims
     ['pr', 'E1hr', 'm', 3],
     ['evspsbl', 'E1hr', 'm', 3],
-    # ['mrro', 'E1hr', 'm', 3],
-    # ['prsn', 'E1hr', 'm of water equivalent', 3],
-    # ['evspsblpot', 'E1hr', 'm', 3],
-    # ['rss', 'E1hr', 'J m**-2', 3],
-    # ['rsds', 'E1hr', 'J m**-2', 3],
-    # ['rsdt', 'E1hr', 'J m**-2', 3],
-    # ['rls', 'E1hr', 'W m**-2', 3], # variables with explicit fixes
-    # ['uas', 'E1hr', 'm s**-1', 3], # a variable without explicit fixes
+    ['mrro', 'E1hr', 'm', 3],
+    ['prsn', 'E1hr', 'm of water equivalent', 3],
+    ['evspsblpot', 'E1hr', 'm', 3],
+    ['rss', 'E1hr', 'J m**-2', 3],
+    ['rsds', 'E1hr', 'J m**-2', 3],
+    ['rsdt', 'E1hr', 'J m**-2', 3],
+    ['rls', 'E1hr', 'W m**-2', 3], # variables with explicit fixes
+    ['uas', 'E1hr', 'm s**-1', 3], # a variable without explicit fixes
     # ['pr', 'Amon', 'm', 3], # a monthly variable
-    # ['ua', 'E1hr', 'm s**-1', 4], # a 4d variable
+    # ['ua', 'E1hr', 'm s**-1', 4], # a 4d variable (we decided not to do this now)
     # ['orog', 'Fx', 'm**2 s**-2'] # ?? # a funky 2D variable
 ]
 
@@ -110,22 +132,11 @@ def make_target_cubes(long_name, var_name, standard_name, units, ndims=3, mip='E
 def test_cmorization(variable):
     """Verify that cmorization results in the expected target cube."""
     short_name, mip, era5_units, ndims = variable
-    project, dataset = 'native6', 'era5'
 
-    # Look up variable definition in CMOR table
-    cmor_table = CMOR_TABLES[project]
-    vardef = cmor_table.get_variable(mip, short_name)
-    long_name = vardef.long_name
-    var_name = short_name # vardef.cmor_name after #391?
-    standard_name = vardef.standard_name
-    units = vardef.units
-
-    src_cubes = make_src_cubes('era5_long_name', 'era5_var_name', era5_units)
-    target_cubes = make_target_cubes(long_name, var_name, standard_name, units)
-    out_cubes = fix_metadata(src_cubes, short_name, project, dataset, mip)
-    print(out_cubes[0])
-    print(target_cubes[0])
-    assert out_cubes[0] == target_cubes[0]
-
-# TODO:
-# The test now fails because AllVars is fixed before FixUnits.
+    src_cubes = make_src_cubes(era5_units, ndims=3, mip='E1hr')
+    target_cubes = make_target_cubes('native6', mip, short_name)
+    out_cubes = fix_metadata(src_cubes, short_name, 'native6', 'era5', mip)
+
+    print(out_cubes[0].xml()) # for testing purposes
+    print(target_cubes[0].xml()) # during development
+    assert out_cubes[0].xml() == target_cubes[0].xml()

From 8951b54466c601181becb7c631fd21db1dc70b20 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 16 Jan 2020 12:51:25 +0100
Subject: [PATCH 19/40] add monthly mip to test

---
 esmvalcore/cmor/_fixes/native6/era5.py        |  1 -
 .../cmor/_fixes/native6/test_era5.py          | 99 ++++++++++++-------
 2 files changed, 64 insertions(+), 36 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index a0d8853c33..f874423320 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -134,7 +134,6 @@ def _fix_coordinates(self, cube):
             coord.var_name = coord_def.out_name
             coord.long_name = coord_def.long_name
             coord.points = coord.core_points().astype('float64')
-            print(coord_def)
             if len(coord.points) > 1 and coord_def.must_have_bounds == "yes":
                 coord.guess_bounds()
 
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index 411352759d..a9195b1202 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -15,9 +15,20 @@ def test_get_evspsbl_fix():
     assert fix == [Evspsbl(None), AllVars(None)]
 
 
-def make_src_cubes(units, ndims=3, mip='E1hr'):
+def make_src_cubes(units, mip='E1hr'):
     """Make dummy cube that looks like the ERA5 data."""
-    # TODO: Make 2d and 4d dimensions possible
+
+    # Adapt data and time coordinate for different mips
+    data = np.arange(27).reshape(3, 3, 3)
+    if mip == 'E1hr':
+        timestamps = [788928, 788929, 788930]
+    elif mip == 'Amon':
+        timestamps = [788928, 789672, 790344]
+    elif mip == 'Fx':
+        timestamps = [788928]
+        data = np.arange(9).reshape(1, 3, 3)
+
+    # Create coordinates
     latitude = iris.coords.DimCoord(np.array([90., 0., -90.]),
                                     standard_name='latitude',
                                     long_name='latitude',
@@ -29,19 +40,22 @@ def make_src_cubes(units, ndims=3, mip='E1hr'):
                                      var_name='longitude',
                                      units=Unit('degrees'),
                                      circular=True)
-    time = iris.coords.DimCoord(np.arange(788928, 788931, dtype='int32'),
+    time = iris.coords.DimCoord(np.array(timestamps, dtype='int32'),
                                 standard_name='time',
                                 long_name='time',
                                 var_name='time',
-                                units=Unit('hours since 1900-01-01'
-                                           '00:00:00.0', calendar='gregorian'))
-
-    cube = iris.cube.Cube(np.zeros((3, 3, 3)),
-                          long_name='random_long_name',
-                          var_name='random_var_name',
-                          units=Unit(units),
-                          dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
-                        )
+                                units=Unit(
+                                    'hours since 1900-01-01'
+                                    '00:00:00.0',
+                                    calendar='gregorian'))
+
+    cube = iris.cube.Cube(
+        data,
+        long_name='random_long_name',
+        var_name='random_var_name',
+        units=Unit(units),
+        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
+    )
     return iris.cube.CubeList([cube])
 
 
@@ -51,14 +65,34 @@ def make_target_cubes(project, mip, short_name):
     cmor_table = CMOR_TABLES[project]
     vardef = cmor_table.get_variable(mip, short_name)
 
+    # Determine dimensions of the cube and fill with dummy data
+    data = np.arange(27).reshape(3, 3, 3)[:, ::-1, :]
+    bounds = None
+    if mip == 'E1hr':
+        timestamps = [51134.0, 51134.0416667, 51134.0833333]
+        if not 'time1' in vardef.dimensions:
+            bounds = np.array([[51133.9791667, 51134.0208333],
+                               [51134.0208333, 51134.0625],
+                               [51134.0625, 51134.1041667]])
+    elif mip == 'Amon':
+        timestamps = [51134.0, 51134.0416667, 51134.0833333]
+        if not 'time1' in vardef.dimensions:
+            bounds = np.array([[51133.9791667, 51134.0208333],
+                               [51134.0208333, 51134.0625],
+                               [51134.0625, 51134.1041667]])
+    elif mip == 'Fx':
+        data = np.arange(9).reshape(1, 3, 3)[:, ::-1, :]
+        timestamps = [51134.0]
+        if not 'time1' in vardef.dimensions:
+            bounds = np.array([[51133.9791667, 51134.0208333]])
+
     # Make lat/lon/time coordinates
     latitude = iris.coords.DimCoord(np.array([-90., 0., 90.]),
                                     standard_name='latitude',
                                     long_name='Latitude',
                                     var_name='lat',
                                     units=Unit('degrees_north'),
-                                    bounds=np.array([[-90., -45.],
-                                                     [-45., 45.],
+                                    bounds=np.array([[-90., -45.], [-45., 45.],
                                                      [45., 90.]]))
     longitude = iris.coords.DimCoord(np.array([0, 180, 359.75]),
                                      standard_name='longitude',
@@ -69,12 +103,7 @@ def make_target_cubes(project, mip, short_name):
                                                       [90., 269.875],
                                                       [269.875, 359.875]]),
                                      circular=True)
-    bounds = None if 'time1' in vardef.dimensions else np.array([
-        [51133.9791667, 51134.0208333],
-        [51134.0208333, 51134.0625],
-        [51134.0625, 51134.1041667]])
-    time = iris.coords.DimCoord(np.array([51134.0, 51134.0416667,
-                                          51134.0833333], dtype=float),
+    time = iris.coords.DimCoord(np.array(timestamps, dtype=float),
                                 standard_name='time',
                                 long_name='time',
                                 var_name='time',
@@ -87,14 +116,14 @@ def make_target_cubes(project, mip, short_name):
     if vardef.positive:
         attributes['positive'] = vardef.positive
 
-    cube = iris.cube.Cube(np.zeros((3, 3, 3), dtype='float32'),
-                          long_name=vardef.long_name,
-                          var_name = short_name, # vardef.cmor_name after #391?,
-                          standard_name = vardef.standard_name,
-                          units=Unit(vardef.units),
-                          dim_coords_and_dims=[(time, 0), (latitude, 1),
-                                               (longitude, 2)],
-                          attributes = attributes)
+    cube = iris.cube.Cube(
+        data.astype('float32'),
+        long_name=vardef.long_name,
+        var_name=short_name,  # vardef.cmor_name after #391?,
+        standard_name=vardef.standard_name,
+        units=Unit(vardef.units),
+        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
+        attributes=attributes)
 
     # Add auxiliary height coordinate for certain variables
     if short_name in ['uas', 'vas', 'tas', 'tasmin', 'tasmax']:
@@ -120,11 +149,11 @@ def make_target_cubes(project, mip, short_name):
     ['rss', 'E1hr', 'J m**-2', 3],
     ['rsds', 'E1hr', 'J m**-2', 3],
     ['rsdt', 'E1hr', 'J m**-2', 3],
-    ['rls', 'E1hr', 'W m**-2', 3], # variables with explicit fixes
-    ['uas', 'E1hr', 'm s**-1', 3], # a variable without explicit fixes
-    # ['pr', 'Amon', 'm', 3], # a monthly variable
+    ['rls', 'E1hr', 'W m**-2', 3],  # variables with explicit fixes
+    ['uas', 'E1hr', 'm s**-1', 3],  # a variable without explicit fixes
+    ['pr', 'Amon', 'm', 3],  # a monthly variable
     # ['ua', 'E1hr', 'm s**-1', 4], # a 4d variable (we decided not to do this now)
-    # ['orog', 'Fx', 'm**2 s**-2'] # ?? # a funky 2D variable
+    ['orog', 'Fx', 'm**2 s**-2', 2]  # a 2D variable (but keep time coord)
 ]
 
 
@@ -133,10 +162,10 @@ def test_cmorization(variable):
     """Verify that cmorization results in the expected target cube."""
     short_name, mip, era5_units, ndims = variable
 
-    src_cubes = make_src_cubes(era5_units, ndims=3, mip='E1hr')
+    src_cubes = make_src_cubes(era5_units, mip='E1hr')
     target_cubes = make_target_cubes('native6', mip, short_name)
     out_cubes = fix_metadata(src_cubes, short_name, 'native6', 'era5', mip)
 
-    print(out_cubes[0].xml()) # for testing purposes
-    print(target_cubes[0].xml()) # during development
+    print(out_cubes[0].xml())  # for testing purposes
+    print(target_cubes[0].xml())  # during development
     assert out_cubes[0].xml() == target_cubes[0].xml()

From bf060b9a72fc0e18a8e0c0a5cb7136d258b30952 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 16 Jan 2020 16:06:23 +0100
Subject: [PATCH 20/40] Finish writing test

---
 esmvalcore/cmor/_fixes/native6/era5.py        | 26 ++++++----
 .../cmor/_fixes/native6/test_era5.py          | 50 ++++++++++---------
 2 files changed, 41 insertions(+), 35 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index f874423320..5dde0ab9da 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -61,17 +61,6 @@ def fix_metadata(self, cubes):
 class Fx(FixEra5):
     """Fixes for time invariant variables."""
 
-    def fix_metadata(self, cubes):
-        super().fix_metadata(cubes)
-        for cube in cubes:
-            if cube.var_name in ['zg', 'orog']:
-                # Divide by acceleration of gravity [m s-2],
-                # required for geopotential height, see:
-                # https://apps.ecmwf.int/codes/grib/param-db?id=129
-                cube.units = cube.units / 'm s-2'
-                cube.data = cube.core_data() / 9.80665
-        return cubes
-
 class Evspsbl(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
@@ -102,6 +91,21 @@ class Rls(Radiation):
 class Orog(Fx):
     """Fixes for orography"""
 
+    @staticmethod
+    def _fix_units(cube):
+        # Divide by acceleration of gravity [m s-2],
+        # required for geopotential height, see:
+        # https://apps.ecmwf.int/codes/grib/param-db?id=129
+        cube.units = cube.units / 'm s-2'
+        cube.data = cube.core_data() / 9.80665
+        return cube
+
+    def fix_metadata(self, cubes):
+        super().fix_metadata(cubes)
+        for cube in cubes:
+            self._fix_units(cube)
+        return cubes
+
 class AllVars(FixEra5):
     """Fixes for all variables."""
 
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index a9195b1202..a613f0f2d9 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -21,11 +21,11 @@ def make_src_cubes(units, mip='E1hr'):
     # Adapt data and time coordinate for different mips
     data = np.arange(27).reshape(3, 3, 3)
     if mip == 'E1hr':
-        timestamps = [788928, 788929, 788930]
+        timestamps = [788928, 788929, 788930] # 3 consecutive hours
     elif mip == 'Amon':
-        timestamps = [788928, 789672, 790344]
+        timestamps = [788928, 789672, 790344] # 3 consecutive months
     elif mip == 'Fx':
-        timestamps = [788928]
+        timestamps = [788928]                 # 1 single timestamp
         data = np.arange(9).reshape(1, 3, 3)
 
     # Create coordinates
@@ -53,7 +53,7 @@ def make_src_cubes(units, mip='E1hr'):
         data,
         long_name='random_long_name',
         var_name='random_var_name',
-        units=Unit(units),
+        units=units,
         dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
     )
     return iris.cube.CubeList([cube])
@@ -65,7 +65,7 @@ def make_target_cubes(project, mip, short_name):
     cmor_table = CMOR_TABLES[project]
     vardef = cmor_table.get_variable(mip, short_name)
 
-    # Determine dimensions of the cube and fill with dummy data
+    # Make up time dimension and data (shape)
     data = np.arange(27).reshape(3, 3, 3)[:, ::-1, :]
     bounds = None
     if mip == 'E1hr':
@@ -75,11 +75,11 @@ def make_target_cubes(project, mip, short_name):
                                [51134.0208333, 51134.0625],
                                [51134.0625, 51134.1041667]])
     elif mip == 'Amon':
-        timestamps = [51134.0, 51134.0416667, 51134.0833333]
+        timestamps = [51149.5, 51179.0, 51208.5]
         if not 'time1' in vardef.dimensions:
-            bounds = np.array([[51133.9791667, 51134.0208333],
-                               [51134.0208333, 51134.0625],
-                               [51134.0625, 51134.1041667]])
+            bounds = np.array([[51134.0, 51165.0],
+		                       [51165.0, 51193.0],
+		                       [51193.0, 51224.0]])
     elif mip == 'Fx':
         data = np.arange(9).reshape(1, 3, 3)[:, ::-1, :]
         timestamps = [51134.0]
@@ -141,29 +141,31 @@ def make_target_cubes(project, mip, short_name):
 
 variables = [
     # short_name, mip, era5_units, ndims
-    ['pr', 'E1hr', 'm', 3],
-    ['evspsbl', 'E1hr', 'm', 3],
-    ['mrro', 'E1hr', 'm', 3],
-    ['prsn', 'E1hr', 'm of water equivalent', 3],
-    ['evspsblpot', 'E1hr', 'm', 3],
-    ['rss', 'E1hr', 'J m**-2', 3],
-    ['rsds', 'E1hr', 'J m**-2', 3],
-    ['rsdt', 'E1hr', 'J m**-2', 3],
-    ['rls', 'E1hr', 'W m**-2', 3],  # variables with explicit fixes
-    ['uas', 'E1hr', 'm s**-1', 3],  # a variable without explicit fixes
-    ['pr', 'Amon', 'm', 3],  # a monthly variable
-    # ['ua', 'E1hr', 'm s**-1', 4], # a 4d variable (we decided not to do this now)
-    ['orog', 'Fx', 'm**2 s**-2', 2]  # a 2D variable (but keep time coord)
+    ['pr', 'E1hr', 'm'],
+    ['evspsbl', 'E1hr', 'm'],
+    ['mrro', 'E1hr', 'm'],
+    ['prsn', 'E1hr', 'm of water equivalent'],
+    ['evspsblpot', 'E1hr', 'm'],
+    ['rss', 'E1hr', 'J m**-2'],
+    ['rsds', 'E1hr', 'J m**-2'],
+    ['rsdt', 'E1hr', 'J m**-2'],
+    ['rls', 'E1hr', 'W m**-2'],  # variables with explicit fixes
+    ['uas', 'E1hr', 'm s**-1'],  # a variable without explicit fixes
+    ['pr', 'Amon', 'm'],  # a monthly variable
+    # ['ua', 'E1hr', 'm s**-1'], # a 4d variable (we decided not to do this now)
+    ['orog', 'Fx', 'm**2 s**-2']  # a 2D variable (but keep time coord)
 ]
 
 
 @pytest.mark.parametrize('variable', variables)
 def test_cmorization(variable):
     """Verify that cmorization results in the expected target cube."""
-    short_name, mip, era5_units, ndims = variable
+    short_name, mip, era5_units = variable
 
-    src_cubes = make_src_cubes(era5_units, mip='E1hr')
+    src_cubes = make_src_cubes(era5_units, mip=mip)
     target_cubes = make_target_cubes('native6', mip, short_name)
+
+    print(src_cubes[0].xml())
     out_cubes = fix_metadata(src_cubes, short_name, 'native6', 'era5', mip)
 
     print(out_cubes[0].xml())  # for testing purposes

From 908f76b0f180880dce35a1e35b071029497e71a6 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Tue, 21 Jan 2020 13:16:23 +0100
Subject: [PATCH 21/40] Fix old merge conflict remnant

---
 tests/unit/data_finder/test_get_start_end_year.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tests/unit/data_finder/test_get_start_end_year.py b/tests/unit/data_finder/test_get_start_end_year.py
index 2bff34be70..cc4ee5e4a5 100644
--- a/tests/unit/data_finder/test_get_start_end_year.py
+++ b/tests/unit/data_finder/test_get_start_end_year.py
@@ -1,12 +1,6 @@
 """Unit tests for :func:`esmvalcore._data_finder.regrid._stock_cube`"""
 
-<<<<<<< HEAD
 import pytest
-=======
-import unittest
-import os
-import tempfile
->>>>>>> origin/master
 
 from esmvalcore._data_finder import get_start_end_year
 

From 4ca85509f1fa1e17604c57dba034f1e289c979ac Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Tue, 21 Jan 2020 15:53:55 +0100
Subject: [PATCH 22/40] Fix broken tests due to changes in core

---
 esmvalcore/_data_finder.py                    |  2 -
 esmvalcore/cmor/_fixes/native6/era5.py        | 29 +++++++----
 esmvalcore/cmor/table.py                      | 49 ++++++++----------
 .../cmor/_fixes/cmip5/test_access1_0.py       | 16 +++---
 .../cmor/_fixes/cmip5/test_access1_3.py       | 16 +++---
 .../cmor/_fixes/cmip5/test_bcc_csm1_1.py      |  3 +-
 .../cmor/_fixes/cmip5/test_bcc_csm1_1_m.py    |  3 +-
 .../cmor/_fixes/cmip5/test_bnu_esm.py         | 35 ++++++-------
 .../cmor/_fixes/cmip5/test_canesm2.py         |  5 +-
 .../cmor/_fixes/cmip5/test_ccsm4.py           | 51 ++++++++-----------
 .../cmor/_fixes/cmip5/test_cesm1_bgc.py       | 18 +++----
 .../cmor/_fixes/cmip5/test_cnrm_cm5.py        | 12 ++---
 .../cmor/_fixes/cmip5/test_ec_earth.py        | 37 ++++++--------
 .../cmor/_fixes/cmip5/test_fgoals_g2.py       | 13 +++--
 .../cmor/_fixes/cmip5/test_fio_esm.py         | 14 +++--
 .../cmor/_fixes/cmip5/test_gfdl_cm2p1.py      | 19 +++----
 .../cmor/_fixes/cmip5/test_gfdl_cm3.py        | 17 +++----
 .../cmor/_fixes/cmip5/test_gfdl_esm2g.py      | 41 ++++++++-------
 .../cmor/_fixes/cmip5/test_gfdl_esm2m.py      | 25 ++++-----
 .../cmor/_fixes/cmip5/test_hadgem2_cc.py      |  8 +--
 .../cmor/_fixes/cmip5/test_hadgem2_es.py      |  8 +--
 .../cmor/_fixes/cmip5/test_inmcm4.py          | 21 ++++----
 .../cmor/_fixes/cmip5/test_miroc5.py          | 13 +++--
 .../cmor/_fixes/cmip5/test_miroc_esm.py       | 28 +++++-----
 .../cmor/_fixes/cmip5/test_miroc_esm_chem.py  |  6 +--
 .../cmor/_fixes/cmip5/test_mpi_esm_lr.py      |  6 +--
 .../cmor/_fixes/cmip5/test_mri_cgcm3.py       | 10 ++--
 .../cmor/_fixes/cmip5/test_mri_esm1.py        |  5 +-
 .../cmor/_fixes/cmip5/test_noresm1_me.py      |  8 +--
 .../cmor/_fixes/cmip6/test_cesm2.py           |  4 +-
 .../cmor/_fixes/cmip6/test_cesm2_waccm.py     |  6 +--
 .../cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py |  8 +--
 .../cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py    | 22 +++-----
 .../cmor/_fixes/cmip6/test_mcm_ua_1_0.py      | 24 +++++----
 .../cmor/_fixes/cmip6/test_ukesm1_0_ll.py     |  8 +--
 .../cmor/_fixes/native6/test_era5.py          | 17 +++----
 .../cmor/_fixes/obs4mips/test_ssmi.py         |  6 +--
 .../cmor/_fixes/obs4mips/test_ssmi_meris.py   |  5 +-
 tests/integration/cmor/_fixes/test_fix.py     | 28 +++++-----
 tests/unit/cmor/test_fix.py                   |  2 +-
 40 files changed, 298 insertions(+), 350 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index 8b814e9a05..f035df05b6 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -10,8 +10,6 @@
 import re
 import glob
 
-import iris
-
 from ._config import get_project_config
 
 logger = logging.getLogger(__name__)
diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 5dde0ab9da..e1ab1cc5e6 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -8,19 +8,18 @@
 
 class FixEra5(Fix):
     """Fixes for ERA5 variables"""
-
     @staticmethod
     def _frequency(cube):
-        if not cube.coords(axis='T'):
-            return 'fx'
         coord = cube.coord(axis='T')
-        if 27 < coord.points[1] - coord.points[0] < 32:
+        if len(coord.points) == 1:
+            return 'fx'
+        elif 27 < coord.points[1] - coord.points[0] < 32:
             return 'monthly'
         return 'hourly'
 
+
 class Accumulated(FixEra5):
     """Fixes for accumulated variables."""
-
     def _fix_frequency(self, cube):
         if self._frequency(cube) == 'monthly':
             cube.units = cube.units * 'd-1'
@@ -34,9 +33,9 @@ def fix_metadata(self, cubes):
             self._fix_frequency(cube)
         return cubes
 
+
 class Hydrological(FixEra5):
     """Fixes for hydrological variables."""
-
     @staticmethod
     def _fix_units(cube):
         cube.units = 'kg m-2 s-1'
@@ -49,48 +48,58 @@ def fix_metadata(self, cubes):
             self._fix_units(cube)
         return cubes
 
+
 class Radiation(FixEra5):
     """Fixes for accumulated radiation variables."""
-
     def fix_metadata(self, cubes):
         super().fix_metadata(cubes)
         for cube in cubes:
             cube.attributes['positive'] = 'down'
         return cubes
 
+
 class Fx(FixEra5):
     """Fixes for time invariant variables."""
 
+
 class Evspsbl(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
+
 class Mrro(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
+
 class Prsn(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
+
 class Pr(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
+
 class Evspsblpot(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
+
 class Rss(Radiation, Accumulated):
     """Fixes for Rss."""
 
+
 class Rsds(Radiation, Accumulated):
     """Fixes for Rsds."""
 
+
 class Rsdt(Radiation, Accumulated):
     """Fixes for Rsdt."""
 
+
 class Rls(Radiation):
     """Fixes for Rls."""
 
+
 class Orog(Fx):
     """Fixes for orography"""
-
     @staticmethod
     def _fix_units(cube):
         # Divide by acceleration of gravity [m s-2],
@@ -106,9 +115,9 @@ def fix_metadata(self, cubes):
             self._fix_units(cube)
         return cubes
 
+
 class AllVars(FixEra5):
     """Fixes for all variables."""
-
     def _fix_coordinates(self, cube):
         """Fix coordinates."""
         # Fix coordinate increasing direction
@@ -126,7 +135,6 @@ def _fix_coordinates(self, cube):
         if 'height10m' in self.vardef.dimensions:
             add_scalar_height_coord(cube, 10.)
 
-
         for coord_def in self.vardef.coordinates.values():
             axis = coord_def.axis
             coord = cube.coord(axis=axis)
@@ -141,7 +149,6 @@ def _fix_coordinates(self, cube):
             if len(coord.points) > 1 and coord_def.must_have_bounds == "yes":
                 coord.guess_bounds()
 
-
         self._fix_monthly_time_coord(cube)
 
         return cube
diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py
index 51fbfed2ec..4af52ad09f 100644
--- a/esmvalcore/cmor/table.py
+++ b/esmvalcore/cmor/table.py
@@ -49,19 +49,22 @@ def read_cmor_tables(cfg_developer=None):
 
         if cmor_type == 'CMIP3':
             CMOR_TABLES[table] = CMIP3Info(
-                table_path, default=custom, strict=cmor_strict,
+                table_path,
+                default=custom,
+                strict=cmor_strict,
             )
         elif cmor_type == 'CMIP5':
             CMOR_TABLES[table] = CMIP5Info(
-                table_path, default=custom, strict=cmor_strict,
+                table_path,
+                default=custom,
+                strict=cmor_strict,
             )
         elif cmor_type == 'CMIP6':
             CMOR_TABLES[table] = CMIP6Info(
-                table_path, default=custom, strict=cmor_strict,
-                default_table_prefix=default_table_prefix
-            )
-
-
+                table_path,
+                default=custom,
+                strict=cmor_strict,
+                default_table_prefix=default_table_prefix)
 
 
 class CMIP6Info(object):
@@ -92,7 +95,10 @@ class CMIP6Info(object):
         'vsi': 'siv',
     }
 
-    def __init__(self, cmor_tables_path, default=None, strict=True,
+    def __init__(self,
+                 cmor_tables_path,
+                 default=None,
+                 strict=True,
                  default_table_prefix=''):
         cmor_tables_path = self._get_cmor_path(cmor_tables_path)
 
@@ -131,8 +137,7 @@ def _get_cmor_path(cmor_tables_path):
         if os.path.isdir(cmor_tables_path):
             return cmor_tables_path
         raise ValueError(
-            'CMOR tables not found in {}'.format(cmor_tables_path)
-        )
+            'CMOR tables not found in {}'.format(cmor_tables_path))
 
     def _load_table(self, json_file):
         with open(json_file) as inf:
@@ -173,14 +178,12 @@ def _assign_dimensions(self, var, generic_levels):
                     coord = self.coords[dimension]
                 except KeyError:
                     logger.exception(
-                        'Can not find dimension %s for variable %s',
-                        dimension, var
-                    )
+                        'Can not find dimension %s for variable %s', dimension,
+                        var)
                     raise
 
             var.coordinates[dimension] = coord
 
-
     def _load_coordinates(self):
         self.coords = {}
         for json_file in glob.glob(
@@ -195,8 +198,8 @@ def _load_coordinates(self):
     def _load_controlled_vocabulary(self):
         self.activities = {}
         self.institutes = {}
-        for json_file in glob.glob(
-                os.path.join(self._cmor_folder, '*_CV.json')):
+        for json_file in glob.glob(os.path.join(self._cmor_folder,
+                                                '*_CV.json')):
             with open(json_file) as inf:
                 table_data = json.loads(inf.read())
                 try:
@@ -296,7 +299,6 @@ def _is_table(table_data):
 @total_ordering
 class TableInfo(dict):
     """Container class for storing a CMOR table."""
-
     def __init__(self, *args, **kwargs):
         """Create a new TableInfo object for storing VariableInfo objects."""
         super(TableInfo, self).__init__(*args, **kwargs)
@@ -323,7 +325,6 @@ class JsonInfo(object):
 
     Provides common utility methods to read json variables
     """
-
     def __init__(self):
         self._json_data = {}
 
@@ -368,7 +369,6 @@ def _read_json_list_variable(self, parameter):
 
 class VariableInfo(JsonInfo):
     """Class to read and store variable information."""
-
     def __init__(self, table_type, short_name):
         """
         Class to read and store variable information.
@@ -455,7 +455,6 @@ def read_json(self, json_data, default_freq):
 
 class CoordinateInfo(JsonInfo):
     """Class to read and store coordinate information."""
-
     def __init__(self, name):
         """
         Class to read and store coordinate information.
@@ -498,7 +497,6 @@ def __init__(self, name):
         """Maximum allowed value"""
         self.must_have_bounds = ""
         """Whether bounds are required on this dimension"""
-
     def read_json(self, json_data):
         """
         Read coordinate information from json.
@@ -545,7 +543,6 @@ class CMIP5Info(object):
         found in the requested one
 
     """
-
     def __init__(self, cmor_tables_path, default=None, strict=True):
         cmor_tables_path = self._get_cmor_path(cmor_tables_path)
 
@@ -648,8 +645,7 @@ def _read_coordinate(self, value):
                 return coord
             if key == 'requested':
                 coord.requested.extend(
-                    (val for val in value.split(' ') if val)
-                )
+                    (val for val in value.split(' ') if val))
                 continue
             if hasattr(coord, key):
                 setattr(coord, key, value)
@@ -744,9 +740,8 @@ class CMIP3Info(CMIP5Info):
         found in the requested one
 
     """
-
     def _read_table_file(self, table_file, table=None):
-        for dim in ('zlevel',):
+        for dim in ('zlevel', ):
             coord = CoordinateInfo(dim)
             coord.generic_level = True
             coord.axis = 'Z'
@@ -778,7 +773,6 @@ class CustomInfo(CMIP5Info):
         ESMValTool repository
 
     """
-
     def __init__(self, cmor_tables_path=None):
         cwd = os.path.dirname(os.path.realpath(__file__))
         self._cmor_folder = os.path.join(cwd, 'tables', 'custom')
@@ -867,4 +861,5 @@ def _read_table_file(self, table_file, table=None):
                 if not self._read_line():
                     return
 
+
 read_cmor_tables()
diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py
index 59784dba6e..3669a2b380 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py
@@ -11,22 +11,18 @@
 
 class TestAllVars(unittest.TestCase):
     """Test all vars fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='co2', units='J')
-        self.cube.add_aux_coord(AuxCoord(
-            0,
-            'time',
-            'time',
-            'time',
-            Unit('days since 1850-01-01', 'julian')
-        ))
-        self.fix = AllVars()
+        self.cube.add_aux_coord(
+            AuxCoord(0, 'time', 'time', 'time',
+                     Unit('days since 1850-01-01', 'julian')))
+        self.fix = AllVars(None)
 
     def test_get(self):
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'ACCESS1-0', 'tas'), [AllVars()])
+            Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas'),
+            [AllVars(None)])
 
     def test_fix_metadata(self):
         """Test fix for bad calendar."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py
index 56caf4c0b5..a09b79f266 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py
@@ -11,22 +11,18 @@
 
 class TestAllVars(unittest.TestCase):
     """Test fixes for all vars."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='co2', units='J')
-        self.cube.add_aux_coord(AuxCoord(
-            0,
-            'time',
-            'time',
-            'time',
-            Unit('days since 1850-01-01', 'julian')
-        ))
-        self.fix = AllVars()
+        self.cube.add_aux_coord(
+            AuxCoord(0, 'time', 'time', 'time',
+                     Unit('days since 1850-01-01', 'julian')))
+        self.fix = AllVars(None)
 
     def test_get(self):
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'ACCESS1-3', 'tas'), [AllVars()])
+            Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas'),
+            [AllVars(None)])
 
     def test_fix_metadata(self):
         """Test calendar fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py
index a7bf80ee1a..30c0c44466 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py
@@ -7,8 +7,7 @@
 
 class TestTos(unittest.TestCase):
     """Test tos fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'BCC-CSM1-1', 'tos'), [Tos()])
+            Fix.get_fixes('CMIP5', 'BCC-CSM1-1', 'Amon', 'tos'), [Tos(None)])
diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py
index 1e25b77243..01f427f478 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py
@@ -7,8 +7,7 @@
 
 class TestTos(unittest.TestCase):
     """Test tos fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'BCC-CSM1-1-M', 'tos'), [Tos()])
+            Fix.get_fixes('CMIP5', 'BCC-CSM1-1-M', 'Amon', 'tos'), [Tos(None)])
diff --git a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py
index 9c3edd574b..6fd29d47a2 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py
@@ -12,16 +12,15 @@
 
 class TestCo2(unittest.TestCase):
     """Test fixes for CO2."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='co2', units='J')
-        self.fix = Co2()
+        self.fix = Co2(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'BNU-ESM', 'co2'), [Co2()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'co2'),
+                             [Co2(None)])
 
     def test_fix_metadata(self):
         """Test unit change."""
@@ -38,16 +37,15 @@ def test_fix_data(self):
 
 class Testfgco2(unittest.TestCase):
     """Test fixes for FgCO2."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='fgco2', units='J')
-        self.fix = FgCo2()
+        self.fix = FgCo2(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'BNU-ESM', 'fgco2'), [FgCo2()])
+            Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'fgco2'), [FgCo2(None)])
 
     def test_fix_metadata(self):
         """Test unit fix."""
@@ -64,16 +62,15 @@ def test_fix_data(self):
 
 class TestCh4(unittest.TestCase):
     """Test fixes for ch4."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='ch4', units='J')
-        self.fix = Ch4()
+        self.fix = Ch4(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'BNU-ESM', 'ch4'), [Ch4()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'),
+                             [Ch4(None)])
 
     def test_fix_metadata(self):
         """Test unit fix."""
@@ -90,16 +87,15 @@ def test_fix_data(self):
 
 class Testspco2(unittest.TestCase):
     """Test fixes for SpCO2."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='spco2', units='J')
-        self.fix = SpCo2()
+        self.fix = SpCo2(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'BNU-ESM', 'spco2'), [SpCo2()])
+            Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'spco2'), [SpCo2(None)])
 
     def test_fix_metadata(self):
         """Test fix."""
@@ -116,18 +112,19 @@ def test_fix_data(self):
 
 class TestOd550Aer(unittest.TestCase):
     """Test fixes for SpCO2."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube(
-            ma.MaskedArray([1.e36], mask=(False,)),
-            var_name='od550aer',)
-        self.fix = Od550Aer()
+            ma.MaskedArray([1.e36], mask=(False, )),
+            var_name='od550aer',
+        )
+        self.fix = Od550Aer(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'BNU-ESM', 'od550aer'), [Od550Aer()])
+            Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'od550aer'),
+            [Od550Aer(None)])
 
     def test_fix_data(self):
         """Test data fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py
index 29ebaaef33..ff9d75082e 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py
@@ -10,16 +10,15 @@
 
 class TestCanESM2Fgco2(unittest.TestCase):
     """Test fgc02 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='fgco2', units='J')
-        self.fix = FgCo2()
+        self.fix = FgCo2(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CANESM2', 'fgco2'), [FgCo2()])
+            Fix.get_fixes('CMIP5', 'CANESM2', 'Amon', 'fgco2'), [FgCo2(None)])
 
     def test_fix_data(self):
         """Test data fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py
index fc5b59f46f..1d326c5a03 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py
@@ -11,26 +11,22 @@
 
 class TestsRlut(unittest.TestCase):
     """Test for rlut fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0, 2.0], var_name='rlut')
         self.cube.add_dim_coord(
-            DimCoord(
-                [0.50001, 1.499999],
-                standard_name='latitude',
-                bounds=[
-                    [0.00001, 0.999999],
-                    [1.00001, 1.999999],
-                ]),
-            0
-        )
-        self.fix = Rlut()
+            DimCoord([0.50001, 1.499999],
+                     standard_name='latitude',
+                     bounds=[
+                         [0.00001, 0.999999],
+                         [1.00001, 1.999999],
+                     ]), 0)
+        self.fix = Rlut(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CCSM4', 'rlut'), [Rlut()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'rlut'),
+                             [Rlut(None)])
 
     def test_fix_metadata(self):
         """Check that latitudes values are rounded."""
@@ -45,26 +41,22 @@ def test_fix_metadata(self):
 
 class TestsRlutcs(unittest.TestCase):
     """Test for rlutcs fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0, 2.0], var_name='rlutcs')
         self.cube.add_dim_coord(
-            DimCoord(
-                [0.50001, 1.499999],
-                standard_name='latitude',
-                bounds=[
-                    [0.00001, 0.999999],
-                    [1.00001, 1.999999],
-                ]),
-            0
-        )
-        self.fix = Rlutcs()
+            DimCoord([0.50001, 1.499999],
+                     standard_name='latitude',
+                     bounds=[
+                         [0.00001, 0.999999],
+                         [1.00001, 1.999999],
+                     ]), 0)
+        self.fix = Rlutcs(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CCSM4', 'rlutcs'), [Rlutcs()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'rlutcs'),
+                             [Rlutcs(None)])
 
     def test_fix_metadata(self):
         """Check that latitudes values are rounded."""
@@ -79,16 +71,15 @@ def test_fix_metadata(self):
 
 class TestSo(unittest.TestCase):
     """Tests for so fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0, 2.0], var_name='so', units='1.0')
-        self.fix = So()
+        self.fix = So(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CCSM4', 'so'), [So()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'so'),
+                             [So(None)])
 
     def test_fix_metadata(self):
         """Checks that units are changed to the correct value."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py b/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py
index a41e759fd0..917a0ded4a 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py
@@ -14,12 +14,12 @@ class TestCo2(unittest.TestCase):
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='co2', units='J')
-        self.fix = Co2()
+        self.fix = Co2(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'CESM1-BGC', 'co2'),
-                             [Co2()])
+        self.assertListEqual(
+            Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'co2'), [Co2(None)])
 
     def test_fix_data(self):
         """Test fix to set units correctly."""
@@ -33,12 +33,12 @@ class TestGpp(unittest.TestCase):
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0, 1.0e33, 2.0])
-        self.fix = Gpp()
+        self.fix = Gpp(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'CESM1-BGC', 'gpp'),
-                             [Gpp()])
+        self.assertListEqual(
+            Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'gpp'), [Gpp(None)])
 
     def test_fix_data(self):
         """Test fix to set missing values correctly."""
@@ -55,9 +55,9 @@ class TestNbp(TestGpp):
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0, 1.0e33, 2.0])
-        self.fix = Nbp()
+        self.fix = Nbp(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'CESM1-BGC', 'nbp'),
-                             [Nbp()])
+        self.assertListEqual(
+            Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'nbp'), [Nbp(None)])
diff --git a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py
index e431ca0dec..ae9da2ef04 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py
@@ -10,16 +10,16 @@
 
 class TestMsftmyz(unittest.TestCase):
     """Test msftmyz fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='msftmyz', units='J')
-        self.fix = Msftmyz()
+        self.fix = Msftmyz(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CNRM-CM5', 'msftmyz'), [Msftmyz()])
+            Fix.get_fixes('CMIP5', 'CNRM-CM5', 'Amon', 'msftmyz'),
+            [Msftmyz(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -30,16 +30,16 @@ def test_fix_data(self):
 
 class TestMsftmyzba(unittest.TestCase):
     """Test msftmyzba fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='msftmyzba', units='J')
-        self.fix = Msftmyzba()
+        self.fix = Msftmyzba(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CNRM-CM5', 'msftmyzba'), [Msftmyzba()])
+            Fix.get_fixes('CMIP5', 'CNRM-CM5', 'Amon', 'msftmyzba'),
+            [Msftmyzba(None)])
 
     def test_fix_data(self):
         """Test data fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py
index 6daaacf8d6..2ed2d1bd02 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py
@@ -11,16 +11,15 @@
 
 class TestSic(unittest.TestCase):
     """Test sic fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='sic', units='J')
-        self.fix = Sic()
+        self.fix = Sic(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'EC-EARTH', 'sic'), [Sic()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'sic'),
+                             [Sic(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -31,16 +30,15 @@ def test_fix_data(self):
 
 class TestSftlf(unittest.TestCase):
     """Test sftlf fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='sftlf', units='J')
-        self.fix = Sftlf()
+        self.fix = Sftlf(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'EC-EARTH', 'sftlf'), [Sftlf()])
+            Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'sftlf'), [Sftlf(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -51,26 +49,23 @@ def test_fix_data(self):
 
 class TestTas(unittest.TestCase):
     """Test tas fixes."""
-
     def setUp(self):
         """Prepare tests."""
 
-        height_coord = DimCoord(
-            2.,
-            standard_name='height',
-            long_name='height',
-            var_name='height',
-            units='m',
-            bounds=None,
-            attributes={'positive': 'up'}
-            )
+        height_coord = DimCoord(2.,
+                                standard_name='height',
+                                long_name='height',
+                                var_name='height',
+                                units='m',
+                                bounds=None,
+                                attributes={'positive': 'up'})
 
         time_coord = DimCoord(
             1.,
             standard_name='time',
             var_name='time',
             units=Unit('days since 2070-01-01 00:00:00', calendar='gregorian'),
-            )
+        )
 
         self.height_coord = height_coord
 
@@ -82,12 +77,12 @@ def setUp(self):
         self.cube_with[0].add_aux_coord(time_coord, 0)
         self.cube_with[0].coord('time').long_name = 'time'
 
-        self.fix = Tas()
+        self.fix = Tas(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'EC-EARTH', 'tas'), [Tas()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'tas'),
+                             [Tas(None)])
 
     def test_tas_fix_metadata(self):
         """Test metadata fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py
index a17eeeb29d..8488632db6 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py
@@ -11,22 +11,21 @@
 
 class TestAll(unittest.TestCase):
     """Test fixes for all vars."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0, 2.0], var_name='co2', units='J')
         self.cube.add_dim_coord(
-            DimCoord(
-                [0.0, 1.0],
-                standard_name='time',
-                units=Unit('days since 0001-01', calendar='gregorian')),
+            DimCoord([0.0, 1.0],
+                     standard_name='time',
+                     units=Unit('days since 0001-01', calendar='gregorian')),
             0)
-        self.fix = AllVars()
+        self.fix = AllVars(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'FGOALS-G2', 'tas'), [AllVars()])
+            Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas'),
+            [AllVars(None)])
 
     def test_fix_metadata(self):
         """Test calendar fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py
index 9f1b237edf..709f78ce24 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py
@@ -10,16 +10,15 @@
 
 class TestCh4(unittest.TestCase):
     """Test ch4 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='ch4', units='J')
-        self.fix = Ch4()
+        self.fix = Ch4(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'FIO-ESM', 'ch4'), [Ch4()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'FIO-ESM', 'Amon', 'ch4'),
+                             [Ch4(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -30,16 +29,15 @@ def test_fix_data(self):
 
 class TestCo2(unittest.TestCase):
     """Test co2 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='co2', units='J')
-        self.fix = Co2()
+        self.fix = Co2(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'FIO-ESM', 'co2'), [Co2()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'FIO-ESM', 'Amon', 'co2'),
+                             [Co2(None)])
 
     def test_fix_data(self):
         """Test data fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py
index 862f74803f..4540645c98 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py
@@ -10,18 +10,16 @@
 
 class TestSftof(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='sftof', units='J')
-        self.fix = Sftof()
+        self.fix = Sftof(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'sftof'),
-            [AllVars(), Sftof()]
-        )
+            Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'sftof'),
+            [Sftof(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -32,27 +30,26 @@ def test_fix_data(self):
 
 class TestAreacello(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='areacello', units='m-2')
-        self.fix = Areacello()
+        self.fix = Areacello(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'areacello'),
-            [AllVars(), Areacello()])
+            Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'areacello'),
+            [Areacello(None), AllVars(None)])
 
     def test_fix_metadata(self):
         """Test data fix."""
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
 
     def test_fix_data(self):
         """Test data fix."""
         self.cube.units = 'm2'
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py
index 8644cf15f1..fdd2f4c4c2 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py
@@ -10,16 +10,16 @@
 
 class TestSftof(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='sftof', units='J')
-        self.fix = Sftof()
+        self.fix = Sftof(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-CM3', 'sftof'), [AllVars(), Sftof()])
+            Fix.get_fixes('CMIP5', 'GFDL-CM3', 'Amon', 'sftof'),
+            [Sftof(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -30,27 +30,26 @@ def test_fix_data(self):
 
 class TestAreacello(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='areacello', units='m-2')
-        self.fix = Areacello()
+        self.fix = Areacello(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-CM3', 'areacello'),
-            [AllVars(), Areacello()])
+            Fix.get_fixes('CMIP5', 'GFDL-CM3', 'Amon', 'areacello'),
+            [Areacello(None), AllVars(None)])
 
     def test_fix_metadata(self):
         """Test data fix."""
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
 
     def test_fix_data(self):
         """Test data fix."""
         self.cube.units = 'm2'
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py
index 9e04694dfe..e22a0d26c1 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py
@@ -31,11 +31,11 @@ def test_get_and_remove(cubes_in, cubes_out):
 CUBES = iris.cube.CubeList([CUBE_1, CUBE_2])
 
 
-@mock.patch(
-    'esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove', autospec=True)
+@mock.patch('esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove',
+            autospec=True)
 def test_allvars(mock_get_and_remove):
     """Test fixes for all vars."""
-    fix = AllVars()
+    fix = AllVars(None)
     fix.fix_metadata(CUBES)
     assert mock_get_and_remove.call_count == 3
     assert mock_get_and_remove.call_args_list == [
@@ -45,11 +45,11 @@ def test_allvars(mock_get_and_remove):
     ]
 
 
-@mock.patch(
-    'esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove', autospec=True)
+@mock.patch('esmvalcore.cmor._fixes.cmip5.gfdl_esm2g._get_and_remove',
+            autospec=True)
 def test_fgco2(mock_get_and_remove):
     """Test fgco2 fixes."""
-    fix = FgCo2()
+    fix = FgCo2(None)
     fix.fix_metadata(CUBES)
     assert mock_get_and_remove.call_count == 2
     assert mock_get_and_remove.call_args_list == [
@@ -60,16 +60,16 @@ def test_fgco2(mock_get_and_remove):
 
 class TestCo2(unittest.TestCase):
     """Test co2 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = iris.cube.Cube([1.0], var_name='co2', units='J')
-        self.fix = Co2()
+        self.fix = Co2(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'co2'), [AllVars(), Co2()])
+            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'co2'),
+            [Co2(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -80,16 +80,16 @@ def test_fix_data(self):
 
 class TestUsi(unittest.TestCase):
     """Test usi fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = iris.cube.Cube([1.0], var_name='usi', units='J')
-        self.fix = Usi()
+        self.fix = Usi(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'usi'), [AllVars(), Usi()])
+            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'usi'),
+            [Usi(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test metadata fix."""
@@ -99,16 +99,16 @@ def test_fix_data(self):
 
 class TestVsi(unittest.TestCase):
     """Test vsi fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = iris.cube.Cube([1.0], var_name='vsi', units='J')
-        self.fix = Vsi()
+        self.fix = Vsi(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'vsi'), [AllVars(), Vsi()])
+            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'vsi'),
+            [Vsi(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test metadata fix."""
@@ -118,27 +118,26 @@ def test_fix_data(self):
 
 class TestAreacello(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='areacello', units='m-2')
-        self.fix = Areacello()
+        self.fix = Areacello(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'areacello'),
-            [AllVars(), Areacello()])
+            Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'areacello'),
+            [Areacello(None), AllVars(None)])
 
     def test_fix_metadata(self):
         """Test data fix."""
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
 
     def test_fix_data(self):
         """Test data fix."""
         self.cube.units = 'm2'
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py
index 5126d8959c..900da08a97 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py
@@ -11,18 +11,16 @@
 
 class TestSftof(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='sftof', units='J')
-        self.fix = Sftof()
+        self.fix = Sftof(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'sftof'),
-            [AllVars(), Sftof()]
-        )
+            Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'sftof'),
+            [Sftof(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -33,16 +31,16 @@ def test_fix_data(self):
 
 class TestCo2(unittest.TestCase):
     """Test co2 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='co2', units='J')
-        self.fix = Co2()
+        self.fix = Co2(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'co2'), [AllVars(), Co2()])
+            Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'co2'),
+            [Co2(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -53,27 +51,26 @@ def test_fix_data(self):
 
 class TestAreacello(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='areacello', units='m-2')
-        self.fix = Areacello()
+        self.fix = Areacello(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'areacello'),
-            [AllVars(), Areacello()])
+            Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'areacello'),
+            [Areacello(None), AllVars(None)])
 
     def test_fix_metadata(self):
         """Test data fix."""
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
 
     def test_fix_data(self):
         """Test data fix."""
         self.cube.units = 'm2'
-        cube = self.fix.fix_metadata((self.cube,))[0]
+        cube = self.fix.fix_metadata((self.cube, ))[0]
         self.assertEqual(cube.data[0], 1.0)
         self.assertEqual(cube.units, Unit('m2'))
diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py
index 1bfc2a0f54..ed2f7c0d1f 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py
@@ -7,17 +7,17 @@
 
 class TestAllVars(unittest.TestCase):
     """Test allvars fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'tas'), [AllVars()])
+            Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'Amon', 'tas'),
+            [AllVars(None)])
 
 
 class TestO2(unittest.TestCase):
     """Test o2 fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'o2'), [AllVars(), O2()])
+            Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'Amon', 'o2'),
+            [O2(None), AllVars(None)])
diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py
index 46343f4b7b..b6eea1cd25 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py
@@ -7,17 +7,17 @@
 
 class TestAllVars(unittest.TestCase):
     """Test allvars fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'tas'), [AllVars()])
+            Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'Amon', 'tas'),
+            [AllVars(None)])
 
 
 class TestO2(unittest.TestCase):
     """Test o2 fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'o2'), [AllVars(), O2()])
+            Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'Amon', 'o2'),
+            [O2(None), AllVars(None)])
diff --git a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py
index 65094c567e..9da7f16103 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py
@@ -10,16 +10,15 @@
 
 class TestGpp(unittest.TestCase):
     """Test gpp fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='gpp', units='J')
-        self.fix = Gpp()
+        self.fix = Gpp(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'INMCM4', 'gpp'), [Gpp()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'gpp'),
+                             [Gpp(None)])
 
     def test_fix_data(self):
         """Test data fox."""
@@ -30,16 +29,15 @@ def test_fix_data(self):
 
 class TestLai(unittest.TestCase):
     """Test lai fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='lai', units='J')
-        self.fix = Lai()
+        self.fix = Lai(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'INMCM4', 'lai'), [Lai()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'lai'),
+                             [Lai(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -50,16 +48,15 @@ def test_fix_data(self):
 
 class TestNbp(unittest.TestCase):
     """Tests for nbp."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='nbp')
-        self.fix = Nbp()
+        self.fix = Nbp(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'INMCM4', 'nbp'), [Nbp()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'nbp'),
+                             [Nbp(None)])
 
     def test_fix_metadata(self):
         """Test fix on nbp files to set standard_name."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc5.py b/tests/integration/cmor/_fixes/cmip5/test_miroc5.py
index d35ac414e9..d44093c405 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_miroc5.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_miroc5.py
@@ -11,16 +11,15 @@
 
 class TestSftof(unittest.TestCase):
     """Test sftof fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='sftof', units='J')
-        self.fix = Sftof()
+        self.fix = Sftof(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'sftof'),
-                             [Sftof()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'sftof'),
+                             [Sftof(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -31,7 +30,6 @@ def test_fix_data(self):
 
 class TestTas(unittest.TestCase):
     """Test tas fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.coord_name = 'latitude'
@@ -39,11 +37,12 @@ def setUp(self):
                                           bounds=[[1.23, 4.5678910]],
                                           standard_name=self.coord_name)
         self.cube = Cube([1.0], dim_coords_and_dims=[(self.coord, 0)])
-        self.fix = Tas()
+        self.fix = Tas(None)
 
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'tas'), [Tas()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'tas'),
+                             [Tas(None)])
 
     def test_fix_metadata(self):
         """Test metadata fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py
index 4660444cd5..51d887c597 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py
@@ -12,16 +12,16 @@
 
 class TestCo2(unittest.TestCase):
     """Test c02 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='co2', units='J')
-        self.fix = Co2()
+        self.fix = Co2(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MIROC-ESM', 'co2'), [AllVars(), Co2()])
+            Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'co2'),
+            [Co2(None), AllVars(None)])
 
     def test_fix_metadata(self):
         """Test unit fix."""
@@ -32,16 +32,16 @@ def test_fix_metadata(self):
 
 class TestTro3(unittest.TestCase):
     """Test tro3 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='tro3', units='J')
-        self.fix = Tro3()
+        self.fix = Tro3(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MIROC-ESM', 'tro3'), [AllVars(), Tro3()])
+            Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'tro3'),
+            [Tro3(None), AllVars(None)])
 
     def test_fix_data(self):
         """Test data fix."""
@@ -52,25 +52,23 @@ def test_fix_data(self):
 
 class TestAll(unittest.TestCase):
     """Test fixes for allvars."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([[1.0, 2.0], [3.0, 4.0]], var_name='co2', units='J')
         self.cube.add_dim_coord(
-            DimCoord(
-                [0, 1],
-                standard_name='time',
-                units=Unit(
-                    'days since 0000-01-01 00:00:00', calendar='gregorian')),
-            0)
+            DimCoord([0, 1],
+                     standard_name='time',
+                     units=Unit('days since 0000-01-01 00:00:00',
+                                calendar='gregorian')), 0)
         self.cube.add_dim_coord(DimCoord([0, 1], long_name='AR5PL35'), 1)
 
-        self.fix = AllVars()
+        self.fix = AllVars(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MIROC-ESM', 'tos'), [AllVars()])
+            Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'tos'),
+            [AllVars(None)])
 
     def test_fix_metadata_plev(self):
         """Test plev fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py
index 276df42a2e..186f321710 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py
@@ -11,16 +11,16 @@
 
 class TestTro3(unittest.TestCase):
     """Test tro3 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='tro3', units='J')
-        self.fix = Tro3()
+        self.fix = Tro3(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MIROC-ESM-CHEM', 'tro3'), [Tro3()])
+            Fix.get_fixes('CMIP5', 'MIROC-ESM-CHEM', 'Amon', 'tro3'),
+            [Tro3(None)])
 
     def test_fix_data(self):
         """Test data fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py
index f20ecd3f09..f8c90ec3e2 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py
@@ -10,16 +10,16 @@
 
 class TestPctisccp2(unittest.TestCase):
     """Test Pctisccp2 fixes."""
-
     def setUp(self):
         """Prepare tests."""
         self.cube = Cube([1.0], var_name='pctisccp', units='J')
-        self.fix = Pctisccp()
+        self.fix = Pctisccp(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MPI-ESM-LR', 'pctisccp'), [Pctisccp()])
+            Fix.get_fixes('CMIP5', 'MPI-ESM-LR', 'Amon', 'pctisccp'),
+            [Pctisccp(None)])
 
     def test_fix_data(self):
         """Test data fix."""
diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py
index 926484f300..1b93fb3004 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py
@@ -7,19 +7,17 @@
 
 class TestMsftmyz(unittest.TestCase):
     """Test msftmyz fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'msftmyz'), [Msftmyz()]
-        )
+            Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'Amon', 'msftmyz'),
+            [Msftmyz(None)])
 
 
 class TestThetao(unittest.TestCase):
     """Test thetao fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'thetao'), [ThetaO()]
-        )
+            Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'Amon', 'thetao'),
+            [ThetaO(None)])
diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py
index a9e664381a..c00af1d6fe 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py
@@ -7,9 +7,8 @@
 
 class TestMsftmyz(unittest.TestCase):
     """Test msftmyz fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'MRI-ESM1', 'msftmyz'), [Msftmyz()]
-        )
+            Fix.get_fixes('CMIP5', 'MRI-ESM1', 'Amon', 'msftmyz'),
+            [Msftmyz(None)])
diff --git a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py
index a7a285a1ed..ab286d78bd 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py
@@ -53,15 +53,15 @@
 CUBES_TO_FIX = [
     (CubeList([CUBE_IN_SHORT]), CubeList([CUBE_IN_SHORT])),
     (CubeList([CUBE_IN_LONG]), CubeList([CUBE_OUT_LONG])),
-    (CubeList([CUBE_IN_LONG, CUBE_IN_SHORT]),
-     CubeList([CUBE_OUT_LONG, CUBE_IN_SHORT])),
+    (CubeList([CUBE_IN_LONG,
+               CUBE_IN_SHORT]), CubeList([CUBE_OUT_LONG, CUBE_IN_SHORT])),
 ]
 
 
 @pytest.mark.parametrize('cubes_in,cubes_out', CUBES_TO_FIX)
 def test_tas(cubes_in, cubes_out):
     """Test tas fixes."""
-    fix = Tas()
+    fix = Tas(None)
     new_cubes = fix.fix_metadata(cubes_in)
     assert new_cubes is cubes_in
     assert new_cubes == cubes_out
@@ -69,4 +69,4 @@ def test_tas(cubes_in, cubes_out):
 
 def test_get():
     """Test fix get"""
-    assert Fix.get_fixes('CMIP5', 'NORESM1-ME', 'tas') == [Tas()]
+    assert Fix.get_fixes('CMIP5', 'NORESM1-ME', 'Amon', 'tas') == [Tas(None)]
diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py
index e7793ecabc..826d2f6118 100644
--- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py
+++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py
@@ -16,7 +16,7 @@ def tas_cubes():
 
 def test_get_tas_fix():
     fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'tas')
-    assert fix == [Tas()]
+    assert fix == [Tas(None)]
 
 
 def test_tas_fix_metadata(tas_cubes):
@@ -29,7 +29,7 @@ def test_tas_fix_metadata(tas_cubes):
                                         long_name='height',
                                         units=Unit('m'),
                                         attributes={'positive': 'up'})
-    fix = Tas()
+    fix = Tas(None)
     out_cubes = fix.fix_metadata(tas_cubes)
     assert out_cubes is tas_cubes
     for cube in out_cubes:
diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py
index 75553adccb..f5410a8393 100644
--- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py
+++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py
@@ -15,8 +15,8 @@ def tas_cubes():
 
 
 def test_get_tas_fix():
-    fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'tas')
-    assert fix == [Tas()]
+    fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'Amon', 'tas')
+    assert fix == [Tas(None)]
 
 
 def test_tas_fix_metadata(tas_cubes):
@@ -29,7 +29,7 @@ def test_tas_fix_metadata(tas_cubes):
                                         long_name='height',
                                         units=Unit('m'),
                                         attributes={'positive': 'up'})
-    fix = Tas()
+    fix = Tas(None)
     out_cubes = fix.fix_metadata(tas_cubes)
     assert out_cubes is tas_cubes
     for cube in out_cubes:
diff --git a/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py b/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py
index 52635a6ff6..3fe21b3d29 100644
--- a/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py
+++ b/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py
@@ -14,14 +14,14 @@ def sample_cubes():
 
 
 def test_get_tas_fix():
-    fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'tas')
-    assert fix == [AllVars()]
+    fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'Amon', 'tas')
+    assert fix == [AllVars(None)]
 
 
 def test_allvars_fix_metadata(sample_cubes):
     for cube in sample_cubes:
         cube.attributes['parent_time_units'] = 'days since 1850-01-01'
-    out_cubes = AllVars().fix_metadata(sample_cubes)
+    out_cubes = AllVars(None).fix_metadata(sample_cubes)
     assert out_cubes is sample_cubes
     for cube in out_cubes:
         assert cube.attributes['parent_time_units'] == 'days since 1850-01-01'
@@ -30,7 +30,7 @@ def test_allvars_fix_metadata(sample_cubes):
 def test_allvars_no_need_tofix_metadata(sample_cubes):
     for cube in sample_cubes:
         cube.attributes['parent_time_units'] = 'days since 1850-01-01'
-    out_cubes = AllVars().fix_metadata(sample_cubes)
+    out_cubes = AllVars(None).fix_metadata(sample_cubes)
     assert out_cubes is sample_cubes
     for cube in out_cubes:
         assert cube.attributes['parent_time_units'] == 'days since 1850-01-01'
diff --git a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py
index 266ae2d9f3..7f1d41e2cf 100644
--- a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py
+++ b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py
@@ -11,24 +11,16 @@
 
 class TestAllVars(unittest.TestCase):
     def setUp(self):
-        self.fix = AllVars()
+        self.fix = AllVars(None)
         self.cube = Cube(np.random.rand(2, 2, 2), var_name='ch4')
         self.cube.add_aux_coord(
-            AuxCoord(
-                np.random.rand(2, 2),
-                var_name='nav_lat',
-                standard_name='latitude'
-            ),
-            (1, 2)
-        )
+            AuxCoord(np.random.rand(2, 2),
+                     var_name='nav_lat',
+                     standard_name='latitude'), (1, 2))
         self.cube.add_aux_coord(
-            AuxCoord(
-                np.random.rand(2, 2),
-                var_name='nav_lon',
-                standard_name='longitude'
-            ),
-            (1, 2)
-        )
+            AuxCoord(np.random.rand(2, 2),
+                     var_name='nav_lon',
+                     standard_name='longitude'), (1, 2))
 
     def test_fix_metadata_ocean_var(self):
         cell_area = Cube(np.random.rand(2, 2), standard_name='cell_area')
diff --git a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py
index 8723a8bbac..2239576ec5 100644
--- a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py
+++ b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py
@@ -9,13 +9,17 @@
 
 @pytest.fixture
 def cubes():
-    correct_lat_coord = iris.coords.DimCoord([0.0], var_name='lat',
+    correct_lat_coord = iris.coords.DimCoord([0.0],
+                                             var_name='lat',
                                              standard_name='latitude')
-    wrong_lat_coord = iris.coords.DimCoord([0.0], var_name='latitudeCoord',
+    wrong_lat_coord = iris.coords.DimCoord([0.0],
+                                           var_name='latitudeCoord',
                                            standard_name='latitude')
-    correct_lon_coord = iris.coords.DimCoord([0.0], var_name='lon',
+    correct_lon_coord = iris.coords.DimCoord([0.0],
+                                             var_name='lon',
                                              standard_name='longitude')
-    wrong_lon_coord = iris.coords.DimCoord([0.0], var_name='longitudeCoord',
+    wrong_lon_coord = iris.coords.DimCoord([0.0],
+                                           var_name='longitudeCoord',
                                            standard_name='longitude')
     correct_cube = iris.cube.Cube(
         [[10.0]],
@@ -33,17 +37,17 @@ def cubes():
 
 
 def test_get_allvars_fix():
-    fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'arbitrary_var_name')
-    assert fix == [AllVars()]
+    fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Amon', 'arbitrary_var_name')
+    assert fix == [AllVars(None)]
 
 
 def test_get_tas_fix():
-    fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'tas')
-    assert fix == [AllVars(), Tas()]
+    fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Amon', 'tas')
+    assert fix == [Tas(None), AllVars(None)]
 
 
 def test_allvars_fix_metadata(cubes):
-    fix = AllVars()
+    fix = AllVars(None)
     out_cubes = fix.fix_metadata(cubes)
     assert cubes is out_cubes
     for cube in out_cubes:
@@ -76,7 +80,7 @@ def test_tas_fix_metadata(cubes):
                                         long_name='height',
                                         units=Unit('m'),
                                         attributes={'positive': 'up'})
-    fix = Tas()
+    fix = Tas(None)
 
     # Check fix
     out_cubes = fix.fix_metadata(cubes)
diff --git a/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py b/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py
index b4f6d1aecf..fa09029520 100644
--- a/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py
+++ b/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py
@@ -14,14 +14,14 @@ def sample_cubes():
 
 
 def test_get_tas_fix():
-    fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'tas')
-    assert fix == [AllVars()]
+    fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'Amon', 'tas')
+    assert fix == [AllVars(None)]
 
 
 def test_allvars_fix_metadata(sample_cubes):
     for cube in sample_cubes:
         cube.attributes['parent_time_units'] = 'days since 1850-01-01'
-    out_cubes = AllVars().fix_metadata(sample_cubes)
+    out_cubes = AllVars(None).fix_metadata(sample_cubes)
     assert out_cubes is sample_cubes
     for cube in out_cubes:
         assert cube.attributes['parent_time_units'] == 'days since 1850-01-01'
@@ -30,7 +30,7 @@ def test_allvars_fix_metadata(sample_cubes):
 def test_allvars_no_need_tofix_metadata(sample_cubes):
     for cube in sample_cubes:
         cube.attributes['parent_time_units'] = 'days since 1850-01-01'
-    out_cubes = AllVars().fix_metadata(sample_cubes)
+    out_cubes = AllVars(None).fix_metadata(sample_cubes)
     assert out_cubes is sample_cubes
     for cube in out_cubes:
         assert cube.attributes['parent_time_units'] == 'days since 1850-01-01'
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index a613f0f2d9..73565dc72b 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -21,11 +21,11 @@ def make_src_cubes(units, mip='E1hr'):
     # Adapt data and time coordinate for different mips
     data = np.arange(27).reshape(3, 3, 3)
     if mip == 'E1hr':
-        timestamps = [788928, 788929, 788930] # 3 consecutive hours
+        timestamps = [788928, 788929, 788930]  # 3 consecutive hours
     elif mip == 'Amon':
-        timestamps = [788928, 789672, 790344] # 3 consecutive months
+        timestamps = [788928, 789672, 790344]  # 3 consecutive months
     elif mip == 'Fx':
-        timestamps = [788928]                 # 1 single timestamp
+        timestamps = [788928]  # 1 single timestamp
         data = np.arange(9).reshape(1, 3, 3)
 
     # Create coordinates
@@ -70,20 +70,19 @@ def make_target_cubes(project, mip, short_name):
     bounds = None
     if mip == 'E1hr':
         timestamps = [51134.0, 51134.0416667, 51134.0833333]
-        if not 'time1' in vardef.dimensions:
+        if 'time1' not in vardef.dimensions:
             bounds = np.array([[51133.9791667, 51134.0208333],
                                [51134.0208333, 51134.0625],
                                [51134.0625, 51134.1041667]])
     elif mip == 'Amon':
         timestamps = [51149.5, 51179.0, 51208.5]
-        if not 'time1' in vardef.dimensions:
-            bounds = np.array([[51134.0, 51165.0],
-		                       [51165.0, 51193.0],
-		                       [51193.0, 51224.0]])
+        if 'time1' not in vardef.dimensions:
+            bounds = np.array([[51134.0, 51165.0], [51165.0, 51193.0],
+                               [51193.0, 51224.0]])
     elif mip == 'Fx':
         data = np.arange(9).reshape(1, 3, 3)[:, ::-1, :]
         timestamps = [51134.0]
-        if not 'time1' in vardef.dimensions:
+        if 'time1' not in vardef.dimensions:
             bounds = np.array([[51133.9791667, 51134.0208333]])
 
     # Make lat/lon/time coordinates
diff --git a/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py b/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py
index 87bb3d3cf2..62be67de55 100644
--- a/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py
+++ b/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py
@@ -7,9 +7,7 @@
 
 class TestPrw(unittest.TestCase):
     """Test prw fixes."""
-
     def test_get(self):
         """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('OBS4MIPS', 'SSMI', 'prw'), [Prw()]
-        )
+        self.assertListEqual(Fix.get_fixes('obs4mips', 'SSMI', 'Amon', 'prw'),
+                             [Prw(None)])
diff --git a/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py b/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py
index cd130f5f05..03b0dac3ab 100644
--- a/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py
+++ b/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py
@@ -7,9 +7,8 @@
 
 class TestPrw(unittest.TestCase):
     """Test prw fixes."""
-
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('OBS4MIPS', 'SSMI-MERIS', 'prw'), [Prw()]
-        )
+            Fix.get_fixes('obs4mips', 'SSMI-MERIS', 'Amon', 'prw'),
+            [Prw(None)])
diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py
index 30ecd23c84..3a54e24127 100644
--- a/tests/integration/cmor/_fixes/test_fix.py
+++ b/tests/integration/cmor/_fixes/test_fix.py
@@ -3,6 +3,7 @@
 import tempfile
 import unittest
 
+import pytest
 from iris.cube import Cube
 
 from esmvalcore.cmor.fix import Fix
@@ -20,51 +21,54 @@ def tearDown(self):
     def test_get_fix(self):
         from esmvalcore.cmor._fixes.cmip5.canesm2 import FgCo2
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CanESM2', 'fgco2'), [FgCo2()])
+            Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgco2'), [FgCo2(None)])
 
     def test_get_fix_case_insensitive(self):
         from esmvalcore.cmor._fixes.cmip5.canesm2 import FgCo2
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CanESM2', 'fgCo2'), [FgCo2()])
+            Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgCo2'), [FgCo2(None)])
 
     def test_get_fixes_with_replace(self):
         from esmvalcore.cmor._fixes.cmip5.bnu_esm import Ch4
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'ch4'), [Ch4()])
+        self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'),
+                             [Ch4(None)])
 
     def test_get_fixes_with_generic(self):
         from esmvalcore.cmor._fixes.cmip5.cesm1_bgc import Co2
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'CESM1-BGC', 'co2'), [Co2()])
+            Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'co2'), [Co2(None)])
 
     def test_get_fix_no_project(self):
-        self.assertListEqual(
-            Fix.get_fixes('BAD_PROJECT', 'BNU-ESM', 'ch4'), [])
+        with pytest.raises(KeyError):
+            Fix.get_fixes('BAD_PROJECT', 'BNU-ESM', 'Amon', 'ch4')
 
     def test_get_fix_no_model(self):
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'BAD_MODEL', 'ch4'), [])
+        self.assertListEqual(
+            Fix.get_fixes('CMIP5', 'BAD_MODEL', 'Amon', 'ch4'), [])
 
     def test_get_fix_no_var(self):
-        self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'BAD_VAR'), [])
+        self.assertListEqual(
+            Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'BAD_VAR'), [])
 
     def test_fix_metadata(self):
         cube = Cube([0])
         reference = Cube([0])
 
-        self.assertEqual(Fix().fix_metadata(cube), reference)
+        self.assertEqual(Fix(None).fix_metadata(cube), reference)
 
     def test_fix_data(self):
         cube = Cube([0])
         reference = Cube([0])
 
-        self.assertEqual(Fix().fix_data(cube), reference)
+        self.assertEqual(Fix(None).fix_data(cube), reference)
 
     def test_fix_file(self):
         filepath = 'sample_filepath'
-        self.assertEqual(Fix().fix_file(filepath, 'preproc'), filepath)
+        self.assertEqual(Fix(None).fix_file(filepath, 'preproc'), filepath)
 
     def test_fixed_filenam(self):
         filepath = os.path.join(self.temp_folder, 'file.nc')
         output_dir = os.path.join(self.temp_folder, 'fixed')
         os.makedirs(output_dir)
-        fixed_filepath = Fix().get_fixed_filepath(output_dir, filepath)
+        fixed_filepath = Fix(None).get_fixed_filepath(output_dir, filepath)
         self.assertTrue(fixed_filepath, os.path.join(output_dir, 'file.nc'))
diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py
index a449c822b2..1a06cdabd8 100644
--- a/tests/unit/cmor/test_fix.py
+++ b/tests/unit/cmor/test_fix.py
@@ -45,7 +45,7 @@ def setUp(self):
         self.cube_2 = mock.Mock()
         self.cube_2.var_name = 'cube2'
         self.cubes = [self.cube_1, self.cube_2]
-        self.fix = Fix()
+        self.fix = Fix(None)
 
     def test_get_first_cube(self):
         """Test selecting first cube."""

From 6774a980fe207a555d3fb615989171fa45e1b351 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Tue, 21 Jan 2020 16:59:26 +0100
Subject: [PATCH 23/40] Add documentation

---
 doc/esmvalcore/fixing_data.rst         | 9 ++++++++-
 esmvalcore/cmor/_fixes/native6/era5.py | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/doc/esmvalcore/fixing_data.rst b/doc/esmvalcore/fixing_data.rst
index 3448c16ff1..792c51a279 100644
--- a/doc/esmvalcore/fixing_data.rst
+++ b/doc/esmvalcore/fixing_data.rst
@@ -16,6 +16,13 @@ coordinate bounds like ''lat_bnds'') or problems with the actual data
 The ESMValTool can apply on the fly fixes to data sets that have
 known errors that can be fixed automatically.
 
+.. note::
+  **CMORization as a fix**. As of early 2020, we've started implementing cmorization as fixes for
+  observational datasets. Previously, cmorization was an additional function implemented in ESMValTool.
+  This meant that users always had to store 2 copies of their observational data: both raw and cmorized.
+  Implementing cmorization as a fix removes this redundancy, as the fixes are applied 'on the fly' when
+  running a recipe. **ERA5** is the first dataset for which this "cmorization on the fly" is supported.
+
 Fix structure
 =============
 
@@ -44,7 +51,7 @@ Check the output
 Next to the error message, you should see some info about the iris cube: size,
 coordinates. In our example it looks like this:
 
-.. code-block::
+.. code-block:: python
 
     air_temperature/ (K) (time: 312; altitude: 90; longitude: 180)
         Dimension coordinates:
diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index e1ab1cc5e6..cb798ec87b 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -146,7 +146,7 @@ def _fix_coordinates(self, cube):
             coord.var_name = coord_def.out_name
             coord.long_name = coord_def.long_name
             coord.points = coord.core_points().astype('float64')
-            if len(coord.points) > 1 and coord_def.must_have_bounds == "yes":
+            if coord_def.must_have_bounds == "yes":
                 coord.guess_bounds()
 
         self._fix_monthly_time_coord(cube)

From faba70052e40ba759a68f5e18f0e711fbbcf3faf Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 22 Jan 2020 09:46:50 +0100
Subject: [PATCH 24/40] Add link to cmorizer documentation

---
 doc/esmvalcore/fixing_data.rst | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/doc/esmvalcore/fixing_data.rst b/doc/esmvalcore/fixing_data.rst
index 792c51a279..2d661881a5 100644
--- a/doc/esmvalcore/fixing_data.rst
+++ b/doc/esmvalcore/fixing_data.rst
@@ -17,11 +17,13 @@ The ESMValTool can apply on the fly fixes to data sets that have
 known errors that can be fixed automatically.
 
 .. note::
-  **CMORization as a fix**. As of early 2020, we've started implementing cmorization as fixes for
-  observational datasets. Previously, cmorization was an additional function implemented in ESMValTool.
-  This meant that users always had to store 2 copies of their observational data: both raw and cmorized.
-  Implementing cmorization as a fix removes this redundancy, as the fixes are applied 'on the fly' when
-  running a recipe. **ERA5** is the first dataset for which this "cmorization on the fly" is supported.
+  **CMORization as a fix**. As of early 2020, we've started implementing CMORization as fixes for
+  observational datasets. Previously, CMORization was an additional function implemented in ESMValTool.
+  This meant that users always had to store 2 copies of their observational data: both raw and CMORized.
+  Implementing CMORization as a fix removes this redundancy, as the fixes are applied 'on the fly' when
+  running a recipe. **ERA5** is the first dataset for which this 'CMORization on the fly' is supported.
+  For more information about CMORization, see:
+  `Contributing a CMORizing script for an observational dataset <https://esmvaltool.readthedocs.io/en/latest/esmvaldiag/observations.html>`_.
 
 Fix structure
 =============

From 6ee3eae6c4ce2957a52969866ee7508ec02485c8 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 22 Jan 2020 11:58:32 +0100
Subject: [PATCH 25/40] Add fix for hourly accumulations

---
 esmvalcore/cmor/_fixes/native6/era5.py | 49 ++++++++++++++++++++------
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index cb798ec87b..d599f05f97 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -10,26 +10,39 @@ class FixEra5(Fix):
     """Fixes for ERA5 variables"""
     @staticmethod
     def _frequency(cube):
-        coord = cube.coord(axis='T')
-        if len(coord.points) == 1:
+        try:
+            time = cube.coord(axis='T')
+        except CoordinateNotFoundError:
+            time = None
+
+        if not time or len(time.points) == 1:
             return 'fx'
-        elif 27 < coord.points[1] - coord.points[0] < 32:
-            return 'monthly'
-        return 'hourly'
+        elif time.points[1] - time.points[0] == 1:
+            return 'hourly'
+        return 'monthly'
 
 
 class Accumulated(FixEra5):
     """Fixes for accumulated variables."""
     def _fix_frequency(self, cube):
-        if self._frequency(cube) == 'monthly':
+        if cube.varname in ['mx2t', 'mn2t']:
+            pass
+        elif self._frequency(cube) == 'monthly':
             cube.units = cube.units * 'd-1'
         elif self._frequency(cube) == 'hourly':
             cube.units = cube.units * 'h-1'
         return cube
 
+    def _fix_hourly_time_coordinate(self, cube):
+        if self._frequency == 'hourly':
+            time = cube.coord(axis='T')
+            time.points = time.points - 0.5
+        return cube
+
     def fix_metadata(self, cubes):
         super().fix_metadata(cubes)
         for cube in cubes:
+            self._fix_hourly_time_coordinate(...)
             self._fix_frequency(cube)
         return cubes
 
@@ -60,6 +73,23 @@ def fix_metadata(self, cubes):
 
 class Fx(FixEra5):
     """Fixes for time invariant variables."""
+    @staticmethod
+    def _remove_time_coordinate(cube):
+        cube = iris.util.squeeze(cube)
+        cube.remove_coord('time')
+        return cube
+
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            self._remove_time_coordinate(cube)
+
+
+class Tasmin(Accumulated):
+    """Fixes for tasmin."""
+
+
+class Tasmax(Accumulated):
+    """Fixes for tasmax."""
 
 
 class Evspsbl(Hydrological, Accumulated):
@@ -101,10 +131,7 @@ class Rls(Radiation):
 class Orog(Fx):
     """Fixes for orography"""
     @staticmethod
-    def _fix_units(cube):
-        # Divide by acceleration of gravity [m s-2],
-        # required for geopotential height, see:
-        # https://apps.ecmwf.int/codes/grib/param-db?id=129
+    def _divide_by_gravity(cube):
         cube.units = cube.units / 'm s-2'
         cube.data = cube.core_data() / 9.80665
         return cube
@@ -112,7 +139,7 @@ def _fix_units(cube):
     def fix_metadata(self, cubes):
         super().fix_metadata(cubes)
         for cube in cubes:
-            self._fix_units(cube)
+            self._divide_by_gravity(cube)
         return cubes
 
 

From c881ae7af8af41b70136cb5f381a8ff4b3c62d7d Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 22 Jan 2020 15:59:43 +0100
Subject: [PATCH 26/40] Fix some of the remaining issues

---
 esmvalcore/cmor/_fixes/native6/era5.py        | 34 +++++++----
 esmvalcore/config-developer.yml               |  1 +
 .../cmor/_fixes/native6/test_era5.py          | 56 +++++++++++++------
 3 files changed, 62 insertions(+), 29 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index d599f05f97..841cf0fc1a 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -1,6 +1,8 @@
 """Fixes for ERA5."""
+import iris
 import numpy as np
 from iris.cube import CubeList
+from iris.exceptions import CoordinateNotFoundError
 
 from ..fix import Fix
 from ..shared import add_scalar_height_coord
@@ -10,14 +12,22 @@ class FixEra5(Fix):
     """Fixes for ERA5 variables"""
     @staticmethod
     def _frequency(cube):
-        try:
+
+        if not cube.coords(axis='T'):
+            return 'fx'
+        else:
             time = cube.coord(axis='T')
-        except CoordinateNotFoundError:
-            time = None
 
-        if not time or len(time.points) == 1:
+        if len(time.points) == 1:
             return 'fx'
-        elif time.points[1] - time.points[0] == 1:
+
+        interval = time.points[1] - time.points[0]
+        unit = 'hours' if 'hour' in time.units.name else 'days'
+        print(time.units.name)
+        print(unit)
+        if (unit == 'hours'
+                and interval == 1) or (unit == 'days'
+                                       and interval - 1 / 24 < 1e-4):
             return 'hourly'
         return 'monthly'
 
@@ -25,7 +35,7 @@ def _frequency(cube):
 class Accumulated(FixEra5):
     """Fixes for accumulated variables."""
     def _fix_frequency(self, cube):
-        if cube.varname in ['mx2t', 'mn2t']:
+        if cube.var_name in ['mx2t', 'mn2t']:
             pass
         elif self._frequency(cube) == 'monthly':
             cube.units = cube.units * 'd-1'
@@ -73,15 +83,17 @@ def fix_metadata(self, cubes):
 
 class Fx(FixEra5):
     """Fixes for time invariant variables."""
-    @staticmethod
-    def _remove_time_coordinate(cube):
+    def _remove_time_coordinate(self, cube):
         cube = iris.util.squeeze(cube)
         cube.remove_coord('time')
         return cube
 
     def fix_metadata(self, cubes):
+        squeezed_cubes = []
         for cube in cubes:
-            self._remove_time_coordinate(cube)
+            cube = self._remove_time_coordinate(cube)
+            squeezed_cubes.append(cube)
+        return CubeList(squeezed_cubes)
 
 
 class Tasmin(Accumulated):
@@ -137,7 +149,7 @@ def _divide_by_gravity(cube):
         return cube
 
     def fix_metadata(self, cubes):
-        super().fix_metadata(cubes)
+        cubes = super().fix_metadata(cubes)
         for cube in cubes:
             self._divide_by_gravity(cube)
         return cubes
@@ -173,7 +185,7 @@ def _fix_coordinates(self, cube):
             coord.var_name = coord_def.out_name
             coord.long_name = coord_def.long_name
             coord.points = coord.core_points().astype('float64')
-            if coord_def.must_have_bounds == "yes":
+            if len(coord.points) > 1 and coord_def.must_have_bounds == "yes":
                 coord.guess_bounds()
 
         self._fix_monthly_time_coord(cube)
diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml
index 0b875e1fb1..0285665990 100644
--- a/esmvalcore/config-developer.yml
+++ b/esmvalcore/config-developer.yml
@@ -198,6 +198,7 @@ native6:
     era5cli: 'era5_{era5_name}*{era5_freq}.nc'
   output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}'
   cmor_type: 'CMIP6'
+  cmor_default_table_prefix: 'CMIP6_'
 
 obs4mips:
   cmor_strict: false
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index 73565dc72b..8bdb0c3b4d 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -4,7 +4,7 @@
 import pytest
 from cf_units import Unit
 
-from esmvalcore.cmor._fixes.native6.era5 import AllVars, Evspsbl
+from esmvalcore.cmor._fixes.native6.era5 import AllVars, Evspsbl, FixEra5
 from esmvalcore.cmor.fix import Fix, fix_metadata
 from esmvalcore.cmor.table import CMOR_TABLES
 
@@ -15,20 +15,36 @@ def test_get_evspsbl_fix():
     assert fix == [Evspsbl(None), AllVars(None)]
 
 
+def test_get_frequency():
+    fix = FixEra5(None)
+    frequencies = {'hourly': [0, 1, 2], 'monthly': [0, 31, 59], 'fx': [0]}
+    for frequency, points in frequencies.items():
+        data = np.zeros(len(points))
+        time = iris.coords.DimCoord(points,
+                                    standard_name='time',
+                                    units=Unit('hours since 1900-01-01'))
+        cube = iris.cube.Cube(data,
+                              var_name='random_varname',
+                              dim_coords_and_dims=[(time, 0)])
+        assert fix._frequency(cube) == frequency
+        cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0')
+        assert fix._frequency(cube) == frequency
+    cube = iris.cube.Cube(1., var_name='random')
+    assert fix._frequency(cube) == 'fx'
+
+
 def make_src_cubes(units, mip='E1hr'):
     """Make dummy cube that looks like the ERA5 data."""
 
-    # Adapt data and time coordinate for different mips
     data = np.arange(27).reshape(3, 3, 3)
     if mip == 'E1hr':
-        timestamps = [788928, 788929, 788930]  # 3 consecutive hours
+        timestamps = [788928, 788929, 788930]
     elif mip == 'Amon':
-        timestamps = [788928, 789672, 790344]  # 3 consecutive months
-    elif mip == 'Fx':
-        timestamps = [788928]  # 1 single timestamp
+        timestamps = [788928, 789672, 790344]
+    elif mip == 'fx':
+        timestamps = [788928]
         data = np.arange(9).reshape(1, 3, 3)
 
-    # Create coordinates
     latitude = iris.coords.DimCoord(np.array([90., 0., -90.]),
                                     standard_name='latitude',
                                     long_name='latitude',
@@ -79,11 +95,9 @@ def make_target_cubes(project, mip, short_name):
         if 'time1' not in vardef.dimensions:
             bounds = np.array([[51134.0, 51165.0], [51165.0, 51193.0],
                                [51193.0, 51224.0]])
-    elif mip == 'Fx':
-        data = np.arange(9).reshape(1, 3, 3)[:, ::-1, :]
-        timestamps = [51134.0]
-        if 'time1' not in vardef.dimensions:
-            bounds = np.array([[51133.9791667, 51134.0208333]])
+    elif mip == 'fx':
+        data = np.arange(9).reshape(3, 3)[::-1, :]
+        timestamps = [51143]
 
     # Make lat/lon/time coordinates
     latitude = iris.coords.DimCoord(np.array([-90., 0., 90.]),
@@ -115,13 +129,18 @@ def make_target_cubes(project, mip, short_name):
     if vardef.positive:
         attributes['positive'] = vardef.positive
 
+    if mip == 'fx':
+        dims = [(latitude, 0), (longitude, 1)]
+    else:
+        dims = [(time, 0), (latitude, 1), (longitude, 2)]
+
     cube = iris.cube.Cube(
         data.astype('float32'),
         long_name=vardef.long_name,
         var_name=short_name,  # vardef.cmor_name after #391?,
         standard_name=vardef.standard_name,
         units=Unit(vardef.units),
-        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
+        dim_coords_and_dims=dims,
         attributes=attributes)
 
     # Add auxiliary height coordinate for certain variables
@@ -143,7 +162,8 @@ def make_target_cubes(project, mip, short_name):
     ['pr', 'E1hr', 'm'],
     ['evspsbl', 'E1hr', 'm'],
     ['mrro', 'E1hr', 'm'],
-    ['prsn', 'E1hr', 'm of water equivalent'],
+    ['prsn', 'E1hr',
+     'unknown'],  # can't create testcube with 'm of water equivalent'
     ['evspsblpot', 'E1hr', 'm'],
     ['rss', 'E1hr', 'J m**-2'],
     ['rsds', 'E1hr', 'J m**-2'],
@@ -152,7 +172,7 @@ def make_target_cubes(project, mip, short_name):
     ['uas', 'E1hr', 'm s**-1'],  # a variable without explicit fixes
     ['pr', 'Amon', 'm'],  # a monthly variable
     # ['ua', 'E1hr', 'm s**-1'], # a 4d variable (we decided not to do this now)
-    ['orog', 'Fx', 'm**2 s**-2']  # a 2D variable (but keep time coord)
+    ['orog', 'fx', 'm**2 s**-2']  # a 2D variable
 ]
 
 
@@ -164,9 +184,9 @@ def test_cmorization(variable):
     src_cubes = make_src_cubes(era5_units, mip=mip)
     target_cubes = make_target_cubes('native6', mip, short_name)
 
-    print(src_cubes[0].xml())
     out_cubes = fix_metadata(src_cubes, short_name, 'native6', 'era5', mip)
 
-    print(out_cubes[0].xml())  # for testing purposes
-    print(target_cubes[0].xml())  # during development
+    print('src_cube:', src_cubes[0].xml())
+    print('out_cube:', out_cubes[0].xml())
+    print('target_cube:', target_cubes[0].xml())
     assert out_cubes[0].xml() == target_cubes[0].xml()

From 5dbb3a8617e13a944b197ce58001c244d1a563f7 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 22 Jan 2020 17:33:45 +0100
Subject: [PATCH 27/40] fix formatting issues

---
 esmvalcore/cmor/_fixes/native6/era5.py        | 38 +++++++++++++------
 .../cmor/_fixes/native6/test_era5.py          |  8 ++--
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 841cf0fc1a..c26993daac 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -1,23 +1,21 @@
 """Fixes for ERA5."""
-import iris
 import numpy as np
-from iris.cube import CubeList
-from iris.exceptions import CoordinateNotFoundError
+import iris
 
 from ..fix import Fix
 from ..shared import add_scalar_height_coord
 
 
 class FixEra5(Fix):
-    """Fixes for ERA5 variables"""
+    """Fixes for ERA5 variables."""
+
     @staticmethod
     def _frequency(cube):
 
         if not cube.coords(axis='T'):
             return 'fx'
-        else:
-            time = cube.coord(axis='T')
 
+        time = cube.coord(axis='T')
         if len(time.points) == 1:
             return 'fx'
 
@@ -34,6 +32,7 @@ def _frequency(cube):
 
 class Accumulated(FixEra5):
     """Fixes for accumulated variables."""
+
     def _fix_frequency(self, cube):
         if cube.var_name in ['mx2t', 'mn2t']:
             pass
@@ -44,12 +43,13 @@ def _fix_frequency(self, cube):
         return cube
 
     def _fix_hourly_time_coordinate(self, cube):
-        if self._frequency == 'hourly':
+        if self._frequency(cube) == 'hourly':
             time = cube.coord(axis='T')
             time.points = time.points - 0.5
         return cube
 
     def fix_metadata(self, cubes):
+        """Fix metadata."""
         super().fix_metadata(cubes)
         for cube in cubes:
             self._fix_hourly_time_coordinate(...)
@@ -59,6 +59,7 @@ def fix_metadata(self, cubes):
 
 class Hydrological(FixEra5):
     """Fixes for hydrological variables."""
+
     @staticmethod
     def _fix_units(cube):
         cube.units = 'kg m-2 s-1'
@@ -66,6 +67,7 @@ def _fix_units(cube):
         return cube
 
     def fix_metadata(self, cubes):
+        """Fix metadata."""
         super().fix_metadata(cubes)
         for cube in cubes:
             self._fix_units(cube)
@@ -74,26 +76,35 @@ def fix_metadata(self, cubes):
 
 class Radiation(FixEra5):
     """Fixes for accumulated radiation variables."""
+
+    @staticmethod
+    def _fix_direction(cube):
+        cube.attributes['positive'] = 'down'
+
     def fix_metadata(self, cubes):
+        """Fix metadata."""
         super().fix_metadata(cubes)
         for cube in cubes:
-            cube.attributes['positive'] = 'down'
+            self._fix_direction(cube)
         return cubes
 
 
 class Fx(FixEra5):
     """Fixes for time invariant variables."""
-    def _remove_time_coordinate(self, cube):
+
+    @staticmethod
+    def _remove_time_coordinate(cube):
         cube = iris.util.squeeze(cube)
         cube.remove_coord('time')
         return cube
 
     def fix_metadata(self, cubes):
+        """Fix metadata."""
         squeezed_cubes = []
         for cube in cubes:
             cube = self._remove_time_coordinate(cube)
             squeezed_cubes.append(cube)
-        return CubeList(squeezed_cubes)
+        return iris.cube.CubeList(squeezed_cubes)
 
 
 class Tasmin(Accumulated):
@@ -141,7 +152,8 @@ class Rls(Radiation):
 
 
 class Orog(Fx):
-    """Fixes for orography"""
+    """Fixes for orography."""
+
     @staticmethod
     def _divide_by_gravity(cube):
         cube.units = cube.units / 'm s-2'
@@ -149,6 +161,7 @@ def _divide_by_gravity(cube):
         return cube
 
     def fix_metadata(self, cubes):
+        """Fix metadata."""
         cubes = super().fix_metadata(cubes)
         for cube in cubes:
             self._divide_by_gravity(cube)
@@ -157,6 +170,7 @@ def fix_metadata(self, cubes):
 
 class AllVars(FixEra5):
     """Fixes for all variables."""
+
     def _fix_coordinates(self, cube):
         """Fix coordinates."""
         # Fix coordinate increasing direction
@@ -215,7 +229,7 @@ def _fix_units(self, cube):
 
     def fix_metadata(self, cubes):
         """Fix metadata."""
-        fixed_cubes = CubeList()
+        fixed_cubes = iris.cube.CubeList()
         for cube in cubes:
             cube.var_name = self.vardef.short_name
             cube.standard_name = self.vardef.standard_name
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index 8bdb0c3b4d..c9f0289e3a 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -16,6 +16,7 @@ def test_get_evspsbl_fix():
 
 
 def test_get_frequency():
+    """Test whether FixEra5._frequency infers the frequency correctly."""
     fix = FixEra5(None)
     frequencies = {'hourly': [0, 1, 2], 'monthly': [0, 31, 59], 'fx': [0]}
     for frequency, points in frequencies.items():
@@ -35,7 +36,6 @@ def test_get_frequency():
 
 def make_src_cubes(units, mip='E1hr'):
     """Make dummy cube that looks like the ERA5 data."""
-
     data = np.arange(27).reshape(3, 3, 3)
     if mip == 'E1hr':
         timestamps = [788928, 788929, 788930]
@@ -157,7 +157,7 @@ def make_target_cubes(project, mip, short_name):
     return [cube]
 
 
-variables = [
+VARIABLES = [
     # short_name, mip, era5_units, ndims
     ['pr', 'E1hr', 'm'],
     ['evspsbl', 'E1hr', 'm'],
@@ -171,12 +171,12 @@ def make_target_cubes(project, mip, short_name):
     ['rls', 'E1hr', 'W m**-2'],  # variables with explicit fixes
     ['uas', 'E1hr', 'm s**-1'],  # a variable without explicit fixes
     ['pr', 'Amon', 'm'],  # a monthly variable
-    # ['ua', 'E1hr', 'm s**-1'], # a 4d variable (we decided not to do this now)
+    # ['ua', 'E1hr', 'm s**-1'], # a 4d variable (decided not to do this now)
     ['orog', 'fx', 'm**2 s**-2']  # a 2D variable
 ]
 
 
-@pytest.mark.parametrize('variable', variables)
+@pytest.mark.parametrize('variable', VARIABLES)
 def test_cmorization(variable):
     """Verify that cmorization results in the expected target cube."""
     short_name, mip, era5_units = variable

From 2a148f2bb1c1aa102ef9b771c0d568dcd5945cbe Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 23 Jan 2020 15:49:54 +0100
Subject: [PATCH 28/40] Refactor tests

---
 esmvalcore/cmor/_fixes/native6/era5.py        |  11 +-
 .../cmor/_fixes/native6/test_era5.py          | 489 +++++++++++++-----
 2 files changed, 368 insertions(+), 132 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index c26993daac..d8739ee671 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -14,18 +14,15 @@ def _frequency(cube):
 
         if not cube.coords(axis='T'):
             return 'fx'
-
         time = cube.coord(axis='T')
         if len(time.points) == 1:
             return 'fx'
 
         interval = time.points[1] - time.points[0]
         unit = 'hours' if 'hour' in time.units.name else 'days'
-        print(time.units.name)
-        print(unit)
-        if (unit == 'hours'
-                and interval == 1) or (unit == 'days'
-                                       and interval - 1 / 24 < 1e-4):
+        if (unit == 'hours' and interval == 1):
+            return 'hourly'
+        if (unit == 'days' and interval - 1 / 24 < 1e-4):
             return 'hourly'
         return 'monthly'
 
@@ -52,7 +49,7 @@ def fix_metadata(self, cubes):
         """Fix metadata."""
         super().fix_metadata(cubes)
         for cube in cubes:
-            self._fix_hourly_time_coordinate(...)
+            self._fix_hourly_time_coordinate(cube)
             self._fix_frequency(cube)
         return cubes
 
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index c9f0289e3a..d1c23b1da4 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -16,7 +16,7 @@ def test_get_evspsbl_fix():
 
 
 def test_get_frequency():
-    """Test whether FixEra5._frequency infers the frequency correctly."""
+    """Test for FixEra5._frequency."""
     fix = FixEra5(None)
     frequencies = {'hourly': [0, 1, 2], 'monthly': [0, 31, 59], 'fx': [0]}
     for frequency, points in frequencies.items():
@@ -34,29 +34,31 @@ def test_get_frequency():
     assert fix._frequency(cube) == 'fx'
 
 
-def make_src_cubes(units, mip='E1hr'):
-    """Make dummy cube that looks like the ERA5 data."""
-    data = np.arange(27).reshape(3, 3, 3)
-    if mip == 'E1hr':
+def _era5_latitude():
+    return iris.coords.DimCoord(np.array([90., 0., -90.]),
+                                standard_name='latitude',
+                                long_name='latitude',
+                                var_name='latitude',
+                                units=Unit('degrees'))
+
+
+def _era5_longitude():
+    return iris.coords.DimCoord(np.array([0, 180, 359.75]),
+                                standard_name='longitude',
+                                long_name='longitude',
+                                var_name='longitude',
+                                units=Unit('degrees'),
+                                circular=True)
+
+
+def _era5_time(frequency):
+    if frequency == 'invariant':
+        timestamps = [788928]
+    elif frequency == 'hourly':
         timestamps = [788928, 788929, 788930]
-    elif mip == 'Amon':
+    elif frequency == 'monthly':
         timestamps = [788928, 789672, 790344]
-    elif mip == 'fx':
-        timestamps = [788928]
-        data = np.arange(9).reshape(1, 3, 3)
-
-    latitude = iris.coords.DimCoord(np.array([90., 0., -90.]),
-                                    standard_name='latitude',
-                                    long_name='latitude',
-                                    var_name='latitude',
-                                    units=Unit('degrees'))
-    longitude = iris.coords.DimCoord(np.array([0, 180, 359.75]),
-                                     standard_name='longitude',
-                                     long_name='longitude',
-                                     var_name='longitude',
-                                     units=Unit('degrees'),
-                                     circular=True)
-    time = iris.coords.DimCoord(np.array(timestamps, dtype='int32'),
+    return iris.coords.DimCoord(np.array(timestamps, dtype='int32'),
                                 standard_name='time',
                                 long_name='time',
                                 var_name='time',
@@ -65,58 +67,56 @@ def make_src_cubes(units, mip='E1hr'):
                                     '00:00:00.0',
                                     calendar='gregorian'))
 
-    cube = iris.cube.Cube(
-        data,
-        long_name='random_long_name',
-        var_name='random_var_name',
-        units=units,
-        dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
-    )
-    return iris.cube.CubeList([cube])
 
+def _era5_data(frequency):
+    if frequency == 'invariant':
+        return np.arange(9).reshape(1, 3, 3)
+    return np.arange(27).reshape(3, 3, 3)
+
+
+def _cmor_latitude():
+    return iris.coords.DimCoord(np.array([-90., 0., 90.]),
+                                standard_name='latitude',
+                                long_name='Latitude',
+                                var_name='lat',
+                                units=Unit('degrees_north'),
+                                bounds=np.array([[-90., -45.], [-45., 45.],
+                                                 [45., 90.]]))
 
-def make_target_cubes(project, mip, short_name):
-    """Make dummy cube that looks like the CMORized data."""
-    # Look up variable definition in CMOR table
-    cmor_table = CMOR_TABLES[project]
-    vardef = cmor_table.get_variable(mip, short_name)
-
-    # Make up time dimension and data (shape)
-    data = np.arange(27).reshape(3, 3, 3)[:, ::-1, :]
-    bounds = None
-    if mip == 'E1hr':
-        timestamps = [51134.0, 51134.0416667, 51134.0833333]
-        if 'time1' not in vardef.dimensions:
-            bounds = np.array([[51133.9791667, 51134.0208333],
-                               [51134.0208333, 51134.0625],
-                               [51134.0625, 51134.1041667]])
-    elif mip == 'Amon':
-        timestamps = [51149.5, 51179.0, 51208.5]
-        if 'time1' not in vardef.dimensions:
+
+def _cmor_longitude():
+    return iris.coords.DimCoord(np.array([0, 180, 359.75]),
+                                standard_name='longitude',
+                                long_name='Longitude',
+                                var_name='lon',
+                                units=Unit('degrees_east'),
+                                bounds=np.array([[-0.125, 90.], [90., 269.875],
+                                                 [269.875, 359.875]]),
+                                circular=True)
+
+
+def _cmor_time(mip, bounds=None, shifted=False):
+    """Provide expected time coordinate after fixes."""
+    if mip is 'E1hr':
+        if not shifted:
+            timestamps = [51134.0, 51134.0416667, 51134.0833333]
+            if bounds is not None:
+                bounds = np.array([[51133.9791667, 51134.0208333],
+                                   [51134.0208333, 51134.0625],
+                                   [51134.0625, 51134.1041667]])
+        else:
+            timestamps = [51133.97916667, 51134.02083333, 51134.0625]
+            if bounds is not None:
+                bounds = np.array([[51133.95833333, 51134.0],
+                                   [51134.0, 51134.04166667],
+                                   [51134.04166667, 51134.08333333]])
+    elif mip is 'Amon':
+        timestamps = np.array([51149.5, 51179.0, 51208.5])
+        if bounds is not None:
             bounds = np.array([[51134.0, 51165.0], [51165.0, 51193.0],
                                [51193.0, 51224.0]])
-    elif mip == 'fx':
-        data = np.arange(9).reshape(3, 3)[::-1, :]
-        timestamps = [51143]
-
-    # Make lat/lon/time coordinates
-    latitude = iris.coords.DimCoord(np.array([-90., 0., 90.]),
-                                    standard_name='latitude',
-                                    long_name='Latitude',
-                                    var_name='lat',
-                                    units=Unit('degrees_north'),
-                                    bounds=np.array([[-90., -45.], [-45., 45.],
-                                                     [45., 90.]]))
-    longitude = iris.coords.DimCoord(np.array([0, 180, 359.75]),
-                                     standard_name='longitude',
-                                     long_name='Longitude',
-                                     var_name='lon',
-                                     units=Unit('degrees_east'),
-                                     bounds=np.array([[-0.125, 90.],
-                                                      [90., 269.875],
-                                                      [269.875, 359.875]]),
-                                     circular=True)
-    time = iris.coords.DimCoord(np.array(timestamps, dtype=float),
+
+    return iris.coords.DimCoord(np.array(timestamps, dtype=float),
                                 standard_name='time',
                                 long_name='time',
                                 var_name='time',
@@ -124,69 +124,308 @@ def make_target_cubes(project, mip, short_name):
                                            calendar='gregorian'),
                                 bounds=bounds)
 
-    # Make dummy cube that's the cmor equivalent of the era5 dummy cube.
-    attributes = {}
-    if vardef.positive:
-        attributes['positive'] = vardef.positive
 
-    if mip == 'fx':
-        dims = [(latitude, 0), (longitude, 1)]
-    else:
-        dims = [(time, 0), (latitude, 1), (longitude, 2)]
+def _cmor_aux_height(value):
+    return iris.coords.AuxCoord(value,
+                                long_name="height",
+                                standard_name="height",
+                                units=Unit('m'),
+                                var_name="height",
+                                attributes={'positive': 'up'})
+
+
+def _cmor_data(mip):
+    if mip is 'fx':
+        return np.arange(9).reshape(3, 3)[::-1, :]
+    return np.arange(27).reshape(3, 3, 3)[:, ::-1, :]
+
+
+def pr_era5_monthly():
+    time = _era5_time('monthly')
+    cube = iris.cube.Cube(
+        _era5_data('monthly'),
+        long_name='total_precipitation',
+        var_name='tp',
+        units='m',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def pr_cmor_amon():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('Amon', 'pr')
+    time = _cmor_time('Amon', bounds=True)
+    data = _cmor_data('Amon') * 1000.
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)])
+    return iris.cube.CubeList([cube])
+
+
+def pr_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='total_precipitation',
+        var_name='tp',
+        units='m',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def pr_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'pr')
+    time = _cmor_time('E1hr', bounds=True, shifted=True)
+    data = _cmor_data('E1hr') * 1000.
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)])
+    return iris.cube.CubeList([cube])
+
+
+def orog_era5_hourly():
+    time = _era5_time('invariant')
+    cube = iris.cube.Cube(
+        _era5_data('invariant'),
+        long_name='geopotential height',
+        var_name='zg',
+        units='m**2 s**-2',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def orog_cmor_fx():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('fx', 'orog')
+    data = _cmor_data('fx') / 9.80665
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(_cmor_latitude(), 0),
+                                               (_cmor_longitude(), 1)])
+    return iris.cube.CubeList([cube])
+
+
+def uas_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='10m_u_component_of_wind',
+        var_name='u10',
+        units='m s-1',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def uas_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'uas')
+    time = _cmor_time('E1hr')
+    data = _cmor_data('E1hr')
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)])
+    cube.add_aux_coord(_cmor_aux_height(10.))
+    return iris.cube.CubeList([cube])
+
+
+def tas_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='2m_temperature',
+        var_name='t2m',
+        units='degC',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def tas_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'tas')
+    time = _cmor_time('E1hr')
+    data = _cmor_data('E1hr') + 273.15
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)])
+    cube.add_aux_coord(_cmor_aux_height(2.))
+    return iris.cube.CubeList([cube])
+
+
+def rsds_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='solar_radiation_downwards',
+        var_name='rlwd',
+        units='J m**-2',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def rsds_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'rsds')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') / 3600
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={'positive': 'down'})
+    return iris.cube.CubeList([cube])
+
 
+def prsn_era5_hourly():
+    time = _era5_time('hourly')
     cube = iris.cube.Cube(
-        data.astype('float32'),
-        long_name=vardef.long_name,
-        var_name=short_name,  # vardef.cmor_name after #391?,
-        standard_name=vardef.standard_name,
-        units=Unit(vardef.units),
-        dim_coords_and_dims=dims,
-        attributes=attributes)
-
-    # Add auxiliary height coordinate for certain variables
-    if short_name in ['uas', 'vas', 'tas', 'tasmin', 'tasmax']:
-        value = 10. if short_name in ['uas', 'vas'] else 2.
-        aux_coord = iris.coords.AuxCoord([value],
-                                         long_name="height",
-                                         standard_name="height",
-                                         units=Unit('m'),
-                                         var_name="height",
-                                         attributes={'positive': 'up'})
-        cube.add_aux_coord(aux_coord, ())
-
-    return [cube]
+        _era5_data('hourly'),
+        long_name='snow',
+        var_name='snow',
+        units='unknown',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def prsn_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'prsn')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') * 1000
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)])
+    return iris.cube.CubeList([cube])
+
+
+def mrro_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='runoff',
+        var_name='runoff',
+        units='m',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
 
+def mrro_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'mrro')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') * 1000
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)])
+    return iris.cube.CubeList([cube])
+
+
+def rls_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='runoff',
+        var_name='runoff',
+        units='W m**-2',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def rls_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'rls')
+    time = _cmor_time('E1hr', shifted=False, bounds=True)
+    data = _cmor_data('E1hr')
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={'positive': 'down'})
+    return iris.cube.CubeList([cube])
 
 VARIABLES = [
-    # short_name, mip, era5_units, ndims
-    ['pr', 'E1hr', 'm'],
-    ['evspsbl', 'E1hr', 'm'],
-    ['mrro', 'E1hr', 'm'],
-    ['prsn', 'E1hr',
-     'unknown'],  # can't create testcube with 'm of water equivalent'
-    ['evspsblpot', 'E1hr', 'm'],
-    ['rss', 'E1hr', 'J m**-2'],
-    ['rsds', 'E1hr', 'J m**-2'],
-    ['rsdt', 'E1hr', 'J m**-2'],
-    ['rls', 'E1hr', 'W m**-2'],  # variables with explicit fixes
-    ['uas', 'E1hr', 'm s**-1'],  # a variable without explicit fixes
-    ['pr', 'Amon', 'm'],  # a monthly variable
-    # ['ua', 'E1hr', 'm s**-1'], # a 4d variable (decided not to do this now)
-    ['orog', 'fx', 'm**2 s**-2']  # a 2D variable
+    pytest.param(a, b, c, d, id=c + '_' + d) for (a, b, c, d) in [
+        (pr_era5_monthly(), pr_cmor_amon(), 'pr', 'Amon'),
+        (pr_era5_hourly(), pr_cmor_e1hr(), 'pr', 'E1hr'),
+        (orog_era5_hourly(), orog_cmor_fx(), 'orog', 'fx'),
+        (uas_era5_hourly(), uas_cmor_e1hr(), 'uas', 'E1hr'),
+        (tas_era5_hourly(), tas_cmor_e1hr(), 'tas', 'E1hr'),
+        (rsds_era5_hourly(), rsds_cmor_e1hr(), 'rsds', 'E1hr'),
+        (prsn_era5_hourly(), prsn_cmor_e1hr(), 'prsn', 'E1hr'),
+        (mrro_era5_hourly(), mrro_cmor_e1hr(), 'mrro', 'E1hr'),
+        (rls_era5_hourly(), rls_cmor_e1hr(), 'rls', 'E1hr'),
+    ]
 ]
 
 
-@pytest.mark.parametrize('variable', VARIABLES)
-def test_cmorization(variable):
+@pytest.mark.parametrize('era5_cubes, cmor_cubes, var, mip', VARIABLES)
+def test_cmorization(era5_cubes, cmor_cubes, var, mip):
     """Verify that cmorization results in the expected target cube."""
-    short_name, mip, era5_units = variable
-
-    src_cubes = make_src_cubes(era5_units, mip=mip)
-    target_cubes = make_target_cubes('native6', mip, short_name)
-
-    out_cubes = fix_metadata(src_cubes, short_name, 'native6', 'era5', mip)
 
-    print('src_cube:', src_cubes[0].xml())
-    print('out_cube:', out_cubes[0].xml())
-    print('target_cube:', target_cubes[0].xml())
-    assert out_cubes[0].xml() == target_cubes[0].xml()
+    fixed_cubes = fix_metadata(era5_cubes, var, 'native6', 'era5', mip)
+    print('era5_cube:', era5_cubes[0].xml())
+    print('cmor_cube:', cmor_cubes[0].xml())
+    print('fixed_cube:', fixed_cubes[0].xml())
+    assert fixed_cubes[0].xml() == cmor_cubes[0].xml()
+    assert (fixed_cubes[0].data == cmor_cubes[0].data).all()
+    # assert fixed_cubes[0].coords() == cmor_cubes[0].coords()
+    # assert fixed_cubes[0] == cmor_cubes[0]

From 951be2d9991ff7eba97cb8ad89e3eb77f0b5beb5 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 23 Jan 2020 17:43:28 +0100
Subject: [PATCH 29/40] remove print statement

---
 esmvalcore/_data_finder.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index f035df05b6..cc98368f6d 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -52,7 +52,6 @@ def get_start_end_year(filename):
         end_year = end_date[:4]
     else:
         dates = re.findall(r'([0-9]{4,12})', filename)
-        print(dates)
         if not dates:
             raise ValueError('Name {0} does not match a recognized '
                              'pattern'.format(filename))

From b79889b8c48412d6a21db086137c14e01320e82b Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 29 Jan 2020 13:17:08 +0100
Subject: [PATCH 30/40] Integrate recent updates in get_start_end_year

---
 esmvalcore/_data_finder.py | 46 +++++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index cc98368f6d..9d0eec3a47 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -30,41 +30,47 @@ def find_files(dirnames, filenames):
 
 
 def get_start_end_year(filename):
-    """Get the start and end year from a file name.
-
-    We assume that the filename *must* contain at least 1 year/date.
-    We look for four-to-eight-digit numbers.
-    If two potential dates are separated by either - or _,
-    we interpret this as the _daterange_ (startdate_enddate).
-    If no date range is present, but multiple potential dates are
-    present (e.g. including a version number), we assume that the last
-    number represents the date.
-    """
-    # Strip only filename from full path (and discard extension?)
+    """Get the start and end year from a file name."""
     filename = os.path.splitext(filename)[0]
     filename = filename.split(os.sep)[-1]
 
-    # Check for a block of two potential dates separated by _ or -
+    # First check for a block of two potential dates separated by _ or -
     daterange = re.findall(r'([0-9]{4,12}[-_][0-9]{4,12})', filename)
     if daterange:
         start_date, end_date = re.findall(r'([0-9]{4,12})', daterange[0])
         start_year = start_date[:4]
         end_year = end_date[:4]
     else:
+        # Check for single dates in the filename
         dates = re.findall(r'([0-9]{4,12})', filename)
-        if not dates:
-            raise ValueError('Name {0} does not match a recognized '
-                             'pattern'.format(filename))
-        elif len(dates) == 1:
+        if len(dates) == 1:
             start_year = end_year = dates[0][:4]
-        else:
+        elif len(dates) > 1:
             # Check for dates at start or end of filename
             outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$', filename)
             if len(outerdates) == 1:
                 start_year = end_year = outerdates[0][:4]
-            else:
-                raise ValueError('Name {0} does not match a recognized '
-                                 'pattern'.format(filename))
+
+    # As final resort, try to get the dates from the file contents
+    if start_year is None or end_year is None:
+        try:
+            cubes = iris.load(filename)
+        except OSError:
+            raise ValueError(f'File {filename} can not be read')
+
+        for cube in cubes:
+            logger.debug(cube)
+            try:
+                time = cube.coord('time')
+            except iris.exceptions.CoordinateNotFoundError:
+                continue
+            start_year = time.cell(0).point.year
+            end_year = time.cell(-1).point.year
+            break
+
+    if start_year is None or end_year is None:
+        raise ValueError(f'File {filename} dates do not match a recognized'
+                         'pattern and time can not be read from the file')
 
     logger.debug("Found start_year %s and end_year %s", start_year, end_year)
     return int(start_year), int(end_year)

From de75535ad864deecfbdf66564adaa1f4e89ad28c Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Wed, 29 Jan 2020 14:56:13 +0100
Subject: [PATCH 31/40] Improve _frequency method for FixEra5

---
 esmvalcore/cmor/_fixes/native6/era5.py        | 28 +++++-----
 .../cmor/_fixes/native6/test_era5.py          | 54 +++++++++++++------
 2 files changed, 52 insertions(+), 30 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index d8739ee671..14722e1b5d 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -1,35 +1,38 @@
 """Fixes for ERA5."""
 import numpy as np
 import iris
+import logging
 
 from ..fix import Fix
 from ..shared import add_scalar_height_coord
 
+logger = logging.getLogger(__name__)
+
 
 class FixEra5(Fix):
     """Fixes for ERA5 variables."""
-
     @staticmethod
     def _frequency(cube):
-
-        if not cube.coords(axis='T'):
+        """Determine time frequency of input cube."""
+        try:
+            time = cube.coord(axis='T')
+        except iris.exceptions.CoordinateNotFoundError:
             return 'fx'
-        time = cube.coord(axis='T')
+
+        time.convert_units('days since 1850-1-1 00:00:00.0')
         if len(time.points) == 1:
+            if cube.long_name is not 'Geopotential':
+                raise ValueError('Unable to infer frequency of cube '
+                                 f'with length 1 time dimension: {cube}')
             return 'fx'
-
         interval = time.points[1] - time.points[0]
-        unit = 'hours' if 'hour' in time.units.name else 'days'
-        if (unit == 'hours' and interval == 1):
-            return 'hourly'
-        if (unit == 'days' and interval - 1 / 24 < 1e-4):
+        if interval - 1 / 24 < 1e-4:
             return 'hourly'
         return 'monthly'
 
 
 class Accumulated(FixEra5):
     """Fixes for accumulated variables."""
-
     def _fix_frequency(self, cube):
         if cube.var_name in ['mx2t', 'mn2t']:
             pass
@@ -56,7 +59,6 @@ def fix_metadata(self, cubes):
 
 class Hydrological(FixEra5):
     """Fixes for hydrological variables."""
-
     @staticmethod
     def _fix_units(cube):
         cube.units = 'kg m-2 s-1'
@@ -73,7 +75,6 @@ def fix_metadata(self, cubes):
 
 class Radiation(FixEra5):
     """Fixes for accumulated radiation variables."""
-
     @staticmethod
     def _fix_direction(cube):
         cube.attributes['positive'] = 'down'
@@ -88,7 +89,6 @@ def fix_metadata(self, cubes):
 
 class Fx(FixEra5):
     """Fixes for time invariant variables."""
-
     @staticmethod
     def _remove_time_coordinate(cube):
         cube = iris.util.squeeze(cube)
@@ -150,7 +150,6 @@ class Rls(Radiation):
 
 class Orog(Fx):
     """Fixes for orography."""
-
     @staticmethod
     def _divide_by_gravity(cube):
         cube.units = cube.units / 'm s-2'
@@ -167,7 +166,6 @@ def fix_metadata(self, cubes):
 
 class AllVars(FixEra5):
     """Fixes for all variables."""
-
     def _fix_coordinates(self, cube):
         """Fix coordinates."""
         # Fix coordinate increasing direction
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index d1c23b1da4..ce15e67366 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -15,23 +15,47 @@ def test_get_evspsbl_fix():
     assert fix == [Evspsbl(None), AllVars(None)]
 
 
-def test_get_frequency():
-    """Test for FixEra5._frequency."""
+def test_get_frequency_hourly():
     fix = FixEra5(None)
-    frequencies = {'hourly': [0, 1, 2], 'monthly': [0, 31, 59], 'fx': [0]}
-    for frequency, points in frequencies.items():
-        data = np.zeros(len(points))
-        time = iris.coords.DimCoord(points,
-                                    standard_name='time',
-                                    units=Unit('hours since 1900-01-01'))
-        cube = iris.cube.Cube(data,
-                              var_name='random_varname',
-                              dim_coords_and_dims=[(time, 0)])
-        assert fix._frequency(cube) == frequency
-        cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0')
-        assert fix._frequency(cube) == frequency
-    cube = iris.cube.Cube(1., var_name='random')
+    time = iris.coords.DimCoord([0, 1, 2],
+                                standard_name='time',
+                                units=Unit('hours since 1900-01-01'))
+    cube = iris.cube.Cube([1, 6, 3],
+                          var_name='random_var',
+                          dim_coords_and_dims=[(time, 0)])
+    assert fix._frequency(cube) == 'hourly'
+    cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0')
+    assert fix._frequency(cube) == 'hourly'
+
+
+def test_get_frequency_monthly():
+    fix = FixEra5(None)
+    time = iris.coords.DimCoord([0, 31, 59],
+                                standard_name='time',
+                                units=Unit('hours since 1900-01-01'))
+    cube = iris.cube.Cube([1, 6, 3],
+                          var_name='random_var',
+                          dim_coords_and_dims=[(time, 0)])
+    assert fix._frequency(cube) == 'monthly'
+    cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0')
+    assert fix._frequency(cube) == 'monthly'
+
+
+def test_get_frequency_fx():
+    fix = FixEra5(None)
+    cube = iris.cube.Cube(1., long_name='Cube without time coordinate')
+    assert fix._frequency(cube) == 'fx'
+    time = iris.coords.DimCoord(0,
+                                standard_name='time',
+                                units=Unit('hours since 1900-01-01'))
+    cube = iris.cube.Cube([1],
+                          var_name='cube_with_length_1_time_coord',
+                          long_name='Geopotential',
+                          dim_coords_and_dims=[(time, 0)])
     assert fix._frequency(cube) == 'fx'
+    cube.long_name = 'Not geopotential'
+    with pytest.raises(ValueError):
+        fix._frequency(cube)
 
 
 def _era5_latitude():

From 2ffc6fac482a91c3b24a6f3093badadaeb83e53f Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 30 Jan 2020 08:45:10 +0100
Subject: [PATCH 32/40] Resolve merge conflict in test_get_start_end_year

---
 esmvalcore/_data_finder.py                    | 12 +++++++-----
 .../data_finder/test_get_start_end_year.py    | 19 +++++++++++++++++++
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index 9d0eec3a47..c80b33f38e 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -5,6 +5,7 @@
 # Mattia Righi (DLR, Germany - mattia.righi@dlr.de)
 
 import fnmatch
+import iris
 import logging
 import os
 import re
@@ -31,23 +32,24 @@ def find_files(dirnames, filenames):
 
 def get_start_end_year(filename):
     """Get the start and end year from a file name."""
-    filename = os.path.splitext(filename)[0]
-    filename = filename.split(os.sep)[-1]
+    stripped_name = os.path.splitext(filename)[0]
+    stripped_name = stripped_name.split(os.sep)[-1]
+    start_year = end_year = None
 
     # First check for a block of two potential dates separated by _ or -
-    daterange = re.findall(r'([0-9]{4,12}[-_][0-9]{4,12})', filename)
+    daterange = re.findall(r'([0-9]{4,12}[-_][0-9]{4,12})', stripped_name)
     if daterange:
         start_date, end_date = re.findall(r'([0-9]{4,12})', daterange[0])
         start_year = start_date[:4]
         end_year = end_date[:4]
     else:
         # Check for single dates in the filename
-        dates = re.findall(r'([0-9]{4,12})', filename)
+        dates = re.findall(r'([0-9]{4,12})', stripped_name)
         if len(dates) == 1:
             start_year = end_year = dates[0][:4]
         elif len(dates) > 1:
             # Check for dates at start or end of filename
-            outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$', filename)
+            outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$', stripped_name)
             if len(outerdates) == 1:
                 start_year = end_year = outerdates[0][:4]
 
diff --git a/tests/unit/data_finder/test_get_start_end_year.py b/tests/unit/data_finder/test_get_start_end_year.py
index cc4ee5e4a5..f2b36ff81d 100644
--- a/tests/unit/data_finder/test_get_start_end_year.py
+++ b/tests/unit/data_finder/test_get_start_end_year.py
@@ -1,6 +1,9 @@
 """Unit tests for :func:`esmvalcore._data_finder.regrid._stock_cube`"""
 
+import iris
 import pytest
+import tempfile
+import os
 
 from esmvalcore._data_finder import get_start_end_year
 
@@ -19,6 +22,7 @@
     ['var_control-1950_whatever_19800101.nc', 1980, 1980],
 ]
 
+
 @pytest.mark.parametrize('case', FILENAME_CASES)
 def test_get_start_end_year(case):
     """Tests for get_start_end_year function"""
@@ -27,6 +31,21 @@ def test_get_start_end_year(case):
     assert case_start == start
     assert case_end == end
 
+
+def test_read_file_if_no_date_present():
+    """Test raises if no date is present"""
+    temp_file = tempfile.NamedTemporaryFile(suffix='.nc')
+    cube = iris.cube.Cube([0, 0], var_name='var')
+    time = iris.coords.DimCoord([0, 366],
+                                'time',
+                                units='days since 1990-01-01')
+    cube.add_dim_coord(time, 0)
+    iris.save(cube, temp_file.name)
+    start, end = get_start_end_year(temp_file.name)
+    assert start == 1990
+    assert end == 1991
+
+
 def test_fails_if_no_date_present():
     """Test raises if no date is present"""
     with pytest.raises(ValueError):

From 5ece3c4d5041ee5ba415cc03efbb9e8a1e0483cf Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 30 Jan 2020 16:23:53 +0100
Subject: [PATCH 33/40] Address remaining issues

---
 doc/esmvalcore/fixing_data.rst                | 22 ++++++--
 esmvalcore/_data_finder.py                    |  8 ++-
 esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py    |  6 ++-
 esmvalcore/cmor/_fixes/native6/era5.py        | 42 ++++++++++-----
 esmvalcore/preprocessor/_regrid.py            |  2 +-
 .../cmor/_fixes/native6/test_era5.py          | 53 +++++++++----------
 .../_regrid/test_get_file_levels.py           |  2 +-
 tests/integration/test_recipe.py              |  4 --
 tests/unit/cmor/test_fix.py                   | 17 ++++--
 .../data_finder/test_get_start_end_year.py    |  9 ++--
 10 files changed, 98 insertions(+), 67 deletions(-)

diff --git a/doc/esmvalcore/fixing_data.rst b/doc/esmvalcore/fixing_data.rst
index 2d661881a5..2b9f0f30bc 100644
--- a/doc/esmvalcore/fixing_data.rst
+++ b/doc/esmvalcore/fixing_data.rst
@@ -211,16 +211,28 @@ with the actual data units.
 
 .. code-block:: python
 
-    ...
-        def fix_metadata(self, cubes):
-            ...
-            cube.units = 'real_units'
-            ...
+    def fix_metadata(self, cubes):
+        cube.units = 'real_units'
+
 
 Detecting this error can be tricky if the units are similar enough. It also
 has a good chance of going undetected until you notice strange results in
 your diagnostic.
 
+For the above example, it can be useful to access the variable definition
+and associated coordinate definitions as provided by the CMOR table.
+For example:
+
+.. code-block:: python
+
+    def fix_metadata(self, cubes):
+        cube.units = self.vardef.units
+
+To learn more about what is available in these definitions, see:
+:class:`esmvalcore.cmor.table.VariableInfo` and
+:class:`esmvalcore.cmor.table.CoordinateInfo`.
+
+
 
 Coordinates missing
 -------------------
diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index c80b33f38e..16892fc2a3 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -49,16 +49,14 @@ def get_start_end_year(filename):
             start_year = end_year = dates[0][:4]
         elif len(dates) > 1:
             # Check for dates at start or end of filename
-            outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$', stripped_name)
+            outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$',
+                                    stripped_name)
             if len(outerdates) == 1:
                 start_year = end_year = outerdates[0][:4]
 
     # As final resort, try to get the dates from the file contents
     if start_year is None or end_year is None:
-        try:
-            cubes = iris.load(filename)
-        except OSError:
-            raise ValueError(f'File {filename} can not be read')
+        cubes = iris.load(filename)
 
         for cube in cubes:
             logger.debug(cube)
diff --git a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py
index 9efeac6111..8d59f16510 100644
--- a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py
+++ b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py
@@ -121,7 +121,8 @@ def fix_metadata(self, cubes):
         -------
         iris.cube.CubeList
         """
-        cubes[0].standard_name = 'sea_ice_x_velocity'
+        cube = self.get_cube_from_list(cubes)
+        cube.standard_name = 'sea_ice_x_velocity'
         return cubes
 
 
@@ -141,5 +142,6 @@ def fix_metadata(self, cubes):
         -------
         iris.cube.CubeList
         """
-        cubes[0].standard_name = 'sea_ice_y_velocity'
+        cube = self.get_cube_from_list(cubes)
+        cube.standard_name = 'sea_ice_y_velocity'
         return cubes
diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 14722e1b5d..ab3ccdd4ea 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -1,8 +1,10 @@
 """Fixes for ERA5."""
-import numpy as np
-import iris
+import datetime
 import logging
 
+import iris
+import numpy as np
+
 from ..fix import Fix
 from ..shared import add_scalar_height_coord
 
@@ -21,7 +23,7 @@ def _frequency(cube):
 
         time.convert_units('days since 1850-1-1 00:00:00.0')
         if len(time.points) == 1:
-            if cube.long_name is not 'Geopotential':
+            if cube.long_name != 'Geopotential':
                 raise ValueError('Unable to infer frequency of cube '
                                  f'with length 1 time dimension: {cube}')
             return 'fx'
@@ -34,9 +36,7 @@ def _frequency(cube):
 class Accumulated(FixEra5):
     """Fixes for accumulated variables."""
     def _fix_frequency(self, cube):
-        if cube.var_name in ['mx2t', 'mn2t']:
-            pass
-        elif self._frequency(cube) == 'monthly':
+        if self._frequency(cube) == 'monthly':
             cube.units = cube.units * 'd-1'
         elif self._frequency(cube) == 'hourly':
             cube.units = cube.units * 'h-1'
@@ -45,7 +45,8 @@ def _fix_frequency(self, cube):
     def _fix_hourly_time_coordinate(self, cube):
         if self._frequency(cube) == 'hourly':
             time = cube.coord(axis='T')
-            time.points = time.points - 0.5
+            time.points = time.points - 1 / 48
+            time.guess_bounds()
         return cube
 
     def fix_metadata(self, cubes):
@@ -91,7 +92,7 @@ class Fx(FixEra5):
     """Fixes for time invariant variables."""
     @staticmethod
     def _remove_time_coordinate(cube):
-        cube = iris.util.squeeze(cube)
+        cube = cube[0]
         cube.remove_coord('time')
         return cube
 
@@ -104,13 +105,26 @@ def fix_metadata(self, cubes):
         return iris.cube.CubeList(squeezed_cubes)
 
 
-class Tasmin(Accumulated):
+class Tasmin(FixEra5):
     """Fixes for tasmin."""
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            if self._frequency(cube) == 'hourly':
+                time = cube.coord(axis='T')
+                time.points = time.points - 1 / 48
+                time.guess_bounds()
+        return cubes
 
 
-class Tasmax(Accumulated):
+class Tasmax(FixEra5):
     """Fixes for tasmax."""
-
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            if self._frequency(cube) == 'hourly':
+                time = cube.coord(axis='T')
+                time.points = time.points - 1 / 48
+                time.guess_bounds()
+        return cubes
 
 class Evspsbl(Hydrological, Accumulated):
     """Fixes for evspsbl."""
@@ -194,7 +208,7 @@ def _fix_coordinates(self, cube):
             coord.var_name = coord_def.out_name
             coord.long_name = coord_def.long_name
             coord.points = coord.core_points().astype('float64')
-            if len(coord.points) > 1 and coord_def.must_have_bounds == "yes":
+            if (coord.bounds is not None and len(coord.points) > 1 and coord_def.must_have_bounds == "yes"):
                 coord.guess_bounds()
 
         self._fix_monthly_time_coord(cube)
@@ -234,6 +248,10 @@ def fix_metadata(self, cubes):
             self._fix_units(cube)
 
             cube.data = cube.core_data().astype('float32')
+            year = datetime.datetime.now().year
+            cube.attributes['comment'] = (
+                'Contains modified Copernicus Climate Change '
+                f'Service Information {year}')
 
             fixed_cubes.append(cube)
 
diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py
index 58a96491d4..463a8bc6f8 100644
--- a/esmvalcore/preprocessor/_regrid.py
+++ b/esmvalcore/preprocessor/_regrid.py
@@ -498,7 +498,7 @@ def get_reference_levels(filename,
                          dataset,
                          short_name,
                          fix_dir):
-    """Get level definition from a CMOR coordinate.
+    """Get level definition from a reference dataset.
 
     Parameters
     ----------
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index ce15e67366..3e1f35eac2 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -77,7 +77,7 @@ def _era5_longitude():
 
 def _era5_time(frequency):
     if frequency == 'invariant':
-        timestamps = [788928]
+        timestamps = [788928]  # hours since 1900 at 1 january 1990
     elif frequency == 'hourly':
         timestamps = [788928, 788929, 788930]
     elif frequency == 'monthly':
@@ -121,24 +121,18 @@ def _cmor_longitude():
 
 def _cmor_time(mip, bounds=None, shifted=False):
     """Provide expected time coordinate after fixes."""
-    if mip is 'E1hr':
-        if not shifted:
-            timestamps = [51134.0, 51134.0416667, 51134.0833333]
-            if bounds is not None:
-                bounds = np.array([[51133.9791667, 51134.0208333],
-                                   [51134.0208333, 51134.0625],
-                                   [51134.0625, 51134.1041667]])
-        else:
-            timestamps = [51133.97916667, 51134.02083333, 51134.0625]
-            if bounds is not None:
-                bounds = np.array([[51133.95833333, 51134.0],
-                                   [51134.0, 51134.04166667],
-                                   [51134.04166667, 51134.08333333]])
-    elif mip is 'Amon':
-        timestamps = np.array([51149.5, 51179.0, 51208.5])
+    if mip == 'E1hr':
+        offset = 51134  # days since 1850 at 1 january 1990
+        timestamps = offset + np.arange(3) / 24
+        if shifted:
+            timestamps -= 1 / 48
         if bounds is not None:
-            bounds = np.array([[51134.0, 51165.0], [51165.0, 51193.0],
-                               [51193.0, 51224.0]])
+            bounds = [[t - 1 / 48, t + 1 / 48] for t in timestamps]
+    elif mip == 'Amon':
+        timestamps = np.array([51149.5, 51179., 51208.5])
+        if bounds is not None:
+            bounds = np.array([[51134., 51165.], [51165., 51193.],
+                               [51193., 51224.]])
 
     return iris.coords.DimCoord(np.array(timestamps, dtype=float),
                                 standard_name='time',
@@ -159,7 +153,7 @@ def _cmor_aux_height(value):
 
 
 def _cmor_data(mip):
-    if mip is 'fx':
+    if mip == 'fx':
         return np.arange(9).reshape(3, 3)[::-1, :]
     return np.arange(27).reshape(3, 3, 3)[:, ::-1, :]
 
@@ -426,6 +420,7 @@ def rls_cmor_e1hr():
                           attributes={'positive': 'down'})
     return iris.cube.CubeList([cube])
 
+
 VARIABLES = [
     pytest.param(a, b, c, d, id=c + '_' + d) for (a, b, c, d) in [
         (pr_era5_monthly(), pr_cmor_amon(), 'pr', 'Amon'),
@@ -444,12 +439,16 @@ def rls_cmor_e1hr():
 @pytest.mark.parametrize('era5_cubes, cmor_cubes, var, mip', VARIABLES)
 def test_cmorization(era5_cubes, cmor_cubes, var, mip):
     """Verify that cmorization results in the expected target cube."""
-
     fixed_cubes = fix_metadata(era5_cubes, var, 'native6', 'era5', mip)
-    print('era5_cube:', era5_cubes[0].xml())
-    print('cmor_cube:', cmor_cubes[0].xml())
-    print('fixed_cube:', fixed_cubes[0].xml())
-    assert fixed_cubes[0].xml() == cmor_cubes[0].xml()
-    assert (fixed_cubes[0].data == cmor_cubes[0].data).all()
-    # assert fixed_cubes[0].coords() == cmor_cubes[0].coords()
-    # assert fixed_cubes[0] == cmor_cubes[0]
+    assert len(fixed_cubes) == 1
+    fixed_cube = fixed_cubes[0]
+    cmor_cube = cmor_cubes[0]
+    if fixed_cube.coords('time'):
+        for cube in [fixed_cube, cmor_cube]:
+            coord = cube.coord('time')
+            coord.points = np.round(coord.points, decimals=7)
+            if coord.bounds is not None:
+                coord.bounds = np.round(coord.bounds, decimals=7)
+    print('cmor_cube:', cmor_cube.xml())
+    print('fixed_cube:', fixed_cube.xml())
+    assert fixed_cube == cmor_cube
diff --git a/tests/integration/preprocessor/_regrid/test_get_file_levels.py b/tests/integration/preprocessor/_regrid/test_get_file_levels.py
index 2c92a1d2cb..7267611c6f 100644
--- a/tests/integration/preprocessor/_regrid/test_get_file_levels.py
+++ b/tests/integration/preprocessor/_regrid/test_get_file_levels.py
@@ -38,6 +38,6 @@ def tearDown(self):
     def test_get_coord(self):
         self.assertListEqual(
             _regrid.get_reference_levels(
-                self.path, 'project', 'dataset', 'short_name', 'output_dir'),
+                self.path, 'CMIP6', 'dataset', 'short_name', 'output_dir'),
             [0., 1]
         )
diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py
index f2dce12c90..9bab1b3362 100644
--- a/tests/integration/test_recipe.py
+++ b/tests/integration/test_recipe.py
@@ -354,7 +354,6 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user):
             'project': 'CMIP5',
             'dataset': 'CanESM2',
             'short_name': 'chl',
-            'cmor_table': 'CMIP5',
             'mip': 'Oyr',
             'frequency': 'yr',
         },
@@ -362,7 +361,6 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user):
             'project': 'CMIP5',
             'dataset': 'CanESM2',
             'short_name': 'chl',
-            'cmor_table': 'CMIP5',
             'mip': 'Oyr',
             'frequency': 'yr',
         },
@@ -441,7 +439,6 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user):
             'project': 'CMIP5',
             'dataset': 'CanESM2',
             'short_name': 'sftlf',
-            'cmor_table': 'CMIP5',
             'mip': 'fx',
             'frequency': 'fx',
         },
@@ -449,7 +446,6 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user):
             'project': 'CMIP5',
             'dataset': 'CanESM2',
             'short_name': 'sftlf',
-            'cmor_table': 'CMIP5',
             'mip': 'fx',
             'frequency': 'fx',
         },
diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py
index 1a06cdabd8..25e3398758 100644
--- a/tests/unit/cmor/test_fix.py
+++ b/tests/unit/cmor/test_fix.py
@@ -141,15 +141,21 @@ def test_cmor_checker_called(self):
             with mock.patch(
                     'esmvalcore.cmor.fix._get_cmor_checker',
                     return_value=checker) as get_mock:
-                fix_metadata([self.cube], 'short_name', 'project', 'model',
-                             'cmor_table', 'mip', 'frequency')
+                fix_metadata(
+                    cubes=[self.cube],
+                    short_name='short_name',
+                    project='project',
+                    dataset='dataset',
+                    mip='mip',
+                    frequency='frequency',
+                )
                 get_mock.assert_called_once_with(
                     automatic_fixes=True,
                     fail_on_error=False,
                     frequency='frequency',
                     mip='mip',
                     short_name='short_name',
-                    table='cmor_table')
+                    table='project')
                 checker.assert_called_once_with(self.cube)
                 checker.return_value.check_metadata.assert_called_once_with()
 
@@ -193,13 +199,14 @@ def test_cmor_checker_called(self):
                     'esmvalcore.cmor.fix._get_cmor_checker',
                     return_value=checker) as get_mock:
                 fix_data(self.cube, 'short_name', 'project', 'model',
-                         'cmor_table', 'mip', 'frequency')
+                         'mip', 'frequency')
                 get_mock.assert_called_once_with(
+                    table='project',
                     automatic_fixes=True,
                     fail_on_error=False,
                     frequency='frequency',
                     mip='mip',
                     short_name='short_name',
-                    table='cmor_table')
+                    )
                 checker.assert_called_once_with(self.cube)
                 checker.return_value.check_data.assert_called_once_with()
diff --git a/tests/unit/data_finder/test_get_start_end_year.py b/tests/unit/data_finder/test_get_start_end_year.py
index f2b36ff81d..136e4e62dc 100644
--- a/tests/unit/data_finder/test_get_start_end_year.py
+++ b/tests/unit/data_finder/test_get_start_end_year.py
@@ -1,9 +1,8 @@
 """Unit tests for :func:`esmvalcore._data_finder.regrid._stock_cube`"""
 
+import tempfile
 import iris
 import pytest
-import tempfile
-import os
 
 from esmvalcore._data_finder import get_start_end_year
 
@@ -32,8 +31,8 @@ def test_get_start_end_year(case):
     assert case_end == end
 
 
-def test_read_file_if_no_date_present():
-    """Test raises if no date is present"""
+def test_read_time_from_cube():
+    """Try to get time from cube if no date in filename"""
     temp_file = tempfile.NamedTemporaryFile(suffix='.nc')
     cube = iris.cube.Cube([0, 0], var_name='var')
     time = iris.coords.DimCoord([0, 366],
@@ -48,5 +47,5 @@ def test_read_file_if_no_date_present():
 
 def test_fails_if_no_date_present():
     """Test raises if no date is present"""
-    with pytest.raises(ValueError):
+    with pytest.raises((ValueError, OSError)):
         get_start_end_year('var_whatever')

From 71a01cd54ef8a084b7ef33bef5fa89a9b9402f2f Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Fri, 31 Jan 2020 11:07:45 +0100
Subject: [PATCH 34/40] Update tests and flake8 fixes

---
 esmvalcore/_data_finder.py                    |  2 +-
 esmvalcore/cmor/_fixes/native6/era5.py        |  4 +-
 esmvalcore/cmor/table.py                      |  2 +-
 .../cmor/_fixes/native6/test_era5.py          | 75 ++++++++++++++++---
 4 files changed, 71 insertions(+), 12 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index 16892fc2a3..ef61b95451 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -5,11 +5,11 @@
 # Mattia Righi (DLR, Germany - mattia.righi@dlr.de)
 
 import fnmatch
-import iris
 import logging
 import os
 import re
 import glob
+import iris
 
 from ._config import get_project_config
 
diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index ab3ccdd4ea..ab2939d061 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -126,6 +126,7 @@ def fix_metadata(self, cubes):
                 time.guess_bounds()
         return cubes
 
+
 class Evspsbl(Hydrological, Accumulated):
     """Fixes for evspsbl."""
 
@@ -208,7 +209,8 @@ def _fix_coordinates(self, cube):
             coord.var_name = coord_def.out_name
             coord.long_name = coord_def.long_name
             coord.points = coord.core_points().astype('float64')
-            if (coord.bounds is not None and len(coord.points) > 1 and coord_def.must_have_bounds == "yes"):
+            if (coord.bounds is None and len(coord.points) > 1
+                    and coord_def.must_have_bounds == "yes"):
                 coord.guess_bounds()
 
         self._fix_monthly_time_coord(cube)
diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py
index 4af52ad09f..2cedc13a3e 100644
--- a/esmvalcore/cmor/table.py
+++ b/esmvalcore/cmor/table.py
@@ -11,8 +11,8 @@
 import json
 import logging
 import os
-import yaml
 from pathlib import Path
+import yaml
 
 logger = logging.getLogger(__name__)
 
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index 3e1f35eac2..0f32bd3f31 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -183,7 +183,13 @@ def pr_cmor_amon():
                           units=Unit(vardef.units),
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
-                                               (_cmor_longitude(), 2)])
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
     return iris.cube.CubeList([cube])
 
 
@@ -212,7 +218,13 @@ def pr_cmor_e1hr():
                           units=Unit(vardef.units),
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
-                                               (_cmor_longitude(), 2)])
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
     return iris.cube.CubeList([cube])
 
 
@@ -239,7 +251,13 @@ def orog_cmor_fx():
                           standard_name=vardef.standard_name,
                           units=Unit(vardef.units),
                           dim_coords_and_dims=[(_cmor_latitude(), 0),
-                                               (_cmor_longitude(), 1)])
+                                               (_cmor_longitude(), 1)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
     return iris.cube.CubeList([cube])
 
 
@@ -268,7 +286,13 @@ def uas_cmor_e1hr():
                           units=Unit(vardef.units),
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
-                                               (_cmor_longitude(), 2)])
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
     cube.add_aux_coord(_cmor_aux_height(10.))
     return iris.cube.CubeList([cube])
 
@@ -298,7 +322,13 @@ def tas_cmor_e1hr():
                           units=Unit(vardef.units),
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
-                                               (_cmor_longitude(), 2)])
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
     cube.add_aux_coord(_cmor_aux_height(2.))
     return iris.cube.CubeList([cube])
 
@@ -329,7 +359,14 @@ def rsds_cmor_e1hr():
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
                                                (_cmor_longitude(), 2)],
-                          attributes={'positive': 'down'})
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020',
+                              'positive':
+                              'down'
+                          })
     return iris.cube.CubeList([cube])
 
 
@@ -358,7 +395,13 @@ def prsn_cmor_e1hr():
                           units=Unit(vardef.units),
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
-                                               (_cmor_longitude(), 2)])
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
     return iris.cube.CubeList([cube])
 
 
@@ -387,7 +430,13 @@ def mrro_cmor_e1hr():
                           units=Unit(vardef.units),
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
-                                               (_cmor_longitude(), 2)])
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
     return iris.cube.CubeList([cube])
 
 
@@ -417,7 +466,14 @@ def rls_cmor_e1hr():
                           dim_coords_and_dims=[(time, 0),
                                                (_cmor_latitude(), 1),
                                                (_cmor_longitude(), 2)],
-                          attributes={'positive': 'down'})
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020',
+                              'positive':
+                              'down'
+                          })
     return iris.cube.CubeList([cube])
 
 
@@ -451,4 +507,5 @@ def test_cmorization(era5_cubes, cmor_cubes, var, mip):
                 coord.bounds = np.round(coord.bounds, decimals=7)
     print('cmor_cube:', cmor_cube.xml())
     print('fixed_cube:', fixed_cube.xml())
+    assert fixed_cube.xml() == cmor_cube.xml()
     assert fixed_cube == cmor_cube

From 65a38138b83198b9582a8252668f4019f51e6523 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Sat, 1 Feb 2020 11:39:09 +0100
Subject: [PATCH 35/40] Refactor and fix bug

---
 esmvalcore/cmor/_fixes/native6/era5.py        | 265 ++++++++++--------
 .../cmor/_fixes/native6/test_era5.py          |  30 +-
 2 files changed, 164 insertions(+), 131 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index ab2939d061..c06d68760d 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -11,175 +11,208 @@
 logger = logging.getLogger(__name__)
 
 
-class FixEra5(Fix):
-    """Fixes for ERA5 variables."""
-    @staticmethod
-    def _frequency(cube):
-        """Determine time frequency of input cube."""
-        try:
-            time = cube.coord(axis='T')
-        except iris.exceptions.CoordinateNotFoundError:
-            return 'fx'
-
-        time.convert_units('days since 1850-1-1 00:00:00.0')
-        if len(time.points) == 1:
-            if cube.long_name != 'Geopotential':
-                raise ValueError('Unable to infer frequency of cube '
-                                 f'with length 1 time dimension: {cube}')
-            return 'fx'
-        interval = time.points[1] - time.points[0]
-        if interval - 1 / 24 < 1e-4:
-            return 'hourly'
-        return 'monthly'
-
-
-class Accumulated(FixEra5):
-    """Fixes for accumulated variables."""
-    def _fix_frequency(self, cube):
-        if self._frequency(cube) == 'monthly':
-            cube.units = cube.units * 'd-1'
-        elif self._frequency(cube) == 'hourly':
-            cube.units = cube.units * 'h-1'
-        return cube
+def get_frequency(cube):
+    """Determine time frequency of input cube."""
+    try:
+        time = cube.coord(axis='T')
+    except iris.exceptions.CoordinateNotFoundError:
+        return 'fx'
 
-    def _fix_hourly_time_coordinate(self, cube):
-        if self._frequency(cube) == 'hourly':
-            time = cube.coord(axis='T')
-            time.points = time.points - 1 / 48
-            time.guess_bounds()
-        return cube
+    time.convert_units('days since 1850-1-1 00:00:00.0')
+    if len(time.points) == 1:
+        if cube.long_name != 'Geopotential':
+            raise ValueError('Unable to infer frequency of cube '
+                             f'with length 1 time dimension: {cube}')
+        return 'fx'
+
+    interval = time.points[1] - time.points[0]
+    if interval - 1 / 24 < 1e-4:
+        return 'hourly'
+
+    return 'monthly'
+
+
+def fix_invalid_units(cube):
+    """Convert m of water equivalent to m."""
+    cube.units = 'm'
+    return cube
+
+
+def fix_hourly_time_coordinate(cube):
+    """Shift aggregated variables 30 minutes back in time."""
+    if get_frequency(cube) == 'hourly':
+        time = cube.coord(axis='T')
+        time.points = time.points - 1 / 48
+        time.guess_bounds()
+    return cube
+
+
+def fix_accumulated_units(cube):
+    """Convert accumulations to fluxes."""
+    if get_frequency(cube) == 'monthly':
+        cube.units = cube.units * 'd-1'
+    elif get_frequency(cube) == 'hourly':
+        cube.units = cube.units * 'h-1'
+    return cube
 
+
+def multiply_with_density(cube, density=1000):
+    """Convert precipitatin from m to kg/m2."""
+    cube.data = cube.core_data() * density
+    cube.units *= 'kg m**-3'
+    return cube
+
+
+def remove_time_coordinate(cube):
+    """Remove time coordinate for invariant parameters."""
+    cube = cube[0]
+    cube.remove_coord('time')
+    return cube
+
+
+def divide_by_gravity(cube):
+    """Convert geopotential to height."""
+    cube.units = cube.units / 'm s-2'
+    cube.data = cube.core_data() / 9.80665
+    return cube
+
+
+class Evspsbl(Fix):
+    """Fixes for evspsbl."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
-        super().fix_metadata(cubes)
         for cube in cubes:
-            self._fix_hourly_time_coordinate(cube)
-            self._fix_frequency(cube)
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            multiply_with_density(cube)
+
         return cubes
 
 
-class Hydrological(FixEra5):
-    """Fixes for hydrological variables."""
-    @staticmethod
-    def _fix_units(cube):
-        cube.units = 'kg m-2 s-1'
-        cube.data = cube.core_data() * 1000.
-        return cube
-
+class Evspsblpot(Fix):
+    """Fixes for evspsblpot."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
-        super().fix_metadata(cubes)
         for cube in cubes:
-            self._fix_units(cube)
-        return cubes
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            multiply_with_density(cube)
 
+        return cubes
 
-class Radiation(FixEra5):
-    """Fixes for accumulated radiation variables."""
-    @staticmethod
-    def _fix_direction(cube):
-        cube.attributes['positive'] = 'down'
 
+class Mrro(Fix):
+    """Fixes for mrro."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
-        super().fix_metadata(cubes)
         for cube in cubes:
-            self._fix_direction(cube)
-        return cubes
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            multiply_with_density(cube)
 
+        return cubes
 
-class Fx(FixEra5):
-    """Fixes for time invariant variables."""
-    @staticmethod
-    def _remove_time_coordinate(cube):
-        cube = cube[0]
-        cube.remove_coord('time')
-        return cube
 
+class Pr(Fix):
+    """Fixes for pr."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
-        squeezed_cubes = []
         for cube in cubes:
-            cube = self._remove_time_coordinate(cube)
-            squeezed_cubes.append(cube)
-        return iris.cube.CubeList(squeezed_cubes)
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            multiply_with_density(cube)
 
+        return cubes
 
-class Tasmin(FixEra5):
-    """Fixes for tasmin."""
+
+class Prsn(Fix):
+    """Fixes for prsn."""
     def fix_metadata(self, cubes):
+        """Fix metadata."""
         for cube in cubes:
-            if self._frequency(cube) == 'hourly':
-                time = cube.coord(axis='T')
-                time.points = time.points - 1 / 48
-                time.guess_bounds()
+            fix_invalid_units(cube)
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            multiply_with_density(cube)
+
         return cubes
 
 
-class Tasmax(FixEra5):
-    """Fixes for tasmax."""
+class Orog(Fix):
+    """Fixes for orography."""
     def fix_metadata(self, cubes):
+        """Fix metadata."""
+        fixed_cubes = []
         for cube in cubes:
-            if self._frequency(cube) == 'hourly':
-                time = cube.coord(axis='T')
-                time.points = time.points - 1 / 48
-                time.guess_bounds()
-        return cubes
-
+            cube = remove_time_coordinate(cube)
+            divide_by_gravity(cube)
+            fixed_cubes.append(cube)
+        return iris.cube.CubeList(fixed_cubes)
 
-class Evspsbl(Hydrological, Accumulated):
-    """Fixes for evspsbl."""
 
+class Rls(Fix):
+    """Fixes for Rls."""
+    def fix_metadata(self, cubes):
+        """Fix metadata."""
+        for cube in cubes:
+            cube.attributes['positive'] = 'down'
 
-class Mrro(Hydrological, Accumulated):
-    """Fixes for evspsbl."""
+        return cubes
 
 
-class Prsn(Hydrological, Accumulated):
-    """Fixes for evspsbl."""
+class Rsds(Fix):
+    """Fixes for Rsds."""
+    def fix_metadata(self, cubes):
+        """Fix metadata."""
+        for cube in cubes:
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            cube.attributes['positive'] = 'down'
 
+        return cubes
 
-class Pr(Hydrological, Accumulated):
-    """Fixes for evspsbl."""
 
+class Rsdt(Fix):
+    """Fixes for Rsdt."""
+    def fix_metadata(self, cubes):
+        """Fix metadata."""
+        for cube in cubes:
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            cube.attributes['positive'] = 'down'
 
-class Evspsblpot(Hydrological, Accumulated):
-    """Fixes for evspsbl."""
+        return cubes
 
 
-class Rss(Radiation, Accumulated):
+class Rss(Fix):
     """Fixes for Rss."""
+    def fix_metadata(self, cubes):
+        """Fix metadata."""
+        for cube in cubes:
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            cube.attributes['positive'] = 'down'
 
-
-class Rsds(Radiation, Accumulated):
-    """Fixes for Rsds."""
-
-
-class Rsdt(Radiation, Accumulated):
-    """Fixes for Rsdt."""
+        return cubes
 
 
-class Rls(Radiation):
-    """Fixes for Rls."""
+class Tasmax(Fix):
+    """Fixes for tasmax."""
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            fix_hourly_time_coordinate(cube)
+        return cubes
 
 
-class Orog(Fx):
-    """Fixes for orography."""
-    @staticmethod
-    def _divide_by_gravity(cube):
-        cube.units = cube.units / 'm s-2'
-        cube.data = cube.core_data() / 9.80665
-        return cube
-
+class Tasmin(Fix):
+    """Fixes for tasmin."""
     def fix_metadata(self, cubes):
-        """Fix metadata."""
-        cubes = super().fix_metadata(cubes)
         for cube in cubes:
-            self._divide_by_gravity(cube)
+            fix_hourly_time_coordinate(cube)
         return cubes
 
 
-class AllVars(FixEra5):
+class AllVars(Fix):
     """Fixes for all variables."""
     def _fix_coordinates(self, cube):
         """Fix coordinates."""
@@ -219,7 +252,7 @@ def _fix_coordinates(self, cube):
 
     def _fix_monthly_time_coord(self, cube):
         """Set the monthly time coordinates to the middle of the month."""
-        if self._frequency(cube) == 'monthly':
+        if get_frequency(cube) == 'monthly':
             coord = cube.coord(axis='T')
             end = []
             for cell in coord.cells():
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index 0f32bd3f31..1e18b3c9b1 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -4,7 +4,7 @@
 import pytest
 from cf_units import Unit
 
-from esmvalcore.cmor._fixes.native6.era5 import AllVars, Evspsbl, FixEra5
+from esmvalcore.cmor._fixes.native6.era5 import AllVars, Evspsbl, get_frequency
 from esmvalcore.cmor.fix import Fix, fix_metadata
 from esmvalcore.cmor.table import CMOR_TABLES
 
@@ -16,35 +16,35 @@ def test_get_evspsbl_fix():
 
 
 def test_get_frequency_hourly():
-    fix = FixEra5(None)
+    """Test cubes with hourly frequency."""
     time = iris.coords.DimCoord([0, 1, 2],
                                 standard_name='time',
                                 units=Unit('hours since 1900-01-01'))
     cube = iris.cube.Cube([1, 6, 3],
                           var_name='random_var',
                           dim_coords_and_dims=[(time, 0)])
-    assert fix._frequency(cube) == 'hourly'
+    assert get_frequency(cube) == 'hourly'
     cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0')
-    assert fix._frequency(cube) == 'hourly'
+    assert get_frequency(cube) == 'hourly'
 
 
 def test_get_frequency_monthly():
-    fix = FixEra5(None)
+    """Test cubes with monthly frequency."""
     time = iris.coords.DimCoord([0, 31, 59],
                                 standard_name='time',
                                 units=Unit('hours since 1900-01-01'))
     cube = iris.cube.Cube([1, 6, 3],
                           var_name='random_var',
                           dim_coords_and_dims=[(time, 0)])
-    assert fix._frequency(cube) == 'monthly'
+    assert get_frequency(cube) == 'monthly'
     cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0')
-    assert fix._frequency(cube) == 'monthly'
+    assert get_frequency(cube) == 'monthly'
 
 
 def test_get_frequency_fx():
-    fix = FixEra5(None)
+    """Test cubes with time invariant frequency."""
     cube = iris.cube.Cube(1., long_name='Cube without time coordinate')
-    assert fix._frequency(cube) == 'fx'
+    assert get_frequency(cube) == 'fx'
     time = iris.coords.DimCoord(0,
                                 standard_name='time',
                                 units=Unit('hours since 1900-01-01'))
@@ -52,10 +52,10 @@ def test_get_frequency_fx():
                           var_name='cube_with_length_1_time_coord',
                           long_name='Geopotential',
                           dim_coords_and_dims=[(time, 0)])
-    assert fix._frequency(cube) == 'fx'
+    assert get_frequency(cube) == 'fx'
     cube.long_name = 'Not geopotential'
     with pytest.raises(ValueError):
-        fix._frequency(cube)
+        get_frequency(cube)
 
 
 def _era5_latitude():
@@ -175,7 +175,7 @@ def pr_cmor_amon():
     cmor_table = CMOR_TABLES['native6']
     vardef = cmor_table.get_variable('Amon', 'pr')
     time = _cmor_time('Amon', bounds=True)
-    data = _cmor_data('Amon') * 1000.
+    data = _cmor_data('Amon') * 1000. / 3600. / 24.
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
@@ -210,7 +210,7 @@ def pr_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
     vardef = cmor_table.get_variable('E1hr', 'pr')
     time = _cmor_time('E1hr', bounds=True, shifted=True)
-    data = _cmor_data('E1hr') * 1000.
+    data = _cmor_data('E1hr') * 1000. / 3600.
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
@@ -387,7 +387,7 @@ def prsn_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
     vardef = cmor_table.get_variable('E1hr', 'prsn')
     time = _cmor_time('E1hr', shifted=True, bounds=True)
-    data = _cmor_data('E1hr') * 1000
+    data = _cmor_data('E1hr') * 1000 / 3600.
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
@@ -422,7 +422,7 @@ def mrro_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
     vardef = cmor_table.get_variable('E1hr', 'mrro')
     time = _cmor_time('E1hr', shifted=True, bounds=True)
-    data = _cmor_data('E1hr') * 1000
+    data = _cmor_data('E1hr') * 1000 / 3600.
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,

From b6a9d4f9f3f53d5f52e4f2bfd857607afe4f66bb Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Sat, 1 Feb 2020 11:57:51 +0100
Subject: [PATCH 36/40] fix codacy issue

---
 esmvalcore/cmor/_fixes/native6/era5.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index c06d68760d..0492e7d18b 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -250,7 +250,8 @@ def _fix_coordinates(self, cube):
 
         return cube
 
-    def _fix_monthly_time_coord(self, cube):
+    @staticmethod
+    def _fix_monthly_time_coord(cube):
         """Set the monthly time coordinates to the middle of the month."""
         if get_frequency(cube) == 'monthly':
             coord = cube.coord(axis='T')

From daa6ba2d36f84b4b5586ad3d23948406b8d98e7d Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 6 Feb 2020 14:20:46 +0100
Subject: [PATCH 37/40] Fixes and add more tests

---
 esmvalcore/cmor/_fixes/native6/era5.py        |  62 ++-
 esmvalcore/preprocessor/_io.py                |   7 +
 .../cmor/_fixes/native6/test_era5.py          | 489 +++++++++++++++---
 3 files changed, 484 insertions(+), 74 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index 0492e7d18b..f4ab39569a 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -32,12 +32,6 @@ def get_frequency(cube):
     return 'monthly'
 
 
-def fix_invalid_units(cube):
-    """Convert m of water equivalent to m."""
-    cube.units = 'm'
-    return cube
-
-
 def fix_hourly_time_coordinate(cube):
     """Shift aggregated variables 30 minutes back in time."""
     if get_frequency(cube) == 'hourly':
@@ -77,11 +71,24 @@ def divide_by_gravity(cube):
     return cube
 
 
+class Clt(Fix):
+    """Fixes for clt."""
+    def fix_metadata(self, cubes):
+        for cube in cubes:
+            # Invalid input cube units (ignored on load) were '0-1'
+            cube.units = '%'
+            cube.data = cube.core_data()*100.
+
+        return cubes
+
+
 class Evspsbl(Fix):
     """Fixes for evspsbl."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
         for cube in cubes:
+            # Set input cube units for invalid units were ignored on load
+            cube.units = 'm'
             fix_hourly_time_coordinate(cube)
             fix_accumulated_units(cube)
             multiply_with_density(cube)
@@ -94,6 +101,8 @@ class Evspsblpot(Fix):
     def fix_metadata(self, cubes):
         """Fix metadata."""
         for cube in cubes:
+            # Set input cube units for invalid units were ignored on load
+            cube.units = 'm'
             fix_hourly_time_coordinate(cube)
             fix_accumulated_units(cube)
             multiply_with_density(cube)
@@ -112,6 +121,16 @@ def fix_metadata(self, cubes):
 
         return cubes
 
+class Orog(Fix):
+    """Fixes for orography."""
+    def fix_metadata(self, cubes):
+        """Fix metadata."""
+        fixed_cubes = []
+        for cube in cubes:
+            cube = remove_time_coordinate(cube)
+            divide_by_gravity(cube)
+            fixed_cubes.append(cube)
+        return iris.cube.CubeList(fixed_cubes)
 
 class Pr(Fix):
     """Fixes for pr."""
@@ -130,7 +149,8 @@ class Prsn(Fix):
     def fix_metadata(self, cubes):
         """Fix metadata."""
         for cube in cubes:
-            fix_invalid_units(cube)
+            # Set input cube units for invalid units were ignored on load
+            cube.units = 'm'
             fix_hourly_time_coordinate(cube)
             fix_accumulated_units(cube)
             multiply_with_density(cube)
@@ -138,23 +158,34 @@ def fix_metadata(self, cubes):
         return cubes
 
 
-class Orog(Fix):
-    """Fixes for orography."""
+class Ptype(Fix):
+    """Fixes for ptype."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
-        fixed_cubes = []
         for cube in cubes:
-            cube = remove_time_coordinate(cube)
-            divide_by_gravity(cube)
-            fixed_cubes.append(cube)
-        return iris.cube.CubeList(fixed_cubes)
+            cube.units = 1
+
+        return cubes
 
 
+class Rlds(Fix):
+    """Fixes for Rlds."""
+    def fix_metadata(self, cubes):
+        """Fix metadata."""
+        for cube in cubes:
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
+            cube.attributes['positive'] = 'down'
+
+        return cubes
+
 class Rls(Fix):
     """Fixes for Rls."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
         for cube in cubes:
+            fix_hourly_time_coordinate(cube)
+            fix_accumulated_units(cube)
             cube.attributes['positive'] = 'down'
 
         return cubes
@@ -277,7 +308,8 @@ def fix_metadata(self, cubes):
         fixed_cubes = iris.cube.CubeList()
         for cube in cubes:
             cube.var_name = self.vardef.short_name
-            cube.standard_name = self.vardef.standard_name
+            if self.vardef.standard_name:
+                cube.standard_name = self.vardef.standard_name
             cube.long_name = self.vardef.long_name
 
             cube = self._fix_coordinates(cube)
diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py
index 22dc6b1c58..3d393e2488 100644
--- a/esmvalcore/preprocessor/_io.py
+++ b/esmvalcore/preprocessor/_io.py
@@ -61,6 +61,13 @@ def load(file, callback=None):
             category=UserWarning,
             module='iris',
         )
+        filterwarnings(
+            'ignore',
+            message="Ignoring netCDF variable '.*' invalid units '.*'",
+            category=UserWarning,
+            module='iris',
+        )
+
         raw_cubes = iris.load_raw(file, callback=callback)
     if not raw_cubes:
         raise Exception('Can not load cubes from {0}'.format(file))
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index 1e18b3c9b1..f22583d565 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -158,6 +158,179 @@ def _cmor_data(mip):
     return np.arange(27).reshape(3, 3, 3)[:, ::-1, :]
 
 
+def clt_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='cloud cover fraction',
+        var_name='cloud_cover',
+        units='unknown',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def clt_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'clt')
+    time = _cmor_time('E1hr', bounds=True)
+    data = _cmor_data('E1hr') * 100
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
+    return iris.cube.CubeList([cube])
+
+
+def evspsbl_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='total evapotranspiration',
+        var_name='e',
+        units='unknown',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def evspsbl_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'evspsbl')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') * 1000 / 3600.
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
+    return iris.cube.CubeList([cube])
+
+
+def evspsblpot_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='potential evapotranspiration',
+        var_name='epot',
+        units='unknown',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def evspsblpot_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'evspsblpot')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') * 1000 / 3600.
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
+    return iris.cube.CubeList([cube])
+
+
+def mrro_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='runoff',
+        var_name='runoff',
+        units='m',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def mrro_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'mrro')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') * 1000 / 3600.
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
+    return iris.cube.CubeList([cube])
+
+
+def orog_era5_hourly():
+    time = _era5_time('invariant')
+    cube = iris.cube.Cube(
+        _era5_data('invariant'),
+        long_name='geopotential height',
+        var_name='zg',
+        units='m**2 s**-2',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def orog_cmor_fx():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('fx', 'orog')
+    data = _cmor_data('fx') / 9.80665
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(_cmor_latitude(), 0),
+                                               (_cmor_longitude(), 1)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
+    return iris.cube.CubeList([cube])
+
+
 def pr_era5_monthly():
     time = _era5_time('monthly')
     cube = iris.cube.Cube(
@@ -228,30 +401,32 @@ def pr_cmor_e1hr():
     return iris.cube.CubeList([cube])
 
 
-def orog_era5_hourly():
-    time = _era5_time('invariant')
+def prsn_era5_hourly():
+    time = _era5_time('hourly')
     cube = iris.cube.Cube(
-        _era5_data('invariant'),
-        long_name='geopotential height',
-        var_name='zg',
-        units='m**2 s**-2',
+        _era5_data('hourly'),
+        long_name='snow',
+        var_name='snow',
+        units='unknown',
         dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
                              (_era5_longitude(), 2)],
     )
     return iris.cube.CubeList([cube])
 
 
-def orog_cmor_fx():
+def prsn_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
-    vardef = cmor_table.get_variable('fx', 'orog')
-    data = _cmor_data('fx') / 9.80665
+    vardef = cmor_table.get_variable('E1hr', 'prsn')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') * 1000 / 3600.
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
                           standard_name=vardef.standard_name,
                           units=Unit(vardef.units),
-                          dim_coords_and_dims=[(_cmor_latitude(), 0),
-                                               (_cmor_longitude(), 1)],
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
                           attributes={
                               'comment':
                               'Contains modified '
@@ -261,24 +436,60 @@ def orog_cmor_fx():
     return iris.cube.CubeList([cube])
 
 
-def uas_era5_hourly():
+def ptype_era5_hourly():
     time = _era5_time('hourly')
     cube = iris.cube.Cube(
         _era5_data('hourly'),
-        long_name='10m_u_component_of_wind',
-        var_name='u10',
-        units='m s-1',
+        long_name='snow',
+        var_name='snow',
+        units='unknown',
         dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
                              (_era5_longitude(), 2)],
     )
     return iris.cube.CubeList([cube])
 
 
-def uas_cmor_e1hr():
+def ptype_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
-    vardef = cmor_table.get_variable('E1hr', 'uas')
-    time = _cmor_time('E1hr')
+    vardef = cmor_table.get_variable('E1hr', 'ptype')
+    time = _cmor_time('E1hr', shifted=False, bounds=True)
     data = _cmor_data('E1hr')
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          units=1,
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
+    cube.coord('latitude').long_name = 'latitude'
+    cube.coord('longitude').long_name = 'longitude'
+    return iris.cube.CubeList([cube])
+
+
+def rlds_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='surface thermal radiation downwards',
+        var_name='ssrd',
+        units='J m**-2',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def rlds_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'rlds')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') / 3600
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
@@ -291,30 +502,31 @@ def uas_cmor_e1hr():
                               'comment':
                               'Contains modified '
                               'Copernicus Climate Change Service '
-                              'Information 2020'
+                              'Information 2020',
+                              'positive':
+                              'down'
                           })
-    cube.add_aux_coord(_cmor_aux_height(10.))
     return iris.cube.CubeList([cube])
 
 
-def tas_era5_hourly():
+def rls_era5_hourly():
     time = _era5_time('hourly')
     cube = iris.cube.Cube(
         _era5_data('hourly'),
-        long_name='2m_temperature',
-        var_name='t2m',
-        units='degC',
+        long_name='runoff',
+        var_name='runoff',
+        units='J m-2',
         dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
                              (_era5_longitude(), 2)],
     )
     return iris.cube.CubeList([cube])
 
 
-def tas_cmor_e1hr():
+def rls_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
-    vardef = cmor_table.get_variable('E1hr', 'tas')
-    time = _cmor_time('E1hr')
-    data = _cmor_data('E1hr') + 273.15
+    vardef = cmor_table.get_variable('E1hr', 'rls')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') / 3600
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
@@ -327,9 +539,10 @@ def tas_cmor_e1hr():
                               'comment':
                               'Contains modified '
                               'Copernicus Climate Change Service '
-                              'Information 2020'
+                              'Information 2020',
+                              'positive':
+                              'down'
                           })
-    cube.add_aux_coord(_cmor_aux_height(2.))
     return iris.cube.CubeList([cube])
 
 
@@ -370,24 +583,98 @@ def rsds_cmor_e1hr():
     return iris.cube.CubeList([cube])
 
 
-def prsn_era5_hourly():
+def rsdt_era5_hourly():
     time = _era5_time('hourly')
     cube = iris.cube.Cube(
         _era5_data('hourly'),
-        long_name='snow',
-        var_name='snow',
-        units='unknown',
+        long_name='thermal_radiation_downwards',
+        var_name='strd',
+        units='J m**-2',
         dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
                              (_era5_longitude(), 2)],
     )
     return iris.cube.CubeList([cube])
 
 
-def prsn_cmor_e1hr():
+def rsdt_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
-    vardef = cmor_table.get_variable('E1hr', 'prsn')
+    vardef = cmor_table.get_variable('E1hr', 'rsdt')
     time = _cmor_time('E1hr', shifted=True, bounds=True)
-    data = _cmor_data('E1hr') * 1000 / 3600.
+    data = _cmor_data('E1hr') / 3600
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020',
+                              'positive':
+                              'down'
+                          })
+    return iris.cube.CubeList([cube])
+
+
+def rss_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='net_solar_radiation',
+        var_name='ssr',
+        units='J m**-2',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def rss_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'rss')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
+    data = _cmor_data('E1hr') / 3600
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020',
+                              'positive':
+                              'down'
+                          })
+    return iris.cube.CubeList([cube])
+
+
+def tas_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='2m_temperature',
+        var_name='t2m',
+        units='K',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def tas_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'tas')
+    time = _cmor_time('E1hr')
+    data = _cmor_data('E1hr')
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
@@ -402,27 +689,64 @@ def prsn_cmor_e1hr():
                               'Copernicus Climate Change Service '
                               'Information 2020'
                           })
+    cube.add_aux_coord(_cmor_aux_height(2.))
     return iris.cube.CubeList([cube])
 
 
-def mrro_era5_hourly():
+def tas_era5_monthly():
+    time = _era5_time('monthly')
+    cube = iris.cube.Cube(
+        _era5_data('monthly'),
+        long_name='2m_temperature',
+        var_name='t2m',
+        units='K',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def tas_cmor_amon():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('Amon', 'tas')
+    time = _cmor_time('Amon',  bounds=True)
+    data = _cmor_data('Amon')
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
+                          })
+    cube.add_aux_coord(_cmor_aux_height(2.))
+    return iris.cube.CubeList([cube])
+
+
+def tasmax_era5_hourly():
     time = _era5_time('hourly')
     cube = iris.cube.Cube(
         _era5_data('hourly'),
-        long_name='runoff',
-        var_name='runoff',
-        units='m',
+        long_name='maximum 2m temperature',
+        var_name='mx2t',
+        units='K',
         dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
                              (_era5_longitude(), 2)],
     )
     return iris.cube.CubeList([cube])
 
 
-def mrro_cmor_e1hr():
+def tasmax_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
-    vardef = cmor_table.get_variable('E1hr', 'mrro')
+    vardef = cmor_table.get_variable('E1hr', 'tasmax')
     time = _cmor_time('E1hr', shifted=True, bounds=True)
-    data = _cmor_data('E1hr') * 1000 / 3600.
+    data = _cmor_data('E1hr')
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,
@@ -437,26 +761,27 @@ def mrro_cmor_e1hr():
                               'Copernicus Climate Change Service '
                               'Information 2020'
                           })
+    cube.add_aux_coord(_cmor_aux_height(2.))
     return iris.cube.CubeList([cube])
 
 
-def rls_era5_hourly():
+def tasmin_era5_hourly():
     time = _era5_time('hourly')
     cube = iris.cube.Cube(
         _era5_data('hourly'),
-        long_name='runoff',
-        var_name='runoff',
-        units='W m**-2',
+        long_name='minimum 2m temperature',
+        var_name='mn2t',
+        units='K',
         dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
                              (_era5_longitude(), 2)],
     )
     return iris.cube.CubeList([cube])
 
 
-def rls_cmor_e1hr():
+def tasmin_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
-    vardef = cmor_table.get_variable('E1hr', 'rls')
-    time = _cmor_time('E1hr', shifted=False, bounds=True)
+    vardef = cmor_table.get_variable('E1hr', 'tasmin')
+    time = _cmor_time('E1hr', shifted=True, bounds=True)
     data = _cmor_data('E1hr')
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
@@ -470,24 +795,70 @@ def rls_cmor_e1hr():
                               'comment':
                               'Contains modified '
                               'Copernicus Climate Change Service '
-                              'Information 2020',
-                              'positive':
-                              'down'
+                              'Information 2020'
+                          })
+    cube.add_aux_coord(_cmor_aux_height(2.))
+    return iris.cube.CubeList([cube])
+
+
+def uas_era5_hourly():
+    time = _era5_time('hourly')
+    cube = iris.cube.Cube(
+        _era5_data('hourly'),
+        long_name='10m_u_component_of_wind',
+        var_name='u10',
+        units='m s-1',
+        dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
+                             (_era5_longitude(), 2)],
+    )
+    return iris.cube.CubeList([cube])
+
+
+def uas_cmor_e1hr():
+    cmor_table = CMOR_TABLES['native6']
+    vardef = cmor_table.get_variable('E1hr', 'uas')
+    time = _cmor_time('E1hr')
+    data = _cmor_data('E1hr')
+    cube = iris.cube.Cube(data.astype('float32'),
+                          long_name=vardef.long_name,
+                          var_name=vardef.short_name,
+                          standard_name=vardef.standard_name,
+                          units=Unit(vardef.units),
+                          dim_coords_and_dims=[(time, 0),
+                                               (_cmor_latitude(), 1),
+                                               (_cmor_longitude(), 2)],
+                          attributes={
+                              'comment':
+                              'Contains modified '
+                              'Copernicus Climate Change Service '
+                              'Information 2020'
                           })
+    cube.add_aux_coord(_cmor_aux_height(10.))
     return iris.cube.CubeList([cube])
 
 
 VARIABLES = [
     pytest.param(a, b, c, d, id=c + '_' + d) for (a, b, c, d) in [
+        (clt_era5_hourly(), clt_cmor_e1hr(), 'clt', 'E1hr'),
+        (evspsbl_era5_hourly(), evspsbl_cmor_e1hr(), 'evspsbl', 'E1hr'),
+        (evspsblpot_era5_hourly(), evspsblpot_cmor_e1hr(), 'evspsblpot',
+         'E1hr'),
+        (mrro_era5_hourly(), mrro_cmor_e1hr(), 'mrro', 'E1hr'),
+        (orog_era5_hourly(), orog_cmor_fx(), 'orog', 'fx'),
         (pr_era5_monthly(), pr_cmor_amon(), 'pr', 'Amon'),
         (pr_era5_hourly(), pr_cmor_e1hr(), 'pr', 'E1hr'),
-        (orog_era5_hourly(), orog_cmor_fx(), 'orog', 'fx'),
-        (uas_era5_hourly(), uas_cmor_e1hr(), 'uas', 'E1hr'),
-        (tas_era5_hourly(), tas_cmor_e1hr(), 'tas', 'E1hr'),
-        (rsds_era5_hourly(), rsds_cmor_e1hr(), 'rsds', 'E1hr'),
         (prsn_era5_hourly(), prsn_cmor_e1hr(), 'prsn', 'E1hr'),
-        (mrro_era5_hourly(), mrro_cmor_e1hr(), 'mrro', 'E1hr'),
+        (ptype_era5_hourly(), ptype_cmor_e1hr(), 'ptype', 'E1hr'),
+        (rlds_era5_hourly(), rlds_cmor_e1hr(), 'rlds', 'E1hr'),
         (rls_era5_hourly(), rls_cmor_e1hr(), 'rls', 'E1hr'),
+        (rsds_era5_hourly(), rsds_cmor_e1hr(), 'rsds', 'E1hr'),
+        (rsdt_era5_hourly(), rsdt_cmor_e1hr(), 'rsdt', 'E1hr'),
+        (rss_era5_hourly(), rss_cmor_e1hr(), 'rss', 'E1hr'),
+        (tas_era5_hourly(), tas_cmor_e1hr(), 'tas', 'E1hr'),
+        (tas_era5_monthly(), tas_cmor_amon(), 'tas', 'Amon'),
+        (tasmax_era5_hourly(), tasmax_cmor_e1hr(), 'tasmax', 'E1hr'),
+        (tasmin_era5_hourly(), tasmin_cmor_e1hr(), 'tasmin', 'E1hr'),
+        (uas_era5_hourly(), uas_cmor_e1hr(), 'uas', 'E1hr'),
     ]
 ]
 

From 62ee4f9bc8dcadeffc7f5f9fd1fd28fe69e05860 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Thu, 6 Feb 2020 16:10:12 +0100
Subject: [PATCH 38/40] Fix Rls units

---
 esmvalcore/cmor/_fixes/native6/era5.py             | 4 +++-
 tests/integration/cmor/_fixes/native6/test_era5.py | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index f4ab39569a..e1a4c2351f 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -121,6 +121,7 @@ def fix_metadata(self, cubes):
 
         return cubes
 
+
 class Orog(Fix):
     """Fixes for orography."""
     def fix_metadata(self, cubes):
@@ -132,6 +133,7 @@ def fix_metadata(self, cubes):
             fixed_cubes.append(cube)
         return iris.cube.CubeList(fixed_cubes)
 
+
 class Pr(Fix):
     """Fixes for pr."""
     def fix_metadata(self, cubes):
@@ -179,13 +181,13 @@ def fix_metadata(self, cubes):
 
         return cubes
 
+
 class Rls(Fix):
     """Fixes for Rls."""
     def fix_metadata(self, cubes):
         """Fix metadata."""
         for cube in cubes:
             fix_hourly_time_coordinate(cube)
-            fix_accumulated_units(cube)
             cube.attributes['positive'] = 'down'
 
         return cubes
diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py
index f22583d565..1134a4733e 100644
--- a/tests/integration/cmor/_fixes/native6/test_era5.py
+++ b/tests/integration/cmor/_fixes/native6/test_era5.py
@@ -515,7 +515,7 @@ def rls_era5_hourly():
         _era5_data('hourly'),
         long_name='runoff',
         var_name='runoff',
-        units='J m-2',
+        units='W m-2',
         dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1),
                              (_era5_longitude(), 2)],
     )
@@ -526,7 +526,7 @@ def rls_cmor_e1hr():
     cmor_table = CMOR_TABLES['native6']
     vardef = cmor_table.get_variable('E1hr', 'rls')
     time = _cmor_time('E1hr', shifted=True, bounds=True)
-    data = _cmor_data('E1hr') / 3600
+    data = _cmor_data('E1hr')
     cube = iris.cube.Cube(data.astype('float32'),
                           long_name=vardef.long_name,
                           var_name=vardef.short_name,

From b036f469a603a813dac2c1f8f4248fd0a25b5564 Mon Sep 17 00:00:00 2001
From: Peter Kalverla <peter.kalverla@gmx.com>
Date: Fri, 7 Feb 2020 14:09:50 +0100
Subject: [PATCH 39/40] Add Valeriu's suggestions

---
 esmvalcore/_data_finder.py             | 14 +++++++-------
 esmvalcore/cmor/_fixes/native6/era5.py |  1 -
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index da9db089c1..7a27dee4f6 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -5,10 +5,12 @@
 # Mattia Righi (DLR, Germany - mattia.righi@dlr.de)
 
 import fnmatch
+import glob
 import logging
 import os
 import re
-import glob
+from pathlib import Path
+
 import iris
 
 from ._config import get_project_config
@@ -32,25 +34,23 @@ def find_files(dirnames, filenames):
 
 def get_start_end_year(filename):
     """Get the start and end year from a file name."""
-    stripped_name = os.path.splitext(filename)[0]
-    stripped_name = stripped_name.split(os.sep)[-1]
+    stem = Path(filename).stem
     start_year = end_year = None
 
     # First check for a block of two potential dates separated by _ or -
-    daterange = re.findall(r'([0-9]{4,12}[-_][0-9]{4,12})', stripped_name)
+    daterange = re.findall(r'([0-9]{4,12}[-_][0-9]{4,12})', stem)
     if daterange:
         start_date, end_date = re.findall(r'([0-9]{4,12})', daterange[0])
         start_year = start_date[:4]
         end_year = end_date[:4]
     else:
         # Check for single dates in the filename
-        dates = re.findall(r'([0-9]{4,12})', stripped_name)
+        dates = re.findall(r'([0-9]{4,12})', stem)
         if len(dates) == 1:
             start_year = end_year = dates[0][:4]
         elif len(dates) > 1:
             # Check for dates at start or end of filename
-            outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$',
-                                    stripped_name)
+            outerdates = re.findall(r'^[0-9]{4,12}|[0-9]{4,12}$', stem)
             if len(outerdates) == 1:
                 start_year = end_year = outerdates[0][:4]
 
diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py
index e1a4c2351f..1f569db4dc 100644
--- a/esmvalcore/cmor/_fixes/native6/era5.py
+++ b/esmvalcore/cmor/_fixes/native6/era5.py
@@ -37,7 +37,6 @@ def fix_hourly_time_coordinate(cube):
     if get_frequency(cube) == 'hourly':
         time = cube.coord(axis='T')
         time.points = time.points - 1 / 48
-        time.guess_bounds()
     return cube
 
 

From b60c294c3411243b1c03923077a89bb09b88f458 Mon Sep 17 00:00:00 2001
From: Bouwe Andela <b.andela@esciencecenter.nl>
Date: Fri, 7 Feb 2020 22:40:44 +0100
Subject: [PATCH 40/40] Clean up

---
 esmvalcore/_recipe.py                         |  81 +++---
 esmvalcore/cmor/fix.py                        |  57 ++---
 esmvalcore/preprocessor/_regrid.py            |  20 +-
 .../_regrid/test_get_file_levels.py           |  24 +-
 tests/integration/test_recipe.py              |  22 +-
 tests/unit/cmor/test_fix.py                   | 242 +++++++++++-------
 6 files changed, 260 insertions(+), 186 deletions(-)

diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py
index 7554fb20d5..2bcf759634 100644
--- a/esmvalcore/_recipe.py
+++ b/esmvalcore/_recipe.py
@@ -88,40 +88,26 @@ def _get_value(key, datasets):
     return value
 
 
-def _update_from_others(variable, keys, datasets):
-    """Get values for keys by copying from the other datasets."""
-    for key in keys:
-        if key not in variable:
-            value = _get_value(key, datasets)
-            if value is not None:
-                variable[key] = value
-
-
 def _add_cmor_info(variable, override=False):
     """Add information from CMOR tables to variable."""
     logger.debug("If not present: adding keys from CMOR table to %s", variable)
-
-    if 'cmor_table' not in variable or 'mip' not in variable:
-        logger.debug("Skipping because cmor_table or mip not specified")
-        return
-
-    if variable['cmor_table'] not in CMOR_TABLES:
-        logger.warning("Unknown CMOR table %s", variable['cmor_table'])
-
-    derive = variable.get('derive', False)
     # Copy the following keys from CMOR table
     cmor_keys = [
         'standard_name', 'long_name', 'units', 'modeling_realm', 'frequency'
     ]
-    cmor_table = variable['cmor_table']
+    project = variable['project']
     mip = variable['mip']
     short_name = variable['short_name']
-    table_entry = CMOR_TABLES[cmor_table].get_variable(mip, short_name, derive)
-
+    derive = variable.get('derive', False)
+    table = CMOR_TABLES.get(project)
+    if table:
+        table_entry = table.get_variable(mip, short_name, derive)
+    else:
+        table_entry = None
     if table_entry is None:
         raise RecipeError(
-            "Unable to load CMOR table '{}' for variable '{}' with mip '{}'".
-            format(cmor_table, short_name, mip))
+            f"Unable to load CMOR table (project) '{project}' for variable "
+            f"'{short_name}' with mip '{mip}'")
 
     for key in cmor_keys:
         if key not in variable or override:
@@ -180,12 +166,17 @@ def _update_target_levels(variable, variables, settings, config_user):
             del settings['extract_levels']
         else:
             variable_data = _get_dataset_info(dataset, variables)
-            filename = \
-                _dataset_to_file(variable_data, config_user)
+            filename = _dataset_to_file(variable_data, config_user)
             settings['extract_levels']['levels'] = get_reference_levels(
-                filename, variable_data['project'], dataset,
-                variable_data['short_name'],
-                os.path.splitext(variable_data['filename'])[0] + '_fixed')
+                filename=filename,
+                project=variable_data['project'],
+                dataset=dataset,
+                short_name=variable_data['short_name'],
+                mip=variable_data['mip'],
+                frequency=variable_data['frequency'],
+                fix_dir=os.path.splitext(
+                    variable_data['filename'])[0] + '_fixed',
+            )
 
 
 def _update_target_grid(variable, variables, settings, config_user):
@@ -300,16 +291,16 @@ def _get_default_settings(variable, config_user, derive=False):
         'project': variable['project'],
         'dataset': variable['dataset'],
         'short_name': variable['short_name'],
+        'mip': variable['mip'],
     }
     # File fixes
     fix_dir = os.path.splitext(variable['filename'])[0] + '_fixed'
     settings['fix_file'] = dict(fix)
     settings['fix_file']['output_dir'] = fix_dir
     # Cube fixes
-    fix['mip'] = variable['mip']
     fix['frequency'] = variable['frequency']
-    settings['fix_data'] = dict(fix)
     settings['fix_metadata'] = dict(fix)
+    settings['fix_data'] = dict(fix)
 
     # Configure time extraction
     if 'start_year' in variable and 'end_year' in variable \
@@ -332,21 +323,19 @@ def _get_default_settings(variable, config_user, derive=False):
         }
 
     # Configure CMOR metadata check
-    if variable.get('cmor_table'):
-        settings['cmor_check_metadata'] = {
-            'cmor_table': variable['cmor_table'],
-            'mip': variable['mip'],
-            'short_name': variable['short_name'],
-            'frequency': variable['frequency'],
-        }
+    settings['cmor_check_metadata'] = {
+        'cmor_table': variable['project'],
+        'mip': variable['mip'],
+        'short_name': variable['short_name'],
+        'frequency': variable['frequency'],
+    }
     # Configure final CMOR data check
-    if variable.get('cmor_table'):
-        settings['cmor_check_data'] = {
-            'cmor_table': variable['cmor_table'],
-            'mip': variable['mip'],
-            'short_name': variable['short_name'],
-            'frequency': variable['frequency'],
-        }
+    settings['cmor_check_data'] = {
+        'cmor_table': variable['project'],
+        'mip': variable['mip'],
+        'short_name': variable['short_name'],
+        'frequency': variable['frequency'],
+    }
 
     # Clean up fixed files
     if not config_user['save_intermediary_cubes']:
@@ -1026,9 +1015,6 @@ def _initialize_variables(self, raw_variable, raw_datasets):
             variable.update(dataset)
 
             variable['recipe_dataset_index'] = index
-            if ('cmor_table' not in variable
-                    and variable.get('project') in CMOR_TABLES):
-                variable['cmor_table'] = variable['project']
             if 'end_year' in variable and 'max_years' in self._cfg:
                 variable['end_year'] = min(
                     variable['end_year'],
@@ -1046,7 +1032,6 @@ def _initialize_variables(self, raw_variable, raw_datasets):
         if 'fx' not in raw_variable.get('mip', ''):
             required_keys.update({'start_year', 'end_year'})
         for variable in variables:
-            _update_from_others(variable, ['cmor_table', 'mip'], datasets)
             if 'institute' not in variable:
                 institute = get_institutes(variable)
                 if institute:
diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py
index 69054ed141..6ddb13549d 100644
--- a/esmvalcore/cmor/fix.py
+++ b/esmvalcore/cmor/fix.py
@@ -18,7 +18,7 @@
 logger = logging.getLogger(__name__)
 
 
-def fix_file(file, short_name, project, dataset, output_dir):
+def fix_file(file, short_name, project, dataset, mip, output_dir):
     """
     Fix files before ESMValTool can load them.
 
@@ -45,7 +45,7 @@ def fix_file(file, short_name, project, dataset, output_dir):
 
     """
     for fix in Fix.get_fixes(
-            project=project, dataset=dataset, mip='', short_name=short_name):
+            project=project, dataset=dataset, mip=mip, short_name=short_name):
         file = fix.fix_file(file, output_dir)
     return file
 
@@ -54,7 +54,7 @@ def fix_metadata(cubes,
                  short_name,
                  project,
                  dataset,
-                 mip='',
+                 mip,
                  frequency=None):
     """
     Fix cube metadata if fixes are required and check it anyway.
@@ -74,8 +74,8 @@ def fix_metadata(cubes,
 
     dataset: str
 
-    mip: str, optional
-        Variable's MIP, if available
+    mip: str
+        Variable's MIP
 
     frequency: str, optional
         Variable's data frequency, if available
@@ -130,15 +130,14 @@ def fix_metadata(cubes,
         else:
             cube = cube_list[0]
 
-        if mip:
-            checker = _get_cmor_checker(
-                frequency=frequency,
-                table=project,
-                mip=mip,
-                short_name=short_name,
-                fail_on_error=False,
-                automatic_fixes=True)
-            cube = checker(cube).check_metadata()
+        checker = _get_cmor_checker(
+            frequency=frequency,
+            table=project,
+            mip=mip,
+            short_name=short_name,
+            fail_on_error=False,
+            automatic_fixes=True)
+        cube = checker(cube).check_metadata()
         cube.attributes.pop('source_file', None)
         fixed_cubes.append(cube)
     return fixed_cubes
@@ -148,7 +147,7 @@ def fix_data(cube,
              short_name,
              project,
              dataset,
-             mip='',
+             mip,
              frequency=None):
     """
     Fix cube data if fixes add present and check it anyway.
@@ -167,15 +166,9 @@ def fix_data(cube,
     short_name: str
         Variable's short name
     project: str
-
     dataset: str
-
-    cmor_table: str, optional
-        CMOR tables to use for the check, if available
-
-    mip: str, optional
-        Variable's MIP, if available
-
+    mip: str
+        Variable's MIP
     frequency: str, optional
         Variable's data frequency, if available
 
@@ -193,13 +186,13 @@ def fix_data(cube,
     for fix in Fix.get_fixes(
             project=project, dataset=dataset, mip=mip, short_name=short_name):
         cube = fix.fix_data(cube)
-    if mip:
-        checker = _get_cmor_checker(
-            frequency=frequency,
-            table=project,
-            mip=mip,
-            short_name=short_name,
-            fail_on_error=False,
-            automatic_fixes=True)
-        cube = checker(cube).check_data()
+
+    checker = _get_cmor_checker(
+        frequency=frequency,
+        table=project,
+        mip=mip,
+        short_name=short_name,
+        fail_on_error=False,
+        automatic_fixes=True)
+    cube = checker(cube).check_data()
     return cube
diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py
index 66f3d0fd7f..2842fad760 100644
--- a/esmvalcore/preprocessor/_regrid.py
+++ b/esmvalcore/preprocessor/_regrid.py
@@ -498,6 +498,8 @@ def get_reference_levels(filename,
                          project,
                          dataset,
                          short_name,
+                         mip,
+                         frequency,
                          fix_dir):
     """Get level definition from a reference dataset.
 
@@ -517,9 +519,23 @@ def get_reference_levels(filename,
         levels or the string is badly formatted.
 
     """
-    filename = fix_file(filename, short_name, project, dataset, fix_dir)
+    filename = fix_file(
+        file=filename,
+        short_name=short_name,
+        project=project,
+        dataset=dataset,
+        mip=mip,
+        output_dir=fix_dir,
+    )
     cubes = load(filename, callback=concatenate_callback)
-    cubes = fix_metadata(cubes, short_name, project, dataset)
+    cubes = fix_metadata(
+        cubes=cubes,
+        short_name=short_name,
+        project=project,
+        dataset=dataset,
+        mip=mip,
+        frequency=frequency,
+    )
     cube = cubes[0]
     try:
         coord = cube.coord(axis='Z')
diff --git a/tests/integration/preprocessor/_regrid/test_get_file_levels.py b/tests/integration/preprocessor/_regrid/test_get_file_levels.py
index 7267611c6f..128a074453 100644
--- a/tests/integration/preprocessor/_regrid/test_get_file_levels.py
+++ b/tests/integration/preprocessor/_regrid/test_get_file_levels.py
@@ -36,8 +36,22 @@ def tearDown(self):
         os.remove(self.path)
 
     def test_get_coord(self):
-        self.assertListEqual(
-            _regrid.get_reference_levels(
-                self.path, 'CMIP6', 'dataset', 'short_name', 'output_dir'),
-            [0., 1]
-        )
+        fix_file = unittest.mock.create_autospec(_regrid.fix_file)
+        fix_file.side_effect = lambda file, **_: file
+        fix_metadata = unittest.mock.create_autospec(_regrid.fix_metadata)
+        fix_metadata.side_effect = lambda cubes, **_: cubes
+        with unittest.mock.patch('esmvalcore.preprocessor._regrid.fix_file',
+                                 fix_file):
+            with unittest.mock.patch(
+                    'esmvalcore.preprocessor._regrid.fix_metadata',
+                    fix_metadata):
+                reference_levels = _regrid.get_reference_levels(
+                    filename=self.path,
+                    project='CMIP6',
+                    dataset='dataset',
+                    short_name='short_name',
+                    mip='mip',
+                    frequency='mon',
+                    fix_dir='output_dir',
+                )
+        self.assertListEqual(reference_levels, [0., 1])
diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py
index 2c3c815224..864c091c09 100644
--- a/tests/integration/test_recipe.py
+++ b/tests/integration/test_recipe.py
@@ -19,7 +19,6 @@
 from .test_provenance import check_provenance
 
 MANDATORY_DATASET_KEYS = (
-    'cmor_table',
     'dataset',
     'diagnostic',
     'end_year',
@@ -348,6 +347,7 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user):
             'project': 'CMIP5',
             'dataset': 'CanESM2',
             'short_name': 'chl',
+            'mip': 'Oyr',
             'output_dir': fix_dir,
         },
         'fix_data': {
@@ -433,6 +433,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user):
             'project': 'CMIP5',
             'dataset': 'CanESM2',
             'short_name': 'sftlf',
+            'mip': 'fx',
             'output_dir': fix_dir,
         },
         'fix_data': {
@@ -667,7 +668,6 @@ def test_simple_cordex_recipe(tmp_path, patched_datafinder,
             'tas_MOHC-HadGEM3-RA_evaluation_r1i1p1_v1_mon_1991-1993.nc')
     reference = {
         'alias': 'MOHC-HadGEM3-RA',
-        'cmor_table': 'CORDEX',
         'dataset': 'MOHC-HadGEM3-RA',
         'diagnostic': 'test',
         'domain': 'AFR-44',
@@ -764,11 +764,13 @@ def test_reference_dataset(tmp_path, patched_datafinder, config_user,
 
     fix_dir = os.path.splitext(reference.filename)[0] + '_fixed'
     get_reference_levels.assert_called_once_with(
-        reference.files[0],
-        'CMIP5',
-        'MPI-ESM-LR',
-        'ta',
-        fix_dir,
+        filename=reference.files[0],
+        project='CMIP5',
+        dataset='MPI-ESM-LR',
+        short_name='ta',
+        mip='Amon',
+        frequency='mon',
+        fix_dir=fix_dir,
     )
 
     assert 'regrid' not in reference.settings
@@ -2036,9 +2038,11 @@ def test_wrong_project(tmp_path, patched_datafinder, config_user):
                   - {dataset: CanESM2}
             scripts: null
         """)
-    with pytest.raises(ValueError) as wrong_proj:
+    with pytest.raises(RecipeError) as wrong_proj:
         get_recipe(tmp_path, content, config_user)
-        assert wrong_proj == "Project CMIP7 not in config-developer"
+    assert str(wrong_proj.value) == (
+        "Unable to load CMOR table (project) 'CMIP7' for variable 'tos' "
+        "with mip 'Omon'")
 
 
 def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user):
diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py
index ebe887f9cb..814e810dd4 100644
--- a/tests/unit/cmor/test_fix.py
+++ b/tests/unit/cmor/test_fix.py
@@ -1,47 +1,56 @@
 """Unit tests for the variable_info module."""
 
-import unittest
-from unittest import mock
+from unittest import TestCase
+from unittest.mock import Mock, patch
 
 from esmvalcore.cmor.fix import Fix, fix_data, fix_file, fix_metadata
 
 
-class TestFixFile(unittest.TestCase):
+class TestFixFile(TestCase):
     """Fix file tests."""
-
     def setUp(self):
         """Prepare for testing."""
         self.filename = 'filename'
-        self.mock_fix = mock.Mock()
+        self.mock_fix = Mock()
         self.mock_fix.fix_file.return_value = 'new_filename'
 
     def test_fix(self):
         """Check that the returned fix is applied."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes',
-                return_value=[self.mock_fix]):
-            file_returned = fix_file('filename', 'short_name', 'project',
-                                     'model', 'output_dir')
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[self.mock_fix]):
+            file_returned = fix_file(
+                file='filename',
+                short_name='short_name',
+                project='project',
+                dataset='model',
+                mip='mip',
+                output_dir='output_dir',
+            )
             self.assertNotEqual(file_returned, self.filename)
             self.assertEqual(file_returned, 'new_filename')
 
     def test_nofix(self):
         """Check that the same file is returned if no fix is available."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]):
-            file_returned = fix_file('filename', 'short_name', 'project',
-                                     'model', 'output_dir')
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[]):
+            file_returned = fix_file(
+                file='filename',
+                short_name='short_name',
+                project='project',
+                dataset='model',
+                mip='mip',
+                output_dir='output_dir',
+            )
             self.assertEqual(file_returned, self.filename)
 
 
-class TestGetCube(unittest.TestCase):
+class TestGetCube(TestCase):
     """Test get cube by var_name method."""
-
     def setUp(self):
         """Prepare for testing."""
-        self.cube_1 = mock.Mock()
+        self.cube_1 = Mock()
         self.cube_1.var_name = 'cube1'
-        self.cube_2 = mock.Mock()
+        self.cube_2 = Mock()
         self.cube_2.var_name = 'cube2'
         self.cubes = [self.cube_1, self.cube_2]
         self.fix = Fix(None)
@@ -67,83 +76,113 @@ def test_get_default(self):
         self.assertIs(self.cube_1, self.fix.get_cube_from_list(self.cubes))
 
 
-class TestFixMetadata(unittest.TestCase):
+class TestFixMetadata(TestCase):
     """Fix metadata tests."""
-
     def setUp(self):
         """Prepare for testing."""
 
         self.cube = self._create_mock_cube()
+        self.intermediate_cube = self._create_mock_cube()
         self.fixed_cube = self._create_mock_cube()
-        self.mock_fix = mock.Mock()
-        self.mock_fix.fix_metadata.return_value = [self.fixed_cube]
+        self.mock_fix = Mock()
+        self.mock_fix.fix_metadata.return_value = [self.intermediate_cube]
+        self.checker = Mock()
+        self.check_metadata = self.checker.return_value.check_metadata
 
     @staticmethod
     def _create_mock_cube(var_name='short_name'):
-        cube = mock.Mock()
+        cube = Mock()
         cube.var_name = var_name
         cube.attributes = {'source_file': 'source_file'}
         return cube
 
     def test_fix(self):
         """Check that the returned fix is applied."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes',
-                return_value=[self.mock_fix]):
-            cube_returned = fix_metadata([self.cube], 'short_name', 'project',
-                                         'model')[0]
-            self.assertTrue(cube_returned is not self.cube)
-            self.assertTrue(cube_returned is self.fixed_cube)
+        self.check_metadata.side_effect = lambda: self.fixed_cube
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[self.mock_fix]):
+            with patch('esmvalcore.cmor.fix._get_cmor_checker',
+                       return_value=self.checker):
+                cube_returned = fix_metadata(
+                    cubes=[self.cube],
+                    short_name='short_name',
+                    project='project',
+                    dataset='model',
+                    mip='mip',
+                )[0]
+                self.checker.assert_called_once_with(self.intermediate_cube)
+                self.check_metadata.assert_called_once_with()
+                assert cube_returned is not self.cube
+                assert cube_returned is not self.intermediate_cube
+                assert cube_returned is self.fixed_cube
 
     def test_nofix(self):
         """Check that the same cube is returned if no fix is available."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]):
-            cube_returned = fix_metadata([self.cube], 'short_name', 'project',
-                                         'model')[0]
-            self.assertTrue(cube_returned is self.cube)
-            self.assertTrue(cube_returned is not self.fixed_cube)
+        self.check_metadata.side_effect = lambda: self.cube
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[]):
+            with patch('esmvalcore.cmor.fix._get_cmor_checker',
+                       return_value=self.checker):
+                cube_returned = fix_metadata(
+                    cubes=[self.cube],
+                    short_name='short_name',
+                    project='project',
+                    dataset='model',
+                    mip='mip',
+                )[0]
+                self.checker.assert_called_once_with(self.cube)
+                self.check_metadata.assert_called_once_with()
+                assert cube_returned is self.cube
+                assert cube_returned is not self.intermediate_cube
+                assert cube_returned is not self.fixed_cube
 
     def test_select_var(self):
         """Check that the same cube is returned if no fix is available."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]):
-            cube_returned = fix_metadata(
-                [self.cube, self._create_mock_cube('extra')],
-                'short_name',
-                'project',
-                'model'
-            )[0]
-            self.assertTrue(cube_returned is self.cube)
+        self.check_metadata.side_effect = lambda: self.cube
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[]):
+            with patch('esmvalcore.cmor.fix._get_cmor_checker',
+                       return_value=self.checker):
+                cube_returned = fix_metadata(
+                    cubes=[self.cube,
+                           self._create_mock_cube('extra')],
+                    short_name='short_name',
+                    project='CMIP6',
+                    dataset='model',
+                    mip='mip',
+                )[0]
+                self.checker.assert_called_once_with(self.cube)
+                self.check_metadata.assert_called_once_with()
+                assert cube_returned is self.cube
 
     def test_select_var_failed_if_bad_var_name(self):
         """Check that the same cube is returned if no fix is available."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]):
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[]):
             with self.assertRaises(ValueError):
                 fix_metadata(
-                    [
+                    cubes=[
                         self._create_mock_cube('not_me'),
                         self._create_mock_cube('me_neither')
                     ],
-                    'short_name',
-                    'project',
-                    'model'
+                    short_name='short_name',
+                    project='CMIP6',
+                    dataset='model',
+                    mip='mip',
                 )
 
     def test_cmor_checker_called(self):
         """Check that the cmor check is done."""
-        checker = mock.Mock()
-        checker.return_value = mock.Mock()
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]):
-            with mock.patch(
-                    'esmvalcore.cmor.fix._get_cmor_checker',
-                    return_value=checker) as get_mock:
+        checker = Mock()
+        checker.return_value = Mock()
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[]):
+            with patch('esmvalcore.cmor.fix._get_cmor_checker',
+                       return_value=checker) as get_mock:
                 fix_metadata(
                     cubes=[self.cube],
                     short_name='short_name',
-                    project='project',
+                    project='CMIP6',
                     dataset='dataset',
                     mip='mip',
                     frequency='frequency',
@@ -154,58 +193,81 @@ def test_cmor_checker_called(self):
                     frequency='frequency',
                     mip='mip',
                     short_name='short_name',
-                    table='project')
+                    table='CMIP6',
+                )
                 checker.assert_called_once_with(self.cube)
                 checker.return_value.check_metadata.assert_called_once_with()
 
 
-class TestFixData(unittest.TestCase):
+class TestFixData(TestCase):
     """Fix data tests."""
-
     def setUp(self):
         """Prepare for testing."""
-        self.cube = mock.Mock()
-        self.fixed_cube = mock.Mock()
-        self.mock_fix = mock.Mock()
-        self.mock_fix.fix_data.return_value = self.fixed_cube
+        self.cube = Mock()
+        self.intermediate_cube = Mock()
+        self.fixed_cube = Mock()
+        self.mock_fix = Mock()
+        self.mock_fix.fix_data.return_value = self.intermediate_cube
+        self.checker = Mock()
+        self.check_data = self.checker.return_value.check_data
 
     def test_fix(self):
         """Check that the returned fix is applied."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes',
-                return_value=[self.mock_fix]):
-            cube_returned = fix_data(self.cube, 'short_name', 'project',
-                                     'model')
-            self.assertTrue(cube_returned is not self.cube)
-            self.assertTrue(cube_returned is self.fixed_cube)
+        self.check_data.side_effect = lambda: self.fixed_cube
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[self.mock_fix]):
+            with patch('esmvalcore.cmor.fix._get_cmor_checker',
+                       return_value=self.checker):
+                cube_returned = fix_data(
+                    self.cube,
+                    short_name='short_name',
+                    project='project',
+                    dataset='model',
+                    mip='mip',
+                )
+                self.checker.assert_called_once_with(self.intermediate_cube)
+                self.check_data.assert_called_once_with()
+                assert cube_returned is not self.cube
+                assert cube_returned is not self.intermediate_cube
+                assert cube_returned is self.fixed_cube
 
     def test_nofix(self):
         """Check that the same cube is returned if no fix is available."""
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]):
-            cube_returned = fix_data(self.cube, 'short_name', 'project',
-                                     'model')
-            self.assertTrue(cube_returned is self.cube)
-            self.assertTrue(cube_returned is not self.fixed_cube)
+        self.check_data.side_effect = lambda: self.cube
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[]):
+            with patch('esmvalcore.cmor.fix._get_cmor_checker',
+                       return_value=self.checker):
+                cube_returned = fix_data(
+                    self.cube,
+                    short_name='short_name',
+                    project='CMIP6',
+                    dataset='model',
+                    mip='mip',
+                )
+                self.checker.assert_called_once_with(self.cube)
+                self.check_data.assert_called_once_with()
+                assert cube_returned is self.cube
+                assert cube_returned is not self.intermediate_cube
+                assert cube_returned is not self.fixed_cube
 
     def test_cmor_checker_called(self):
         """Check that the cmor check is done."""
-        checker = mock.Mock()
-        checker.return_value = mock.Mock()
-        with mock.patch(
-                'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]):
-            with mock.patch(
-                    'esmvalcore.cmor.fix._get_cmor_checker',
-                    return_value=checker) as get_mock:
-                fix_data(self.cube, 'short_name', 'project', 'model',
-                         'mip', 'frequency')
+        checker = Mock()
+        checker.return_value = Mock()
+        with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes',
+                   return_value=[]):
+            with patch('esmvalcore.cmor.fix._get_cmor_checker',
+                       return_value=checker) as get_mock:
+                fix_data(self.cube, 'short_name', 'CMIP6', 'model', 'mip',
+                         'frequency')
                 get_mock.assert_called_once_with(
-                    table='project',
+                    table='CMIP6',
                     automatic_fixes=True,
                     fail_on_error=False,
                     frequency='frequency',
                     mip='mip',
                     short_name='short_name',
-                    )
+                )
                 checker.assert_called_once_with(self.cube)
                 checker.return_value.check_data.assert_called_once_with()