Added test for check of data availability and fixed some checks for _recipe.py

esmvalcore/_recipe_checks.py

@@ -93,6 +93,7 @@ def variable(var, required_keys):
 
 def data_availability(input_files, var, dirnames, filenames):
     """Check if the required input data is available."""
+    var = dict(var)
     if not input_files:
         var.pop('filename', None)
         logger.error("No input files found for variable %s", var)
@@ -160,8 +161,8 @@ def check_for_temporal_preprocs(profile):
         preproc for preproc in profile if preproc in temporal_preprocs]
     if temp_preprocs:
         raise RecipeError(
-            "Time coordinate preprocessor step {} not permitted on fx vars \
-            please remove them from recipe.".format(", ".join(temp_preprocs)))
+            "Time coordinate preprocessor step(s) {} not permitted on fx "
+            "vars, please remove them from recipe".format(temp_preprocs))
 
 
 def extract_shape(settings):
@@ -174,6 +175,7 @@ def extract_shape(settings):
     valid = {
         'method': {'contains', 'representative'},
         'crop': {True, False},
+        'decomposed': {True, False},
     }
     for key in valid:
         value = settings.get(key)

tests/integration/cmor/_fixes/cmip5/test_ec_earth.py

@@ -6,7 +6,7 @@
 from iris.coords import DimCoord
 from iris.cube import Cube, CubeList
 
-from esmvalcore.cmor._fixes.cmip5.ec_earth import Sftlf, Sic, Tas, Areacello
+from esmvalcore.cmor._fixes.cmip5.ec_earth import Areacello, Sftlf, Sic, Tas
 from esmvalcore.cmor.fix import Fix
 
 
@@ -108,7 +108,6 @@ def test_tas_fix_metadata(self):
 
 class TestAreacello(unittest.TestCase):
     """Test areacello fixes."""
-
     def setUp(self):
         """Prepare tests."""
 
@@ -133,19 +132,19 @@ def setUp(self):
                 np.ones((2, 2)),
                 var_name='areacello',
                 long_name='Areas of grid cell',
-            ),
-            latitude,
-            longitude
-            ])
+            ), latitude, longitude
+        ])
 
-        self.fix = Areacello()
+        self.fix = Areacello(None)
 
     def test_get(self):
         """Test fix get"""
         self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'EC-EARTH', 'areacello'), [Areacello()])
+            Fix.get_fixes('CMIP5', 'EC-EARTH', 'Omon', 'areacello'),
+            [Areacello(None)],
+        )
 
-    def test_tas_fix_metadata(self):
+    def test_areacello_fix_metadata(self):
         """Test metadata fix."""
 
         out_cube = self.fix.fix_metadata(self.cubes)

tests/integration/test_recipe.py

@@ -303,12 +303,11 @@ def test_fx_preproc_error(tmp_path, patched_datafinder, config_user):
                   - dataset: MPI-ESM-LR
             scripts: null
         """)
-    rec_err = "Time coordinate preprocessor step extract_season \
-              not permitted on fx vars \
-              please remove them from recipe."
+    msg = ("Time coordinate preprocessor step(s) ['extract_season'] not "
+           "permitted on fx vars, please remove them from recipe")
     with pytest.raises(Exception) as rec_err_exp:
         get_recipe(tmp_path, content, config_user)
-        assert rec_err == rec_err_exp
+    assert str(rec_err_exp.value) == msg
 
 
 def test_default_preprocessor(tmp_path, patched_datafinder, config_user):
@@ -1451,14 +1450,22 @@ def test_extract_shape(tmp_path, patched_datafinder, config_user):
     assert product.settings['extract_shape']['shapefile'] == str(shapefile)
 
 
-@pytest.mark.parametrize('invalid_arg', ['crop', 'shapefile', 'method'])
+@pytest.mark.parametrize('invalid_arg',
+                         ['shapefile', 'method', 'crop', 'decomposed'])
 def test_extract_shape_raises(tmp_path, patched_datafinder, config_user,
                               invalid_arg):
+    # Create shapefile
+    shapefile = config_user['auxiliary_data_dir'] / Path('test.shp')
+    shapefile.parent.mkdir(parents=True, exist_ok=True)
+    shapefile.touch()
+
     content = dedent(f"""
         preprocessors:
           test:
             extract_shape:
-              {invalid_arg}: x
+              crop: true
+              method: contains
+              shapefile: test.shp
 
         diagnostics:
           test:
@@ -1476,10 +1483,16 @@ def test_extract_shape_raises(tmp_path, patched_datafinder, config_user,
                       dataset: GFDL-CM3
             scripts: null
         """)
+
+    # Add invalid argument
+    recipe = yaml.safe_load(content)
+    recipe['preprocessors']['test']['extract_shape'][invalid_arg] = 'x'
+    content = yaml.safe_dump(recipe)
+
     with pytest.raises(RecipeError) as exc:
         get_recipe(tmp_path, content, config_user)
-        assert 'extract_shape' in exc.value
-        assert invalid_arg in exc.value
+    assert 'extract_shape' in str(exc.value)
+    assert invalid_arg in str(exc.value)
 
 
 def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user):
@@ -2042,11 +2055,12 @@ def test_wrong_project(tmp_path, patched_datafinder, config_user):
                   - {dataset: CanESM2}
             scripts: null
         """)
-    with pytest.raises(RecipeError) as wrong_proj:
-        get_recipe(tmp_path, content, config_user)
-    assert str(wrong_proj.value) == (
+    msg = (
         "Unable to load CMOR table (project) 'CMIP7' for variable 'tos' "
         "with mip 'Omon'")
+    with pytest.raises(RecipeError) as wrong_proj:
+        get_recipe(tmp_path, content, config_user)
+    assert str(wrong_proj.value) == msg
 
 
 def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user):
@@ -2076,11 +2090,11 @@ def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user):
                   - {dataset: CanESM5}
             scripts: null
         """)
-    msg = ("Requested fx variable 'wrong_fx_variable' for CMIP6 not "
-           "available in any 'fx'-related CMOR table")
+    msg = ("Requested fx variable 'wrong_fx_variable' not available in any "
+           "'fx'-related CMOR table")
     with pytest.raises(RecipeError) as rec_err_exp:
         get_recipe(tmp_path, content, config_user)
-        assert msg in rec_err_exp
+    assert msg in str(rec_err_exp.value)
 
 
 def test_fx_var_invalid_project(tmp_path, patched_datafinder, config_user):
@@ -2107,7 +2121,9 @@ def test_fx_var_invalid_project(tmp_path, patched_datafinder, config_user):
                   - {dataset: CanESM5}
             scripts: null
         """)
-    msg = 'Project EMAC not supported with fx variables'
+    msg = (
+        "Unable to load CMOR table (project) 'EMAC' for variable 'areacella' "
+        "with mip 'Amon'")
     with pytest.raises(RecipeError) as rec_err_exp:
         get_recipe(tmp_path, content, config_user)
-        assert msg in rec_err_exp
+    assert str(rec_err_exp.value) == msg

tests/integration/test_recipe_checks.py

@@ -0,0 +1,107 @@
+"""Integration tests for :mod:`esmvalcore._recipe_checks`."""
+from unittest import mock
+
+import pytest
+
+import esmvalcore._recipe_checks as check
+
+
+ERR_ALL = 'Looked for files matching%s'
+ERR_D = ('Looked for files in %s, but did not find any file pattern to match '
+         'against')
+ERR_F = ('Looked for files matching %s, but did not find any existing input '
+         'directory')
+ERR_RANGE = 'No input data available for years {} in files {}'
+VAR = {
+    'filename': 'a/c.nc',
+    'frequency': 'mon',
+    'short_name': 'tas',
+    'start_year': 2020,
+    'end_year': 2025,
+}
+FX_VAR = {
+    'filename': 'a/b.nc',
+    'frequency': 'fx',
+    'short_name': 'areacella',
+}
+FILES = [
+    'a/b/c_20200101-20201231',
+    'a/b/c_20210101-20211231',
+    'a/b/c_20220101-20221231',
+    'a/b/c_20230101-20231231',
+    'a/b/c_20240101-20241231',
+    'a/b/c_20250101-20251231',
+]
+
+
+DATA_AVAILABILITY_DATA = [
+    (FILES, dict(VAR), None),
+    (FILES, dict(FX_VAR), None),
+    (FILES[:-1], dict(VAR), ERR_RANGE.format('2025', FILES[:-1])),
+    (FILES[:-2], dict(VAR), ERR_RANGE.format('2024, 2025', FILES[:-2])),
+    ([FILES[1]] + [FILES[3]], dict(VAR), ERR_RANGE.format(
+        '2024, 2025, 2020, 2022', [FILES[1]] + [FILES[3]])),
+
+]
+
+
+@pytest.mark.parametrize('input_files,var,error', DATA_AVAILABILITY_DATA)
+@mock.patch('esmvalcore._recipe_checks.logger', autospec=True)
+def test_data_availability_data(mock_logger, input_files, var, error):
+    """Test check for data when data is present."""
+    saved_var = dict(var)
+    if error is None:
+        check.data_availability(input_files, var, None, None)
+        mock_logger.error.assert_not_called()
+    else:
+        with pytest.raises(check.RecipeError) as rec_err:
+            check.data_availability(input_files, var, None, None)
+        assert str(rec_err.value) == error
+    assert var == saved_var
+
+
+DATA_AVAILABILITY_NO_DATA = [
+    ([], [], None),
+    ([], None, None),
+    (None, [], None),
+    (None, None, None),
+    (['dir1'], [], (ERR_D, ['dir1'])),
+    (['dir1', 'dir2'], [], (ERR_D, ['dir1', 'dir2'])),
+    (['dir1'], None, (ERR_D, ['dir1'])),
+    (['dir1', 'dir2'], None, (ERR_D, ['dir1', 'dir2'])),
+    ([], ['a*.nc'], (ERR_F, ['a*.nc'])),
+    ([], ['a*.nc', 'b*.nc'], (ERR_F, ['a*.nc', 'b*.nc'])),
+    (None, ['a*.nc'], (ERR_F, ['a*.nc'])),
+    (None, ['a*.nc', 'b*.nc'], (ERR_F, ['a*.nc', 'b*.nc'])),
+    (['1'], ['a'], (ERR_ALL, ': 1/a')),
+    (['1'], ['a', 'b'], (ERR_ALL, '\n1/a\n1/b')),
+    (['1', '2'], ['a'], (ERR_ALL, '\n1/a\n2/a')),
+    (['1', '2'], ['a', 'b'], (ERR_ALL, '\n1/a\n1/b\n2/a\n2/b')),
+]
+
+
+@pytest.mark.parametrize('dirnames,filenames,error', DATA_AVAILABILITY_NO_DATA)
+@mock.patch('esmvalcore._recipe_checks.logger', autospec=True)
+def test_data_availability_no_data(mock_logger, dirnames, filenames, error):
+    """Test check for data when no data is present."""
+    var = dict(VAR)
+    var_no_filename = {
+        'frequency': 'mon',
+        'short_name': 'tas',
+        'start_year': 2020,
+        'end_year': 2025,
+    }
+    error_first = ('No input files found for variable %s', var_no_filename)
+    error_last = ("Set 'log_level' to 'debug' to get more information", )
+    with pytest.raises(check.RecipeError) as rec_err:
+        check.data_availability([], var, dirnames, filenames)
+    assert str(rec_err.value) == 'Missing data'
+    if error is None:
+        assert mock_logger.error.call_count == 2
+        errors = [error_first, error_last]
+    else:
+        assert mock_logger.error.call_count == 3
+        errors = [error_first, error, error_last]
+    calls = [mock.call(*e) for e in errors]
+    assert mock_logger.error.call_args_list == calls
+    assert var == VAR