From 4e4c1db5270c65834af3a702681d473b5fe5db50 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 30 Jan 2020 12:23:58 +0100 Subject: [PATCH 001/158] First attempt, not clean --- esmvalcore/_recipe.py | 42 +++++++++++++++++++++++++- esmvalcore/preprocessor/__init__.py | 34 +++++++++++++++------ esmvalcore/preprocessor/_multimodel.py | 37 +++++++++++++++++++---- 3 files changed, 97 insertions(+), 16 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index c94022b5b5..95deffe2e9 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -3,9 +3,10 @@ import logging import os import re -from collections import OrderedDict +from collections import OrderedDict, defaultdict from copy import deepcopy from pprint import pformat +import itertools import yaml from netCDF4 import Dataset @@ -628,6 +629,44 @@ def _update_statistic_settings(products, order, preproc_dir): settings['output_products'][statistic] = statistic_product +def _update_ensemble_settings(products, order, preproc_dir): + step = 'ensemble_statistics' + if not products: + return + + prods = defaultdict(set) + + for p in products: + if step in p.settings: + ensemble = '{}_{}_{}'.format(p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']) + + prods[ensemble].add(p) + for ensemble, ensemble_products in prods.items(): + some_product = next(iter(ensemble_products)) + for statistic in some_product.settings[step]['statistics']: + attributes = _get_statistic_attributes(ensemble_products) + attributes['dataset'] = '{}_Ensemble{}'.format(ensemble, statistic.title()) + attributes['filename'] = get_statistic_output_file( + attributes, preproc_dir) + common_settings = _get_remaining_common_settings(step, order, ensemble_products) + statistic_product = PreprocessorFile(attributes, common_settings) + for product in ensemble_products: + settings = product.settings[step] + if 'output_products' not in settings: + settings['output_products'] = defaultdict(set) + settings['output_products'][statistic].add(statistic_product) + + + + + + + + + + def _update_extract_shape(settings, config_user): if 'extract_shape' in settings: shapefile = settings['extract_shape'].get('shapefile') @@ -727,6 +766,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, products.add(product) _update_statistic_settings(products, order, config_user['preproc_dir']) + _update_ensemble_settings(products, order, config_user['preproc_dir']) for product in products: product.check() diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 2438dfb876..12f06cd922 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -17,7 +17,7 @@ from ._mask import (mask_above_threshold, mask_below_threshold, mask_fillvalues, mask_glaciated, mask_inside_range, mask_landsea, mask_landseaice, mask_outside_range) -from ._multimodel import multi_model_statistics +from ._multimodel import multi_model_statistics, ensemble_statistics from ._reformat import (cmor_check_data, cmor_check_metadata, fix_data, fix_file, fix_metadata) from ._regrid import extract_levels, regrid @@ -78,6 +78,7 @@ # 'average_zone': average_zone, # 'cross_section': cross_section, 'detrend', + 'ensemble_statistics', 'multi_model_statistics', # Grid-point operations 'extract_named_regions', @@ -113,6 +114,7 @@ MULTI_MODEL_FUNCTIONS = { 'multi_model_statistics', 'mask_fillvalues', + 'ensemble_statistics' } @@ -173,11 +175,12 @@ def _check_multi_model_settings(products): elif reference is None: reference = product elif reference.settings[step] != settings: - raise ValueError( - "Unable to combine differing multi-dataset settings for " - "{} and {}, {} and {}".format( - reference.filename, product.filename, - reference.settings[step], settings)) + continue + #raise ValueError( + # "Unable to combine differing multi-dataset settings for " + # "{} and {}, {} and {}".format( + # reference.filename, product.filename, + # reference.settings[step], settings)) def _get_multi_model_settings(products, step): @@ -374,14 +377,27 @@ def _initialize_product_provenance(self): product.initialize_provenance(self.activity) # Hacky way to initialize the multi model products as well. - step = 'multi_model_statistics' + steps = ['multi_model_statistics',] + for step in steps: + input_products = [p for p in self.products if step in p.settings] + if input_products: + statistic_products = input_products[0].settings[step].get( + 'output_products', {}).values() + for product in statistic_products: + product.initialize_provenance(self.activity) + + step = 'ensemble_statistics' input_products = [p for p in self.products if step in p.settings] if input_products: - statistic_products = input_products[0].settings[step].get( - 'output_products', {}).values() + statistic_products = set() + for input_product in input_products: + for prods in input_product.settings[step].get('output_products', {}).values(): + statistic_products.update(prods) for product in statistic_products: product.initialize_provenance(self.activity) + + def _run(self, _): """Run the preprocessor.""" self._initialize_product_provenance() diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 8b75c17e56..58f786fb0d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -20,6 +20,10 @@ import iris import numpy as np +import itertools + +from collections import OrderedDict, defaultdict + logger = logging.getLogger(__name__) @@ -299,6 +303,8 @@ def _assemble_full_data(cubes, statistic): return stats_cube + + def multi_model_statistics(products, span, output_products, statistics): """ Compute multi-model statistics. @@ -371,13 +377,32 @@ def multi_model_statistics(products, span, output_products, statistics): statistic_cube.data, dtype=np.dtype('float32')) # Add to output product and log provenance - statistic_product = output_products[statistic] - statistic_product.cubes = [statistic_cube] - for product in products: - statistic_product.wasderivedfrom(product) - logger.info("Generated %s", statistic_product) - statistic_products.add(statistic_product) + generated_products = output_products[statistic] + for statistic_product in generated_products: + statistic_product.cubes = [statistic_cube] + for product in products: + statistic_product.wasderivedfrom(product) + logger.info("Generated %s", statistic_product) + statistic_products.add(statistic_product) products |= statistic_products return products + +def ensemble_statistics(products, span, output_products, statistics): + + prods = defaultdict(set) + for p in products: + ensemble = '{}_{}_{}'.format(p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']) + + prods[ensemble].add(p) + + for ensemble, ensemble_products in prods.items(): + output_products = next(iter(ensemble_products)).settings['ensemble_statistics'].get('output_products',{}) + prd = multi_model_statistics(ensemble_products, span, output_products, statistics) + products |= prd + + return products + From a27abebf029598d48408659639d490f4cb35c211 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 31 Jan 2020 15:08:20 +0100 Subject: [PATCH 002/158] Compute ensemble stats, pending to test exclude --- esmvalcore/_recipe.py | 66 ++++++---------- esmvalcore/preprocessor/__init__.py | 19 ++--- esmvalcore/preprocessor/_multimodel.py | 102 ++++++++++++------------- 3 files changed, 78 insertions(+), 109 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 95deffe2e9..7c4476b197 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -609,56 +609,41 @@ def _update_statistic_settings(products, order, preproc_dir): # TODO: move this to multi model statistics function? # But how to check, with a dry-run option? step = 'multi_model_statistics' - - products = {p for p in products if step in p.settings} - if not products: - return - - some_product = next(iter(products)) - for statistic in some_product.settings[step]['statistics']: - attributes = _get_statistic_attributes(products) - attributes['dataset'] = 'MultiModel{}'.format(statistic.title()) - attributes['filename'] = get_statistic_output_file( - attributes, preproc_dir) - common_settings = _get_remaining_common_settings(step, order, products) - statistic_product = PreprocessorFile(attributes, common_settings) - for product in products: - settings = product.settings[step] - if 'output_products' not in settings: - settings['output_products'] = {} - settings['output_products'][statistic] = statistic_product - - -def _update_ensemble_settings(products, order, preproc_dir): - step = 'ensemble_statistics' - if not products: - return - prods = defaultdict(set) - + name = defaultdict(dict) for p in products: if step in p.settings: - ensemble = '{}_{}_{}'.format(p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']) - - prods[ensemble].add(p) - for ensemble, ensemble_products in prods.items(): - some_product = next(iter(ensemble_products)) + group = p.settings[step]['group'] + if len(group) < 2: + if 'ensemble' in group: + key = '{}_{}_{}'.format(p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']) + name[key] = '{}_Ensemble'.format(key) + if 'all' in group: + key = 'all' + name[key] = 'MultiModel' + prods[key].add(p) + # PENDING: compute ensemble stats and then stats over ensemble results + + for key, group_products in prods.items(): + some_product = next(iter(group_products)) for statistic in some_product.settings[step]['statistics']: - attributes = _get_statistic_attributes(ensemble_products) - attributes['dataset'] = '{}_Ensemble{}'.format(ensemble, statistic.title()) + attributes = _get_statistic_attributes(group_products) + attributes['dataset'] = '{}{}'.format(name[key], statistic.title()) attributes['filename'] = get_statistic_output_file( attributes, preproc_dir) - common_settings = _get_remaining_common_settings(step, order, ensemble_products) + common_settings = _get_remaining_common_settings(step, order, products) statistic_product = PreprocessorFile(attributes, common_settings) - for product in ensemble_products: + for product in products: settings = product.settings[step] if 'output_products' not in settings: - settings['output_products'] = defaultdict(set) - settings['output_products'][statistic].add(statistic_product) + settings['output_products'] = defaultdict(dict) + settings['output_products'][key] = {} + settings['output_products'][key][statistic] = statistic_product + + - @@ -766,7 +751,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, products.add(product) _update_statistic_settings(products, order, config_user['preproc_dir']) - _update_ensemble_settings(products, order, config_user['preproc_dir']) for product in products: product.check() diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 12f06cd922..36e7d6fe82 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -17,7 +17,7 @@ from ._mask import (mask_above_threshold, mask_below_threshold, mask_fillvalues, mask_glaciated, mask_inside_range, mask_landsea, mask_landseaice, mask_outside_range) -from ._multimodel import multi_model_statistics, ensemble_statistics +from ._multimodel import multi_model_statistics from ._reformat import (cmor_check_data, cmor_check_metadata, fix_data, fix_file, fix_metadata) from ._regrid import extract_levels, regrid @@ -78,7 +78,6 @@ # 'average_zone': average_zone, # 'cross_section': cross_section, 'detrend', - 'ensemble_statistics', 'multi_model_statistics', # Grid-point operations 'extract_named_regions', @@ -114,7 +113,6 @@ MULTI_MODEL_FUNCTIONS = { 'multi_model_statistics', 'mask_fillvalues', - 'ensemble_statistics' } @@ -381,20 +379,13 @@ def _initialize_product_provenance(self): for step in steps: input_products = [p for p in self.products if step in p.settings] if input_products: - statistic_products = input_products[0].settings[step].get( - 'output_products', {}).values() + statistic_products = set() + for input_product in input_products: + for key, prods in input_product.settings[step].get('output_products', {}).items(): + statistic_products.update(prods.values()) for product in statistic_products: product.initialize_provenance(self.activity) - step = 'ensemble_statistics' - input_products = [p for p in self.products if step in p.settings] - if input_products: - statistic_products = set() - for input_product in input_products: - for prods in input_product.settings[step].get('output_products', {}).values(): - statistic_products.update(prods) - for product in statistic_products: - product.initialize_provenance(self.activity) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 58f786fb0d..cfa4fa4bac 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -305,7 +305,7 @@ def _assemble_full_data(cubes, statistic): -def multi_model_statistics(products, span, output_products, statistics): +def multi_model_statistics(products, span, output_products, statistics, group): """ Compute multi-model statistics. @@ -343,66 +343,60 @@ def multi_model_statistics(products, span, output_products, statistics): If span is neither overlap nor full. """ - logger.debug('Multimodel statistics: computing: %s', statistics) - if len(products) < 2: - logger.info("Single dataset in list: will not compute statistics.") - return products - - cubes = [cube for product in products for cube in product.cubes] - # check if we have any time overlap - interval = _get_overlap(cubes) - if interval is None: - logger.info("Time overlap between cubes is none or a single point." - "check datasets: will not compute statistics.") - return products - - if span == 'overlap': - logger.debug("Using common time overlap between " - "datasets to compute statistics.") - elif span == 'full': - logger.debug("Using full time spans to compute statistics.") - else: - raise ValueError( - "Unexpected value for span {}, choose from 'overlap', 'full'" - .format(span)) - + prods = defaultdict(set) statistic_products = set() - for statistic in statistics: - # Compute statistic + for p in products: + if len(group) < 2: + if 'ensemble' in group: + key = '{}_{}_{}'.format(p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']) + if 'all' in group: + key = 'all' + prods[key].add(p) + + for key, ensemble_products in prods.items(): + logger.debug('Multimodel statistics: computing: %s', statistics) + if len(ensemble_products) < 2: + logger.info("Single dataset in list: will not compute statistics.") + return products + + cubes = [cube for product in ensemble_products for cube in product.cubes] + # check if we have any time overlap + interval = _get_overlap(cubes) + if interval is None: + logger.info("Time overlap between cubes is none or a single point." + "check datasets: will not compute statistics.") + return products + if span == 'overlap': - statistic_cube = _assemble_overlap_data(cubes, interval, statistic) + logger.debug("Using common time overlap between " + "datasets to compute statistics.") elif span == 'full': - statistic_cube = _assemble_full_data(cubes, statistic) - statistic_cube.data = np.ma.array( - statistic_cube.data, dtype=np.dtype('float32')) + logger.debug("Using full time spans to compute statistics.") + else: + raise ValueError( + "Unexpected value for span {}, choose from 'overlap', 'full'" + .format(span)) + + for statistic in statistics: + # Compute statistic + if span == 'overlap': + statistic_cube = _assemble_overlap_data(cubes, interval, statistic) + elif span == 'full': + statistic_cube = _assemble_full_data(cubes, statistic) + statistic_cube.data = np.ma.array( + statistic_cube.data, dtype=np.dtype('float32')) # Add to output product and log provenance - generated_products = output_products[statistic] - for statistic_product in generated_products: - statistic_product.cubes = [statistic_cube] - for product in products: - statistic_product.wasderivedfrom(product) - logger.info("Generated %s", statistic_product) - statistic_products.add(statistic_product) + generated_product = output_products[key][statistic] + generated_product.cubes = [statistic_cube] + for product in ensemble_products: + generated_product.wasderivedfrom(product) + logger.info("Generated %s", generated_product) + statistic_products.add(generated_product) products |= statistic_products return products -def ensemble_statistics(products, span, output_products, statistics): - - prods = defaultdict(set) - for p in products: - ensemble = '{}_{}_{}'.format(p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']) - - prods[ensemble].add(p) - - for ensemble, ensemble_products in prods.items(): - output_products = next(iter(ensemble_products)).settings['ensemble_statistics'].get('output_products',{}) - prd = multi_model_statistics(ensemble_products, span, output_products, statistics) - products |= prd - - return products - From b405759ad77c438b20ac12d8c6757affbbc29d00 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 3 Feb 2020 15:39:33 +0100 Subject: [PATCH 003/158] Deal with excluded datasets --- esmvalcore/_recipe.py | 31 +++++++++++------------------ esmvalcore/preprocessor/__init__.py | 11 +++++----- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 7c4476b197..5618a37de3 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -616,10 +616,13 @@ def _update_statistic_settings(products, order, preproc_dir): group = p.settings[step]['group'] if len(group) < 2: if 'ensemble' in group: - key = '{}_{}_{}'.format(p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']) - name[key] = '{}_Ensemble'.format(key) + try: + key = '{}_{}_{}'.format(p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']) + name[key] = '{}_Ensemble'.format(key) + except KeyError: + continue # datasets cannot be grouped? if 'all' in group: key = 'all' name[key] = 'MultiModel' @@ -636,21 +639,11 @@ def _update_statistic_settings(products, order, preproc_dir): common_settings = _get_remaining_common_settings(step, order, products) statistic_product = PreprocessorFile(attributes, common_settings) for product in products: - settings = product.settings[step] - if 'output_products' not in settings: - settings['output_products'] = defaultdict(dict) - settings['output_products'][key] = {} - settings['output_products'][key][statistic] = statistic_product - - - - - - - - - - + if step in product.settings: + settings = product.settings[step] + if 'output_products' not in settings: + settings['output_products'] = defaultdict(lambda: defaultdict(dict)) + settings['output_products'][key][statistic] = statistic_product def _update_extract_shape(settings, config_user): if 'extract_shape' in settings: diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 36e7d6fe82..7032551e24 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -173,12 +173,11 @@ def _check_multi_model_settings(products): elif reference is None: reference = product elif reference.settings[step] != settings: - continue - #raise ValueError( - # "Unable to combine differing multi-dataset settings for " - # "{} and {}, {} and {}".format( - # reference.filename, product.filename, - # reference.settings[step], settings)) + raise ValueError( + "Unable to combine differing multi-dataset settings for " + "{} and {}, {} and {}".format( + reference.filename, product.filename, + reference.settings[step], settings)) def _get_multi_model_settings(products, step): From e835ebfdb6f76b566b0a626d8357c5f12198e5e3 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 4 Feb 2020 14:03:18 +0100 Subject: [PATCH 004/158] Compute stats of ensemble stats, pending to clean --- esmvalcore/_recipe.py | 30 ++++++------ esmvalcore/preprocessor/_multimodel.py | 64 +++++++++++++++++++------- 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 5618a37de3..fbd2e060ae 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -614,20 +614,23 @@ def _update_statistic_settings(products, order, preproc_dir): for p in products: if step in p.settings: group = p.settings[step]['group'] - if len(group) < 2: - if 'ensemble' in group: - try: - key = '{}_{}_{}'.format(p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']) - name[key] = '{}_Ensemble'.format(key) - except KeyError: - continue # datasets cannot be grouped? - if 'all' in group: - key = 'all' - name[key] = 'MultiModel' + if 'ensemble' in group[0]: + try: + key = '{}_{}_{}'.format(p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']) + name[key] = '{}_Ensemble'.format(key) + except KeyError: + continue # datasets cannot be grouped? + if len(group) == 2 and 'all' in group[1]: + name['concatenate_stats'] = 'MultiModelEnsemble' + prods['concatenate_stats'].add(p) + elif 'all' in group[0] and len(group) < 2: + key = 'all' + name[key] = 'MultiModel' + else: + raise ValueError # wrong options prods[key].add(p) - # PENDING: compute ensemble stats and then stats over ensemble results for key, group_products in prods.items(): some_product = next(iter(group_products)) @@ -645,6 +648,7 @@ def _update_statistic_settings(products, order, preproc_dir): settings['output_products'] = defaultdict(lambda: defaultdict(dict)) settings['output_products'][key][statistic] = statistic_product + def _update_extract_shape(settings, config_user): if 'extract_shape' in settings: shapefile = settings['extract_shape'].get('shapefile') diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index cfa4fa4bac..9b751c20a9 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -302,9 +302,6 @@ def _assemble_full_data(cubes, statistic): stats_cube = _put_in_cube(cubes[0], stats_dats, statistic, time_axis) return stats_cube - - - def multi_model_statistics(products, span, output_products, statistics, group): """ Compute multi-model statistics. @@ -344,24 +341,23 @@ def multi_model_statistics(products, span, output_products, statistics, group): """ prods = defaultdict(set) - statistic_products = set() + statistic_products = defaultdict(set) for p in products: - if len(group) < 2: - if 'ensemble' in group: - key = '{}_{}_{}'.format(p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']) - if 'all' in group: - key = 'all' + if 'ensemble' in group[0]: + key = '{}_{}_{}'.format(p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']) + if 'all' in group[0]: + key = 'all' prods[key].add(p) - for key, ensemble_products in prods.items(): + for key, grouped_products in prods.items(): logger.debug('Multimodel statistics: computing: %s', statistics) - if len(ensemble_products) < 2: + if len(grouped_products) < 2: logger.info("Single dataset in list: will not compute statistics.") return products - cubes = [cube for product in ensemble_products for cube in product.cubes] + cubes = [cube for product in grouped_products for cube in product.cubes] # check if we have any time overlap interval = _get_overlap(cubes) if interval is None: @@ -391,12 +387,46 @@ def multi_model_statistics(products, span, output_products, statistics, group): # Add to output product and log provenance generated_product = output_products[key][statistic] generated_product.cubes = [statistic_cube] - for product in ensemble_products: + for product in grouped_products: + generated_product.wasderivedfrom(product) + logger.info("Generated %s", generated_product) + statistic_products[statistic].add(generated_product) + + if 'concatenate_stats' in output_products: + for statistic in output_products['concatenate_stats']: + cubes = [cube for product in statistic_products[statistic] for cube in product.cubes] + interval = _get_overlap(cubes) + if interval is None: + logger.info("Time overlap between cubes is none or a single point." + "check datasets: will not compute statistics.") + continue + + if span == 'overlap': + logger.debug("Using common time overlap between " + "datasets to compute statistics.") + elif span == 'full': + logger.debug("Using full time spans to compute statistics.") + else: + raise ValueError( + "Unexpected value for span {}, choose from 'overlap', 'full'" + .format(span)) + + if span == 'overlap': + statistic_cube = _assemble_overlap_data(cubes, interval, statistic) + elif span == 'full': + statistic_cube = _assemble_full_data(cubes, statistic) + statistic_cube.data = np.ma.array( + statistic_cube.data, dtype=np.dtype('float32')) + + generated_product = output_products['concatenate_stats'][statistic] + generated_product.cubes = [statistic_cube] + for product in statistic_products[statistic]: generated_product.wasderivedfrom(product) logger.info("Generated %s", generated_product) - statistic_products.add(generated_product) + statistic_products[statistic].add(generated_product) - products |= statistic_products + for statistic in statistics: + products |= statistic_products[statistic] return products From e4163d0172d97ad31cb4bcab1e2c72bfe39e1f7d Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 5 Feb 2020 16:13:25 +0100 Subject: [PATCH 005/158] Move ensemble means to separate functions --- esmvalcore/_recipe.py | 73 +++++++------ esmvalcore/preprocessor/__init__.py | 30 ++++-- esmvalcore/preprocessor/_multimodel.py | 138 ++++++++++--------------- 3 files changed, 117 insertions(+), 124 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index fbd2e060ae..6f58f94fc4 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -609,44 +609,54 @@ def _update_statistic_settings(products, order, preproc_dir): # TODO: move this to multi model statistics function? # But how to check, with a dry-run option? step = 'multi_model_statistics' - prods = defaultdict(set) - name = defaultdict(dict) - for p in products: - if step in p.settings: - group = p.settings[step]['group'] - if 'ensemble' in group[0]: - try: - key = '{}_{}_{}'.format(p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']) - name[key] = '{}_Ensemble'.format(key) - except KeyError: - continue # datasets cannot be grouped? - if len(group) == 2 and 'all' in group[1]: - name['concatenate_stats'] = 'MultiModelEnsemble' - prods['concatenate_stats'].add(p) - elif 'all' in group[0] and len(group) < 2: - key = 'all' - name[key] = 'MultiModel' - else: - raise ValueError # wrong options - prods[key].add(p) - for key, group_products in prods.items(): - some_product = next(iter(group_products)) + products = {p for p in products if step in p.settings} + if not products: + return + + some_product = next(iter(products)) + for statistic in some_product.settings[step]['statistics']: + attributes = _get_statistic_attributes(products) + attributes['dataset'] = 'MultiModel{}'.format(statistic.title()) + attributes['filename'] = get_statistic_output_file( + attributes, preproc_dir) + common_settings = _get_remaining_common_settings(step, order, products) + statistic_product = PreprocessorFile(attributes, common_settings) + for product in products: + settings = product.settings[step] + if 'output_products' not in settings: + settings['output_products'] = {} + settings['output_products'][statistic] = statistic_product + +def _update_ensemble_settings(products, order, preproc_dir): + step = 'ensemble_statistics' + products = {p for p in products if step in p.settings} + if not products: + return + + prods = defaultdict(set) + for product in products: + try: + dataset = '_'.join([product.attributes['project'], + product.attributes['dataset'], + product.attributes['exp']]) + prods[dataset].add(product) + except KeyError: + continue + for dataset, grouped_products in prods.items(): + some_product = next(iter(grouped_products)) for statistic in some_product.settings[step]['statistics']: - attributes = _get_statistic_attributes(group_products) - attributes['dataset'] = '{}{}'.format(name[key], statistic.title()) + attributes = _get_statistic_attributes(products) + attributes['dataset'] = '{}_Ensemble{}'.format(dataset, statistic.title()) attributes['filename'] = get_statistic_output_file( attributes, preproc_dir) common_settings = _get_remaining_common_settings(step, order, products) statistic_product = PreprocessorFile(attributes, common_settings) for product in products: - if step in product.settings: - settings = product.settings[step] - if 'output_products' not in settings: - settings['output_products'] = defaultdict(lambda: defaultdict(dict)) - settings['output_products'][key][statistic] = statistic_product + settings = product.settings[step] + if 'output_products' not in settings: + settings['output_products'] = defaultdict(lambda: defaultdict(dict)) + settings['output_products'][dataset][statistic] = statistic_product def _update_extract_shape(settings, config_user): @@ -748,6 +758,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, products.add(product) _update_statistic_settings(products, order, config_user['preproc_dir']) + _update_ensemble_settings(products, order, config_user['preproc_dir']) for product in products: product.check() diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 7032551e24..bf293dc137 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -17,7 +17,7 @@ from ._mask import (mask_above_threshold, mask_below_threshold, mask_fillvalues, mask_glaciated, mask_inside_range, mask_landsea, mask_landseaice, mask_outside_range) -from ._multimodel import multi_model_statistics +from ._multimodel import multi_model_statistics, ensemble_statistics from ._reformat import (cmor_check_data, cmor_check_metadata, fix_data, fix_file, fix_metadata) from ._regrid import extract_levels, regrid @@ -61,6 +61,8 @@ 'mask_glaciated', # Mask landseaice, sftgif only 'mask_landseaice', + # Ensemble statistics + 'ensemble_statistics', # Regridding 'regrid', # Masking missing values @@ -113,6 +115,7 @@ MULTI_MODEL_FUNCTIONS = { 'multi_model_statistics', 'mask_fillvalues', + 'ensemble_statistics' } @@ -374,16 +377,23 @@ def _initialize_product_provenance(self): product.initialize_provenance(self.activity) # Hacky way to initialize the multi model products as well. - steps = ['multi_model_statistics',] - for step in steps: - input_products = [p for p in self.products if step in p.settings] - if input_products: - statistic_products = set() - for input_product in input_products: - for key, prods in input_product.settings[step].get('output_products', {}).items(): + step = 'multi_model_statistics' + input_products = [p for p in self.products if step in p.settings] + if input_products: + statistic_products = input_products[0].settings[step].get( + 'output_products', {}).values() + for product in statistic_products: + product.initialize_provenance(self.activity) + + step = 'ensemble_statistics' + input_products = [p for p in self.products if step in p.settings] + if input_products: + statistic_products = set() + for inputs in input_products: + for dataset, prods in inputs.settings[step].get('output_products', {}).items(): statistic_products.update(prods.values()) - for product in statistic_products: - product.initialize_provenance(self.activity) + for product in statistic_products: + product.initialize_provenance(self.activity) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9b751c20a9..5be5f75d59 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -22,7 +22,7 @@ import itertools -from collections import OrderedDict, defaultdict +from collections import defaultdict logger = logging.getLogger(__name__) @@ -302,7 +302,7 @@ def _assemble_full_data(cubes, statistic): stats_cube = _put_in_cube(cubes[0], stats_dats, statistic, time_axis) return stats_cube -def multi_model_statistics(products, span, output_products, statistics, group): +def multi_model_statistics(products, span, output_products, statistics): """ Compute multi-model statistics. @@ -340,93 +340,65 @@ def multi_model_statistics(products, span, output_products, statistics, group): If span is neither overlap nor full. """ - prods = defaultdict(set) - statistic_products = defaultdict(set) - for p in products: - if 'ensemble' in group[0]: - key = '{}_{}_{}'.format(p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']) - if 'all' in group[0]: - key = 'all' - prods[key].add(p) - - for key, grouped_products in prods.items(): - logger.debug('Multimodel statistics: computing: %s', statistics) - if len(grouped_products) < 2: - logger.info("Single dataset in list: will not compute statistics.") - return products - - cubes = [cube for product in grouped_products for cube in product.cubes] - # check if we have any time overlap - interval = _get_overlap(cubes) - if interval is None: - logger.info("Time overlap between cubes is none or a single point." - "check datasets: will not compute statistics.") - return products + logger.debug('Multimodel statistics: computing: %s', statistics) + if len(products) < 2: + logger.info("Single dataset in list: will not compute statistics.") + return products + + cubes = [cube for product in products for cube in product.cubes] + # check if we have any time overlap + interval = _get_overlap(cubes) + if interval is None: + logger.info("Time overlap between cubes is none or a single point." + "check datasets: will not compute statistics.") + return products + + if span == 'overlap': + logger.debug("Using common time overlap between " + "datasets to compute statistics.") + elif span == 'full': + logger.debug("Using full time spans to compute statistics.") + else: + raise ValueError( + "Unexpected value for span {}, choose from 'overlap', 'full'" + .format(span)) + statistic_products = set() + for statistic in statistics: + # Compute statistic if span == 'overlap': - logger.debug("Using common time overlap between " - "datasets to compute statistics.") + statistic_cube = _assemble_overlap_data(cubes, interval, statistic) elif span == 'full': - logger.debug("Using full time spans to compute statistics.") - else: - raise ValueError( - "Unexpected value for span {}, choose from 'overlap', 'full'" - .format(span)) - - for statistic in statistics: - # Compute statistic - if span == 'overlap': - statistic_cube = _assemble_overlap_data(cubes, interval, statistic) - elif span == 'full': - statistic_cube = _assemble_full_data(cubes, statistic) - statistic_cube.data = np.ma.array( - statistic_cube.data, dtype=np.dtype('float32')) + statistic_cube = _assemble_full_data(cubes, statistic) + statistic_cube.data = np.ma.array( + statistic_cube.data, dtype=np.dtype('float32')) # Add to output product and log provenance - generated_product = output_products[key][statistic] - generated_product.cubes = [statistic_cube] - for product in grouped_products: - generated_product.wasderivedfrom(product) - logger.info("Generated %s", generated_product) - statistic_products[statistic].add(generated_product) - - if 'concatenate_stats' in output_products: - for statistic in output_products['concatenate_stats']: - cubes = [cube for product in statistic_products[statistic] for cube in product.cubes] - interval = _get_overlap(cubes) - if interval is None: - logger.info("Time overlap between cubes is none or a single point." - "check datasets: will not compute statistics.") - continue - - if span == 'overlap': - logger.debug("Using common time overlap between " - "datasets to compute statistics.") - elif span == 'full': - logger.debug("Using full time spans to compute statistics.") - else: - raise ValueError( - "Unexpected value for span {}, choose from 'overlap', 'full'" - .format(span)) - - if span == 'overlap': - statistic_cube = _assemble_overlap_data(cubes, interval, statistic) - elif span == 'full': - statistic_cube = _assemble_full_data(cubes, statistic) - statistic_cube.data = np.ma.array( - statistic_cube.data, dtype=np.dtype('float32')) - - generated_product = output_products['concatenate_stats'][statistic] - generated_product.cubes = [statistic_cube] - for product in statistic_products[statistic]: - generated_product.wasderivedfrom(product) - logger.info("Generated %s", generated_product) - statistic_products[statistic].add(generated_product) + statistic_product = output_products[statistic] + statistic_product.cubes = [statistic_cube] + for product in products: + statistic_product.wasderivedfrom(product) + logger.info("Generated %s", statistic_product) + statistic_products.add(statistic_product) - for statistic in statistics: - products |= statistic_products[statistic] + products |= statistic_products + + return products + + +def ensemble_statistics(products, output_products, statistics): + prods = defaultdict(set) + span = 'overlap' + for p in products: + dataset = '_'.join([p.attributes['project'], + p.attributes['dataset'], + p.attributes['exp']]) + + prods[dataset].add(p) + + for dataset, ensemble_products in prods.items(): + statistic_products = multi_model_statistics(ensemble_products, span, output_products[dataset], statistics) + products |= statistic_products return products From bafe50c335caf97a89fffc8060d34349da4a5ccd Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 10 Jun 2020 15:07:24 +0200 Subject: [PATCH 006/158] Update after merging --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 0da6326ddf..1fda10c280 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -414,7 +414,7 @@ def ensemble_statistics(products, output_products, statistics): prods[dataset].add(p) for dataset, ensemble_products in prods.items(): - statistic_products = multi_model_statistics(ensemble_products, span, output_products[dataset], statistics) + statistic_products = multi_model_statistics(ensemble_products, span, statistics, output_products[dataset]) products |= statistic_products return products From 8e914e7b505970f21fa8b0125fd929487f82ddf7 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 11:51:00 +0200 Subject: [PATCH 007/158] Fix calendar units to 'days since 1850-01-01' on a standard calendar --- esmvalcore/preprocessor/_multimodel.py | 44 ++++++++---------- .../_multimodel/test_multimodel.py | 46 ++++++------------- 2 files changed, 33 insertions(+), 57 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index a7e68143c4..5676a1c688 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -25,14 +25,6 @@ logger = logging.getLogger(__name__) -def _get_time_offset(time_unit): - """Return a datetime object equivalent to tunit.""" - # tunit e.g. 'day since 1950-01-01 00:00:00.0000000 UTC' - cfunit = cf_units.Unit(time_unit, calendar=cf_units.CALENDAR_STANDARD) - time_offset = cfunit.num2date(0) - return time_offset - - def _plev_fix(dataset, pl_idx): """Extract valid plev data. @@ -112,11 +104,10 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): times = template_cube.coord('time') else: unit_name = template_cube.coord('time').units.name - tunits = cf_units.Unit(unit_name, calendar="standard") - times = iris.coords.DimCoord( - t_axis, - standard_name='time', - units=tunits) + tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") + times = iris.coords.DimCoord(t_axis, + standard_name='time', + units=tunits) coord_names = [c.long_name for c in template_cube.coords()] coord_names.extend([c.standard_name for c in template_cube.coords()]) @@ -152,8 +143,9 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): # correct dspec if necessary fixed_dspec = np.ma.fix_invalid(cube_data, copy=False, fill_value=1e+20) # put in cube - stats_cube = iris.cube.Cube( - fixed_dspec, dim_coords_and_dims=cspec, long_name=statistic) + stats_cube = iris.cube.Cube(fixed_dspec, + dim_coords_and_dims=cspec, + long_name=statistic) coord_names = [coord.name() for coord in template_cube.coords()] if 'air_pressure' in coord_names: if len(template_cube.shape) == 3: @@ -181,11 +173,9 @@ def _datetime_to_int_days(cube): real_dates.append(real_date) # get the number of days starting from the reference unit - time_unit = cube.coord('time').units.name - time_offset = _get_time_offset(time_unit) - days = [(date_obj - time_offset).days for date_obj in real_dates] - - return days + reference_date = datetime(1850, 1, 1) + integer_days = [(date - reference_date).days for date in real_dates] + return integer_days def _align_yearly_axes(cube): @@ -275,8 +265,10 @@ def _assemble_overlap_data(cubes, interval, statistic): for cube, indx in zip(cubes, indices) ] stats_dats[i] = _compute_statistic(time_data, statistic) - stats_cube = _put_in_cube( - cubes[0][sl_1:sl_2 + 1], stats_dats, statistic, t_axis=None) + stats_cube = _put_in_cube(cubes[0][sl_1:sl_2 + 1], + stats_dats, + statistic, + t_axis=None) return stats_cube @@ -384,8 +376,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): logger.debug("Using full time spans to compute statistics.") else: raise ValueError( - "Unexpected value for span {}, choose from 'overlap', 'full'" - .format(span)) + "Unexpected value for span {}, choose from 'overlap', 'full'". + format(span)) for statistic in statistics: # Compute statistic @@ -393,8 +385,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): statistic_cube = _assemble_overlap_data(cubes, interval, statistic) elif span == 'full': statistic_cube = _assemble_full_data(cubes, statistic) - statistic_cube.data = np.ma.array( - statistic_cube.data, dtype=np.dtype('float32')) + statistic_cube.data = np.ma.array(statistic_cube.data, + dtype=np.dtype('float32')) if output_products: # Add to output product and log provenance diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 4cb7533d49..999969c848 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -2,27 +2,19 @@ import unittest -import cftime import iris import numpy as np from cf_units import Unit import tests from esmvalcore.preprocessor import multi_model_statistics -from esmvalcore.preprocessor._multimodel import (_assemble_full_data, - _assemble_overlap_data, - _compute_statistic, - _datetime_to_int_days, - _get_overlap, - _get_time_offset, - _plev_fix, - _put_in_cube, - _slice_cube) +from esmvalcore.preprocessor._multimodel import ( + _assemble_full_data, _assemble_overlap_data, _compute_statistic, + _datetime_to_int_days, _get_overlap, _plev_fix, _put_in_cube, _slice_cube) class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" - def setUp(self): """Prepare tests.""" coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) @@ -35,35 +27,33 @@ def setUp(self): time = iris.coords.DimCoord([15, 45], standard_name='time', bounds=[[1., 30.], [30., 60.]], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + units=Unit('days since 1850-01-01', + calendar='gregorian')) time2 = iris.coords.DimCoord([1., 2., 3., 4.], standard_name='time', bounds=[ [0.5, 1.5], [1.5, 2.5], [2.5, 3.5], - [3.5, 4.5], ], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + [3.5, 4.5], + ], + units=Unit('days since 1850-01-01', + calendar='gregorian')) yr_time = iris.coords.DimCoord([15, 410], standard_name='time', bounds=[[1., 30.], [395., 425.]], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + units=Unit('days since 1850-01-01', + calendar='gregorian')) yr_time2 = iris.coords.DimCoord([1., 367., 733., 1099.], standard_name='time', bounds=[ [0.5, 1.5], [366, 368], [732, 734], - [1098, 1100], ], - units=Unit( - 'days since 1950-01-01', - calendar='gregorian')) + [1098, 1100], + ], + units=Unit('days since 1850-01-01', + calendar='gregorian')) zcoord = iris.coords.DimCoord([0.5, 5., 50.], standard_name='air_pressure', long_name='air_pressure', @@ -98,12 +88,6 @@ def setUp(self): self.cube2_yr = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5_yr) - def test_get_time_offset(self): - """Test time unit.""" - result = _get_time_offset("days since 1950-01-01") - expected = cftime.real_datetime(1950, 1, 1, 0, 0) - np.testing.assert_equal(result, expected) - def test_compute_statistic(self): """Test statistic.""" data = [self.cube1.data[0], self.cube2.data[0]] From d6b9947073920a393d14dc3767e7cd48308b5363 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 12:18:37 +0200 Subject: [PATCH 008/158] More thorough check on source time frequency; align behaviour with regrid time; raise for daily data --- esmvalcore/preprocessor/_multimodel.py | 50 +++++++++-------- .../_multimodel/test_multimodel.py | 54 ++++++++++++------- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 5676a1c688..347091080d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -159,32 +159,36 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): def _datetime_to_int_days(cube): - """Return list of int(days) converted from cube datetime cells.""" - cube = _align_yearly_axes(cube) - time_cells = [cell.point for cell in cube.coord('time').cells()] - - # extract date info - real_dates = [] - for date_obj in time_cells: - # real_date resets the actual data point day - # to the 1st of the month so that there are no - # wrong overlap indices - real_date = datetime(date_obj.year, date_obj.month, 1, 0, 0, 0) - real_dates.append(real_date) - - # get the number of days starting from the reference unit - reference_date = datetime(1850, 1, 1) - integer_days = [(date - reference_date).days for date in real_dates] - return integer_days + """Return list of int(days) with respect to a common reference. + Cubes may have different calendars. This function extracts the date + information from the cube and re-constructs a default calendar, + resetting the actual dates to the 15th of the month or 1st of july for + yearly data (consistent with `regrid_time`), so that there are no + mismatches in the time arrays. -def _align_yearly_axes(cube): - """Perform a time-regridding operation to align time axes for yr data.""" + Doesn't work for (sub)daily data, because different calendars may have + different number of days in the year. + """ + # Extract date info from cube years = [cell.point.year for cell in cube.coord('time').cells()] - # be extra sure that the first point is not in the previous year - if 0 not in np.diff(years): - return regrid_time(cube, 'yr') - return cube + months = [cell.point.month for cell in cube.coord('time').cells()] + + # Reconstruct default calendar + if not 0 in np.diff(years): + # yearly data + standard_dates = [datetime(year, 7, 1) for year in years] + elif not 0 in np.diff(months): + # monthly data + standard_dates = [datetime(year, month, 15) + for year, month in zip(years, months)] + else: + # (sub)daily data + raise ValueError("Multimodel only supports yearly or monthly data") + + # Get the number of days starting from the reference + reference_date = datetime(1850, 1, 1) + return [(date - reference_date).days for date in standard_dates] def _get_overlap(cubes): diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 999969c848..23552617f6 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -29,16 +29,22 @@ def setUp(self): bounds=[[1., 30.], [30., 60.]], units=Unit('days since 1850-01-01', calendar='gregorian')) - time2 = iris.coords.DimCoord([1., 2., 3., 4.], + time2 = iris.coords.DimCoord([45, 75, 105, 135], standard_name='time', bounds=[ - [0.5, 1.5], - [1.5, 2.5], - [2.5, 3.5], - [3.5, 4.5], - ], - units=Unit('days since 1850-01-01', - calendar='gregorian')) + [30., 60.], + [60., 90.], + [90., 120.], + [120., 150.]], + units=Unit( + 'days since 1850-01-01', + calendar='gregorian')) + day_time = iris.coords.DimCoord([1., 2.], + standard_name='time', + bounds=[[0.5, 1.5], [1.5, 2.5]], + units=Unit( + 'days since 1850-01-01', + calendar='gregorian')) yr_time = iris.coords.DimCoord([15, 410], standard_name='time', bounds=[[1., 30.], [395., 425.]], @@ -87,6 +93,10 @@ def setUp(self): coords_spec5_yr = [(yr_time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] self.cube2_yr = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5_yr) + coords_spec_day = [(day_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube1_day = iris.cube.Cube(data2, + dim_coords_and_dims=coords_spec_day) + def test_compute_statistic(self): """Test statistic.""" @@ -101,9 +111,9 @@ def test_compute_statistic(self): def test_compute_full_statistic_mon_cube(self): data = [self.cube1, self.cube2] stats = multi_model_statistics(data, 'full', ['mean']) - expected_full_mean = np.ma.ones((2, 3, 2, 2)) - expected_full_mean.mask = np.zeros((2, 3, 2, 2)) - expected_full_mean.mask[1] = True + expected_full_mean = np.ma.ones((5, 3, 2, 2)) + expected_full_mean.mask = np.ones((5, 3, 2, 2)) + expected_full_mean.mask[1] = False self.assert_array_equal(stats['mean'].data, expected_full_mean) def test_compute_full_statistic_yr_cube(self): @@ -155,36 +165,36 @@ def test_put_in_cube(self): stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) self.assert_array_equal(stat_cube.data, self.cube1.data) - def test_datetime_to_int_days_no_overlap(self): + def test_datetime_to_int_days(self): """Test _datetime_to_int_days.""" computed_dats = _datetime_to_int_days(self.cube1) - expected_dats = [0, 31] + expected_dats = [14, 45] self.assert_array_equal(computed_dats, expected_dats) def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], - [0, 31], "mean") + [14, 45], "mean") expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) def test_assemble_full_data(self): """Test full data.""" comp_full_mean = _assemble_full_data([self.cube1, self.cube2], "mean") - expected_full_mean = np.ma.ones((2, 3, 2, 2)) - expected_full_mean.mask = np.zeros((2, 3, 2, 2)) - expected_full_mean.mask[1] = True + expected_full_mean = np.ma.ones((5, 3, 2, 2)) + expected_full_mean.mask = np.ones((5, 3, 2, 2)) + expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) def test_slice_cube(self): """Test slice cube.""" - comp_slice = _slice_cube(self.cube1, 0, 31) + comp_slice = _slice_cube(self.cube1, 14, 45) self.assert_array_equal([0, 1], comp_slice) def test_get_overlap(self): """Test get overlap.""" full_ovlp = _get_overlap([self.cube1, self.cube1]) - self.assert_array_equal([0, 31], full_ovlp) + self.assert_array_equal([14, 45], full_ovlp) no_ovlp = _get_overlap([self.cube1, self.cube2]) np.testing.assert_equal(None, no_ovlp) @@ -194,6 +204,12 @@ def test_plev_fix(self): expected_data = np.ma.ones((3, 2, 2)) self.assert_array_equal(expected_data, fixed_data) + def test_raise_daily(self): + """Test raise for daily input data.""" + with self.assertRaises(ValueError): + _datetime_to_int_days(self.cube1_day) + + if __name__ == '__main__': unittest.main() From 12f9aa4c6b5265396c27f109db8622d151f23592 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 12:57:17 +0200 Subject: [PATCH 009/158] Simplify _get_overlap --- esmvalcore/preprocessor/_multimodel.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 347091080d..5caa858c18 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -192,26 +192,12 @@ def _datetime_to_int_days(cube): def _get_overlap(cubes): - """ - Get discrete time overlaps. - - This method gets the bounds of coord time - from the cube and assembles a continuous time - axis with smallest unit 1; then it finds the - overlaps by doing a 1-dim intersect; - takes the floor of first date and - ceil of last date. - """ - all_times = [] - for cube in cubes: - span = _datetime_to_int_days(cube) - start, stop = span[0], span[-1] - all_times.append([start, stop]) - bounds = [range(b[0], b[-1] + 1) for b in all_times] - time_pts = reduce(np.intersect1d, bounds) - if len(time_pts) > 1: - time_bounds_list = [time_pts[0], time_pts[-1]] - return time_bounds_list + """Return the intersection of all cubes' time arrays.""" + time_spans = [_datetime_to_int_days(cube) for cube in cubes] + overlap = reduce(np.intersect1d, time_spans) + if len(overlap) > 1: + return [overlap[0], overlap[-1]] + return def _slice_cube(cube, t_1, t_2): From bf82085f24030545b1c5770def1d8396c6d567fb Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 13:27:57 +0200 Subject: [PATCH 010/158] Align behaviour for union (full) and intersection (overlap) of time arrays --- esmvalcore/preprocessor/_multimodel.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 5caa858c18..2b5e73901a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -192,7 +192,7 @@ def _datetime_to_int_days(cube): def _get_overlap(cubes): - """Return the intersection of all cubes' time arrays.""" + """Return bounds of the intersection of all cubes' time arrays.""" time_spans = [_datetime_to_int_days(cube) for cube in cubes] overlap = reduce(np.intersect1d, time_spans) if len(overlap) > 1: @@ -200,6 +200,12 @@ def _get_overlap(cubes): return +def _get_union(cubes): + """Return the union of all cubes' time arrays.""" + time_spans = [_datetime_to_int_days(cube) for cube in cubes] + return reduce(np.union1d, time_spans) + + def _slice_cube(cube, t_1, t_2): """ Efficient slicer. @@ -216,13 +222,6 @@ def _slice_cube(cube, t_1, t_2): return [idxs[0], idxs[-1]] -def _monthly_t(cubes): - """Rearrange time points for monthly data.""" - # get original cubes tpoints - days = {day for cube in cubes for day in _datetime_to_int_days(cube)} - return sorted(days) - - def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): """Construct a contiguous collection over time.""" for idx_cube, cube in enumerate(cubes): @@ -264,8 +263,9 @@ def _assemble_overlap_data(cubes, interval, statistic): def _assemble_full_data(cubes, statistic): """Get statistical data in iris cubes for FULL.""" - # all times, new MONTHLY data time axis - time_axis = [float(fl) for fl in _monthly_t(cubes)] + # Gather the unique time points in the union of all cubes + time_points = _get_union(cubes) + time_axis = [float(fl) for fl in time_points] # new big time-slice array shape new_shape = [len(time_axis)] + list(cubes[0].shape[1:]) From 3bd1e82b02603852dd3dd3fa74519753eb5279be Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 14:43:59 +0200 Subject: [PATCH 011/158] Add function to make all cubes use the same calendar. This function also checks the frequency of the cubes (previously in _datetime_to_int_days), this seems to be a much more logical place. _datetime_to_int_days now simplifies to a one-liner, and can probably be eliminated completely. --- esmvalcore/preprocessor/_multimodel.py | 61 +++++++++++++------ .../_multimodel/test_multimodel.py | 15 +++-- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 2b5e73901a..46c374ae63 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -158,8 +158,9 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube -def _datetime_to_int_days(cube): - """Return list of int(days) with respect to a common reference. +def _set_common_calendar(cubes): + """ + Make sure all cubes' use the same standard calendar. Cubes may have different calendars. This function extracts the date information from the cube and re-constructs a default calendar, @@ -170,25 +171,42 @@ def _datetime_to_int_days(cube): Doesn't work for (sub)daily data, because different calendars may have different number of days in the year. """ - # Extract date info from cube - years = [cell.point.year for cell in cube.coord('time').cells()] - months = [cell.point.month for cell in cube.coord('time').cells()] - - # Reconstruct default calendar - if not 0 in np.diff(years): - # yearly data - standard_dates = [datetime(year, 7, 1) for year in years] - elif not 0 in np.diff(months): - # monthly data - standard_dates = [datetime(year, month, 15) - for year, month in zip(years, months)] - else: - # (sub)daily data - raise ValueError("Multimodel only supports yearly or monthly data") + # The default time unit + t_unit = cf_units.Unit("days since 1850-01-01", calendar="standard") + + for cube in cubes: + # Extract date info from cube + years = [cell.point.year for cell in cube.coord('time').cells()] + months = [cell.point.month for cell in cube.coord('time').cells()] + + # Reconstruct default calendar + if not 0 in np.diff(years): + # yearly data + dates = [datetime(year, 7, 1) for year in years] + + elif not 0 in np.diff(months): + # monthly data + dates = [datetime(year, month, 15) + for year, month in zip(years, months)] + else: + # (sub)daily data + raise ValueError("Multimodel only supports yearly or monthly data") + + # Update the cubes' time coordinate (both point values and the units!) + cube.coord('time').points = [t_unit.date2num(date) for date in dates] + cube.coord('time').units = t_unit + # Reset bounds + cube.coord('time').bounds = None + cube.coord('time').guess_bounds() + # Remove aux coords that may differ + for auxcoord in cube.aux_coords: + if auxcoord.long_name in ['day_of_month', 'day_of_year']: + cube.remove_coord(auxcoord) - # Get the number of days starting from the reference - reference_date = datetime(1850, 1, 1) - return [(date - reference_date).days for date in standard_dates] + +def _datetime_to_int_days(cube): + """Return the cube's time point values as list of integers.""" + return cube.coord('time').points.astype(int).tolist() def _get_overlap(cubes): @@ -353,6 +371,9 @@ def multi_model_statistics(products, span, statistics, output_products=None): cubes = products statistic_products = {} + # Make cubes share the same calendar, so time points are comparable + _set_common_calendar(cubes) + if span == 'overlap': # check if we have any time overlap interval = _get_overlap(cubes) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 23552617f6..2f59b10278 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -9,7 +9,7 @@ import tests from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( - _assemble_full_data, _assemble_overlap_data, _compute_statistic, + _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, _datetime_to_int_days, _get_overlap, _plev_fix, _put_in_cube, _slice_cube) @@ -24,12 +24,12 @@ def setUp(self): mask3[0, 0, 0, 0] = True data3 = np.ma.array(data3, mask=mask3) - time = iris.coords.DimCoord([15, 45], + time = iris.coords.DimCoord([14, 45], standard_name='time', bounds=[[1., 30.], [30., 60.]], units=Unit('days since 1850-01-01', calendar='gregorian')) - time2 = iris.coords.DimCoord([45, 75, 105, 135], + time2 = iris.coords.DimCoord([45, 73, 104, 134], standard_name='time', bounds=[ [30., 60.], @@ -45,7 +45,7 @@ def setUp(self): units=Unit( 'days since 1850-01-01', calendar='gregorian')) - yr_time = iris.coords.DimCoord([15, 410], + yr_time = iris.coords.DimCoord([14., 410.], standard_name='time', bounds=[[1., 30.], [395., 425.]], units=Unit('days since 1850-01-01', @@ -204,10 +204,15 @@ def test_plev_fix(self): expected_data = np.ma.ones((3, 2, 2)) self.assert_array_equal(expected_data, fixed_data) + def test_set_common_calendar(self): + """Test set common calenar.""" + cubes = [self.cube1, self.cube2] + # TODO: complete this test + def test_raise_daily(self): """Test raise for daily input data.""" with self.assertRaises(ValueError): - _datetime_to_int_days(self.cube1_day) + _set_common_calendar([self.cube1_day]) From 7841a5f39356429d1b7b2af7900a8dda048378fd Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 16:04:56 +0200 Subject: [PATCH 012/158] Remove _datetime_to_int_days, as we can just use the time points --- esmvalcore/preprocessor/_multimodel.py | 19 +++++++------------ .../_multimodel/test_multimodel.py | 8 +------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 46c374ae63..969a206208 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -204,24 +204,19 @@ def _set_common_calendar(cubes): cube.remove_coord(auxcoord) -def _datetime_to_int_days(cube): - """Return the cube's time point values as list of integers.""" - return cube.coord('time').points.astype(int).tolist() - - def _get_overlap(cubes): """Return bounds of the intersection of all cubes' time arrays.""" - time_spans = [_datetime_to_int_days(cube) for cube in cubes] - overlap = reduce(np.intersect1d, time_spans) + time_spans = [cube.coord('time').points for cube in cubes] + overlap = reduce(np.intersect1d, time_spans).astype(int) if len(overlap) > 1: return [overlap[0], overlap[-1]] - return + return None def _get_union(cubes): """Return the union of all cubes' time arrays.""" - time_spans = [_datetime_to_int_days(cube) for cube in cubes] - return reduce(np.union1d, time_spans) + time_spans = [cube.coord('time').points for cube in cubes] + return reduce(np.union1d, time_spans).astype(int) def _slice_cube(cube, t_1, t_2): @@ -232,7 +227,7 @@ def _slice_cube(cube, t_1, t_2): of common time-data elements. """ time_pts = [t for t in cube.coord('time').points] - converted_t = _datetime_to_int_days(cube) + converted_t = cube.coord('time').points.astype(int).tolist() idxs = sorted([ time_pts.index(ii) for ii, jj in zip(time_pts, converted_t) if t_1 <= jj <= t_2 @@ -303,7 +298,7 @@ def _assemble_full_data(cubes, statistic): # loop through cubes and populate empty_arr with points for cube in cubes: - time_redone = _datetime_to_int_days(cube) + time_redone = cube.coord('time').points.astype(int).tolist() oidx = [time_axis.index(s) for s in time_redone] indices_list.append(oidx) for i in range(new_shape[0]): diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 2f59b10278..48df977716 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -10,7 +10,7 @@ from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, - _datetime_to_int_days, _get_overlap, _plev_fix, _put_in_cube, _slice_cube) + _get_overlap, _plev_fix, _put_in_cube, _slice_cube) class Test(tests.Test): @@ -165,12 +165,6 @@ def test_put_in_cube(self): stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) self.assert_array_equal(stat_cube.data, self.cube1.data) - def test_datetime_to_int_days(self): - """Test _datetime_to_int_days.""" - computed_dats = _datetime_to_int_days(self.cube1) - expected_dats = [14, 45] - self.assert_array_equal(computed_dats, expected_dats) - def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], From f95aa7c79adc06b18639f524aa4f7a384f2d1271 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 16:42:43 +0200 Subject: [PATCH 013/158] Simplify and rename _slice_cube --- esmvalcore/preprocessor/_multimodel.py | 25 ++++++------------- .../_multimodel/test_multimodel.py | 6 ++--- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 969a206208..f7bb9a8ee2 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -20,8 +20,6 @@ import iris import numpy as np -from ._time import regrid_time - logger = logging.getLogger(__name__) @@ -219,20 +217,11 @@ def _get_union(cubes): return reduce(np.union1d, time_spans).astype(int) -def _slice_cube(cube, t_1, t_2): - """ - Efficient slicer. - - Simple cube data slicer on indices - of common time-data elements. - """ - time_pts = [t for t in cube.coord('time').points] - converted_t = cube.coord('time').points.astype(int).tolist() - idxs = sorted([ - time_pts.index(ii) for ii, jj in zip(time_pts, converted_t) - if t_1 <= jj <= t_2 - ]) - return [idxs[0], idxs[-1]] +def _get_slice_parameters(cube, tmin, tmax): + """Get the lower and upper array indices for a given time interval.""" + time = cube.coord('time').points + idxs = np.argwhere((time >= tmin) & (time <= tmax)) + return [idxs.min(), idxs.max()] def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): @@ -254,12 +243,12 @@ def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): def _assemble_overlap_data(cubes, interval, statistic): """Get statistical data in iris cubes for OVERLAP.""" start, stop = interval - sl_1, sl_2 = _slice_cube(cubes[0], start, stop) + sl_1, sl_2 = _get_slice_parameters(cubes[0], start, stop) stats_dats = np.ma.zeros(cubes[0].data[sl_1:sl_2 + 1].shape) # keep this outside the following loop # this speeds up the code by a factor of 15 - indices = [_slice_cube(cube, start, stop) for cube in cubes] + indices = [_get_slice_parameters(cube, start, stop) for cube in cubes] for i in range(stats_dats.shape[0]): time_data = [ diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 48df977716..d55d6148c0 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -10,7 +10,7 @@ from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, - _get_overlap, _plev_fix, _put_in_cube, _slice_cube) + _get_overlap, _plev_fix, _put_in_cube, _get_slice_parameters) class Test(tests.Test): @@ -180,9 +180,9 @@ def test_assemble_full_data(self): expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) - def test_slice_cube(self): + def test_get_slice_parameters(self): """Test slice cube.""" - comp_slice = _slice_cube(self.cube1, 14, 45) + comp_slice = _get_slice_parameters(self.cube1, 14, 45) self.assert_array_equal([0, 1], comp_slice) def test_get_overlap(self): From db47522282693f66753cd9905e5c7e8bd369bf2d Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 22:33:16 +0200 Subject: [PATCH 014/158] Align _assemble_overlap_data more with _assemble_full_data --- esmvalcore/preprocessor/_multimodel.py | 63 +++++++++---------- .../_multimodel/test_multimodel.py | 26 +++++--- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index f7bb9a8ee2..ba9561a1f1 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -101,7 +101,6 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): if t_axis is None: times = template_cube.coord('time') else: - unit_name = template_cube.coord('time').units.name tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") times = iris.coords.DimCoord(t_axis, standard_name='time', @@ -202,26 +201,22 @@ def _set_common_calendar(cubes): cube.remove_coord(auxcoord) -def _get_overlap(cubes): +def _get_time_intersection(cubes): """Return bounds of the intersection of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] - overlap = reduce(np.intersect1d, time_spans).astype(int) - if len(overlap) > 1: - return [overlap[0], overlap[-1]] - return None + return reduce(np.intersect1d, time_spans).astype(int) -def _get_union(cubes): +def _get_time_union(cubes): """Return the union of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] return reduce(np.union1d, time_spans).astype(int) -def _get_slice_parameters(cube, tmin, tmax): - """Get the lower and upper array indices for a given time interval.""" +def _get_subset(cube, tmin, tmax): time = cube.coord('time').points idxs = np.argwhere((time >= tmin) & (time <= tmax)) - return [idxs.min(), idxs.max()] + return cube[idxs.min():idxs.max()+1] def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): @@ -240,33 +235,35 @@ def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): return ndatarr -def _assemble_overlap_data(cubes, interval, statistic): +def _assemble_overlap_data(cubes, statistic): """Get statistical data in iris cubes for OVERLAP.""" - start, stop = interval - sl_1, sl_2 = _get_slice_parameters(cubes[0], start, stop) - stats_dats = np.ma.zeros(cubes[0].data[sl_1:sl_2 + 1].shape) - - # keep this outside the following loop - # this speeds up the code by a factor of 15 - indices = [_get_slice_parameters(cube, start, stop) for cube in cubes] - - for i in range(stats_dats.shape[0]): - time_data = [ - cube.data[indx[0]:indx[1] + 1][i] - for cube, indx in zip(cubes, indices) - ] - stats_dats[i] = _compute_statistic(time_data, statistic) - stats_cube = _put_in_cube(cubes[0][sl_1:sl_2 + 1], - stats_dats, - statistic, - t_axis=None) + # Gather overlapping time points + new_times = _get_time_intersection(cubes).tolist() + tmin = min(new_times) + tmax = max(new_times) + n_times = len(new_times) + + # Target array to populate with computed statistics + new_shape = [n_times] + list(cubes[0].shape[1:]) + stats_data = np.ma.zeros(new_shape) + + # Prepare a list of cubes with matching times (so far just pointers) + # Keep this outside the following loop; 15x speedup (still true?) + cubelist = [_get_subset(cube, tmin, tmax) for cube in cubes] + + for i in range(n_times): + time_data = [cube.data[i] for cube in cubelist] + stats_data[i] = _compute_statistic(time_data, statistic) + + template_cube = cubelist[0] + stats_cube = _put_in_cube(template_cube, stats_data, statistic, new_times) return stats_cube def _assemble_full_data(cubes, statistic): """Get statistical data in iris cubes for FULL.""" # Gather the unique time points in the union of all cubes - time_points = _get_union(cubes) + time_points = _get_time_union(cubes) time_axis = [float(fl) for fl in time_points] # new big time-slice array shape @@ -360,8 +357,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): if span == 'overlap': # check if we have any time overlap - interval = _get_overlap(cubes) - if interval is None: + overlap = _get_time_intersection(cubes) + if len(overlap) <= 1: logger.info("Time overlap between cubes is none or a single point." "check datasets: will not compute statistics.") return products @@ -377,7 +374,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic if span == 'overlap': - statistic_cube = _assemble_overlap_data(cubes, interval, statistic) + statistic_cube = _assemble_overlap_data(cubes, statistic) elif span == 'full': statistic_cube = _assemble_full_data(cubes, statistic) statistic_cube.data = np.ma.array(statistic_cube.data, diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index d55d6148c0..e9124464bb 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -10,7 +10,7 @@ from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, - _get_overlap, _plev_fix, _put_in_cube, _get_slice_parameters) + _get_time_intersection, _plev_fix, _put_in_cube) class Test(tests.Test): @@ -39,6 +39,14 @@ def setUp(self): units=Unit( 'days since 1850-01-01', calendar='gregorian')) + time3 = iris.coords.DimCoord([104, 134], + standard_name='time', + bounds=[ + [90., 120.], + [120., 150.]], + units=Unit( + 'days since 1850-01-01', + calendar='gregorian')) day_time = iris.coords.DimCoord([1., 2.], standard_name='time', bounds=[[0.5, 1.5], [1.5, 2.5]], @@ -86,6 +94,9 @@ def setUp(self): coords_spec5 = [(time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] self.cube2 = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5) + coords_spec6 = [(time3, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube3 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec6) + coords_spec4_yr = [(yr_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] self.cube1_yr = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec4_yr) @@ -168,7 +179,7 @@ def test_put_in_cube(self): def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], - [14, 45], "mean") + "mean") expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) @@ -180,16 +191,11 @@ def test_assemble_full_data(self): expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) - def test_get_slice_parameters(self): - """Test slice cube.""" - comp_slice = _get_slice_parameters(self.cube1, 14, 45) - self.assert_array_equal([0, 1], comp_slice) - - def test_get_overlap(self): + def test_get_time_intersection(self): """Test get overlap.""" - full_ovlp = _get_overlap([self.cube1, self.cube1]) + full_ovlp = _get_time_intersection([self.cube1, self.cube1]) self.assert_array_equal([14, 45], full_ovlp) - no_ovlp = _get_overlap([self.cube1, self.cube2]) + no_ovlp = _get_time_intersection([self.cube1, self.cube3]) np.testing.assert_equal(None, no_ovlp) def test_plev_fix(self): From 127c69b231cb2da88374a3ee69650b77a646b889 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 23 Jun 2020 22:49:34 +0200 Subject: [PATCH 015/158] Align _assemble_full_data more with _assemble_overlap_data --- esmvalcore/preprocessor/_multimodel.py | 46 +++++++++++--------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ba9561a1f1..fd0d5d479d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -204,13 +204,13 @@ def _set_common_calendar(cubes): def _get_time_intersection(cubes): """Return bounds of the intersection of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.intersect1d, time_spans).astype(int) + return reduce(np.intersect1d, time_spans) def _get_time_union(cubes): """Return the union of all cubes' time arrays.""" time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.union1d, time_spans).astype(int) + return reduce(np.union1d, time_spans) def _get_subset(cube, tmin, tmax): @@ -255,48 +255,42 @@ def _assemble_overlap_data(cubes, statistic): time_data = [cube.data[i] for cube in cubelist] stats_data[i] = _compute_statistic(time_data, statistic) - template_cube = cubelist[0] - stats_cube = _put_in_cube(template_cube, stats_data, statistic, new_times) + template = cubelist[0] + stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube def _assemble_full_data(cubes, statistic): """Get statistical data in iris cubes for FULL.""" - # Gather the unique time points in the union of all cubes - time_points = _get_time_union(cubes) - time_axis = [float(fl) for fl in time_points] + # Gather time points in the union of all cubes + new_times = _get_time_union(cubes).tolist() + n_times = len(new_times) - # new big time-slice array shape - new_shape = [len(time_axis)] + list(cubes[0].shape[1:]) + # Target array to populate with computed statistics + new_shape = [n_times] + list(cubes[0].shape[1:]) + stats_data = np.ma.zeros(new_shape) # assemble an array to hold all time data # for all cubes; shape is (ncubes,(plev), lat, lon) new_arr = np.ma.empty([len(cubes)] + list(new_shape[1:])) - # data array for stats computation - stats_dats = np.ma.zeros(new_shape) - - # assemble indices list to chop new_arr on - indices_list = [] - # empty data array to hold time slices empty_arr = np.ma.empty(new_shape) - # loop through cubes and populate empty_arr with points + # Prepare a list mapping the cubes' times to the indices of new_times + indices_list = [] for cube in cubes: - time_redone = cube.coord('time').points.astype(int).tolist() - oidx = [time_axis.index(s) for s in time_redone] + oidx = [new_times.index(t) for t in cube.coord('time').points] indices_list.append(oidx) - for i in range(new_shape[0]): + + for i in range(n_times): # hold time slices only - new_datas_array = _full_time_slice(cubes, empty_arr, indices_list, + time_data = _full_time_slice(cubes, empty_arr, indices_list, new_arr, i) - # list to hold time slices - time_data = [] - for j in range(len(cubes)): - time_data.append(new_datas_array[j]) - stats_dats[i] = _compute_statistic(time_data, statistic) - stats_cube = _put_in_cube(cubes[0], stats_dats, statistic, time_axis) + stats_data[i] = _compute_statistic(time_data, statistic) + + template = cubes[0] + stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube From f873ea523dfae49bf4d026e9558da1f394b54699 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 13:52:42 +0200 Subject: [PATCH 016/158] Futher align assemble full and overlap data --- esmvalcore/preprocessor/_multimodel.py | 62 ++++++++++---------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fd0d5d479d..9522069821 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -219,43 +219,42 @@ def _get_subset(cube, tmin, tmax): return cube[idxs.min():idxs.max()+1] -def _full_time_slice(cubes, ndat, indices, ndatarr, t_idx): - """Construct a contiguous collection over time.""" - for idx_cube, cube in enumerate(cubes): - # reset mask - ndat.mask = True - ndat[indices[idx_cube]] = cube.data - if np.ma.is_masked(cube.data): - ndat.mask[indices[idx_cube]] = cube.data.mask +def _get_index(cube, time): + # return cube.coord('time').points.tolist().index(time) + cubetime = cube.coord('time').points + return int(np.argwhere(time == cubetime)) + + +def _get_time_slice(cubes, time): + """Fill time slice array with cubes' data if time in cube, else mask.""" + time_slice = [] + for j, cube in enumerate(cubes): + cube_time = cube.coord('time').points + if time in cube_time: + idx = int(np.argwhere(cube_time == time)) + subset = cube[idx].data else: - ndat.mask[indices[idx_cube]] = False - ndatarr[idx_cube] = ndat[t_idx] - - # return time slice - return ndatarr + subset = np.ma.empty(list(cubes[0].shape[1:])) + subset.mask = True + time_slice.append(subset) + return time_slice def _assemble_overlap_data(cubes, statistic): """Get statistical data in iris cubes for OVERLAP.""" # Gather overlapping time points new_times = _get_time_intersection(cubes).tolist() - tmin = min(new_times) - tmax = max(new_times) n_times = len(new_times) # Target array to populate with computed statistics new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) - # Prepare a list of cubes with matching times (so far just pointers) - # Keep this outside the following loop; 15x speedup (still true?) - cubelist = [_get_subset(cube, tmin, tmax) for cube in cubes] - - for i in range(n_times): - time_data = [cube.data[i] for cube in cubelist] + for i, time in enumerate(new_times): + time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) - template = cubelist[0] + template = cubes[0][:n_times] stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube @@ -270,23 +269,8 @@ def _assemble_full_data(cubes, statistic): new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) - # assemble an array to hold all time data - # for all cubes; shape is (ncubes,(plev), lat, lon) - new_arr = np.ma.empty([len(cubes)] + list(new_shape[1:])) - - # empty data array to hold time slices - empty_arr = np.ma.empty(new_shape) - - # Prepare a list mapping the cubes' times to the indices of new_times - indices_list = [] - for cube in cubes: - oidx = [new_times.index(t) for t in cube.coord('time').points] - indices_list.append(oidx) - - for i in range(n_times): - # hold time slices only - time_data = _full_time_slice(cubes, empty_arr, indices_list, - new_arr, i) + for i, time in enumerate(new_times): + time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) template = cubes[0] From 3b7b55bfd002afca2f32bdbf6981193948d7d795 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 14:05:51 +0200 Subject: [PATCH 017/158] Merge assemble full and overlap data. --- esmvalcore/preprocessor/_multimodel.py | 47 ++++--------------- .../_multimodel/test_multimodel.py | 8 ++-- 2 files changed, 12 insertions(+), 43 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9522069821..f1047a1581 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -213,18 +213,6 @@ def _get_time_union(cubes): return reduce(np.union1d, time_spans) -def _get_subset(cube, tmin, tmax): - time = cube.coord('time').points - idxs = np.argwhere((time >= tmin) & (time <= tmax)) - return cube[idxs.min():idxs.max()+1] - - -def _get_index(cube, time): - # return cube.coord('time').points.tolist().index(time) - cubetime = cube.coord('time').points - return int(np.argwhere(time == cubetime)) - - def _get_time_slice(cubes, time): """Fill time slice array with cubes' data if time in cube, else mask.""" time_slice = [] @@ -240,29 +228,13 @@ def _get_time_slice(cubes, time): return time_slice -def _assemble_overlap_data(cubes, statistic): - """Get statistical data in iris cubes for OVERLAP.""" - # Gather overlapping time points - new_times = _get_time_intersection(cubes).tolist() - n_times = len(new_times) - - # Target array to populate with computed statistics - new_shape = [n_times] + list(cubes[0].shape[1:]) - stats_data = np.ma.zeros(new_shape) - - for i, time in enumerate(new_times): - time_data = _get_time_slice(cubes, time) - stats_data[i] = _compute_statistic(time_data, statistic) - - template = cubes[0][:n_times] - stats_cube = _put_in_cube(template, stats_data, statistic, new_times) - return stats_cube - +def _assemble_data(cubes, statistic, span='overlap'): + """Get statistical data in iris cubes.""" + if span == 'overlap': + new_times = _get_time_intersection(cubes).tolist() + elif span == 'full': + new_times = _get_time_union(cubes).tolist() -def _assemble_full_data(cubes, statistic): - """Get statistical data in iris cubes for FULL.""" - # Gather time points in the union of all cubes - new_times = _get_time_union(cubes).tolist() n_times = len(new_times) # Target array to populate with computed statistics @@ -273,7 +245,7 @@ def _assemble_full_data(cubes, statistic): time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) - template = cubes[0] + template = cubes[0][:n_times] stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube @@ -351,10 +323,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic - if span == 'overlap': - statistic_cube = _assemble_overlap_data(cubes, statistic) - elif span == 'full': - statistic_cube = _assemble_full_data(cubes, statistic) + statistic_cube = _assemble_data(cubes, statistic, span) statistic_cube.data = np.ma.array(statistic_cube.data, dtype=np.dtype('float32')) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index e9124464bb..80eb36593e 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -9,7 +9,7 @@ import tests from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( - _assemble_full_data, _assemble_overlap_data, _compute_statistic, _set_common_calendar, + _assemble_data, _compute_statistic, _set_common_calendar, _get_time_intersection, _plev_fix, _put_in_cube) @@ -178,14 +178,14 @@ def test_put_in_cube(self): def test_assemble_overlap_data(self): """Test overlap data.""" - comp_ovlap_mean = _assemble_overlap_data([self.cube1, self.cube1], - "mean") + comp_ovlap_mean = _assemble_data([self.cube1, self.cube1], + "mean", span='overlap') expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) def test_assemble_full_data(self): """Test full data.""" - comp_full_mean = _assemble_full_data([self.cube1, self.cube2], "mean") + comp_full_mean = _assemble_data([self.cube1, self.cube2], "mean", span='full') expected_full_mean = np.ma.ones((5, 3, 2, 2)) expected_full_mean.mask = np.ones((5, 3, 2, 2)) expected_full_mean.mask[1] = False From 168a701acea37ffd52222b55525e93d0c671ec26 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 15:49:06 +0200 Subject: [PATCH 018/158] Further simplify --- esmvalcore/preprocessor/_multimodel.py | 34 ++++++------------- .../_multimodel/test_multimodel.py | 12 ++----- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index f1047a1581..061f68e665 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -98,13 +98,8 @@ def _compute_statistic(data, statistic_name): def _put_in_cube(template_cube, cube_data, statistic, t_axis): """Quick cube building and saving.""" - if t_axis is None: - times = template_cube.coord('time') - else: - tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") - times = iris.coords.DimCoord(t_axis, - standard_name='time', - units=tunits) + tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") + times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits) coord_names = [c.long_name for c in template_cube.coords()] coord_names.extend([c.standard_name for c in template_cube.coords()]) @@ -201,22 +196,10 @@ def _set_common_calendar(cubes): cube.remove_coord(auxcoord) -def _get_time_intersection(cubes): - """Return bounds of the intersection of all cubes' time arrays.""" - time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.intersect1d, time_spans) - - -def _get_time_union(cubes): - """Return the union of all cubes' time arrays.""" - time_spans = [cube.coord('time').points for cube in cubes] - return reduce(np.union1d, time_spans) - - def _get_time_slice(cubes, time): """Fill time slice array with cubes' data if time in cube, else mask.""" time_slice = [] - for j, cube in enumerate(cubes): + for cube in cubes: cube_time = cube.coord('time').points if time in cube_time: idx = int(np.argwhere(cube_time == time)) @@ -230,17 +213,19 @@ def _get_time_slice(cubes, time): def _assemble_data(cubes, statistic, span='overlap'): """Get statistical data in iris cubes.""" + # New time array representing the union or intersection of all cubes + time_spans = [cube.coord('time').points for cube in cubes] if span == 'overlap': - new_times = _get_time_intersection(cubes).tolist() + new_times = reduce(np.intersect1d, time_spans) elif span == 'full': - new_times = _get_time_union(cubes).tolist() - + new_times = reduce(np.union1d, time_spans) n_times = len(new_times) # Target array to populate with computed statistics new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) + # Make time slices and compute stats for i, time in enumerate(new_times): time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) @@ -307,7 +292,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): if span == 'overlap': # check if we have any time overlap - overlap = _get_time_intersection(cubes) + times = [cube.coord('time').points for cube in cubes] + overlap = reduce(np.intersect1d, times) if len(overlap) <= 1: logger.info("Time overlap between cubes is none or a single point." "check datasets: will not compute statistics.") diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 80eb36593e..78979b13b1 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -9,8 +9,7 @@ import tests from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import ( - _assemble_data, _compute_statistic, _set_common_calendar, - _get_time_intersection, _plev_fix, _put_in_cube) + _assemble_data, _compute_statistic, _set_common_calendar, _plev_fix, _put_in_cube) class Test(tests.Test): @@ -173,7 +172,7 @@ def test_compute_min(self): def test_put_in_cube(self): """Test put in cube.""" cube_data = np.ma.ones((2, 3, 2, 2)) - stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=None) + stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=[1,2]) self.assert_array_equal(stat_cube.data, self.cube1.data) def test_assemble_overlap_data(self): @@ -191,13 +190,6 @@ def test_assemble_full_data(self): expected_full_mean.mask[1] = False self.assert_array_equal(comp_full_mean.data, expected_full_mean) - def test_get_time_intersection(self): - """Test get overlap.""" - full_ovlp = _get_time_intersection([self.cube1, self.cube1]) - self.assert_array_equal([14, 45], full_ovlp) - no_ovlp = _get_time_intersection([self.cube1, self.cube3]) - np.testing.assert_equal(None, no_ovlp) - def test_plev_fix(self): """Test plev fix.""" fixed_data = _plev_fix(self.cube2.data, 1) From 629a9d6f4f6f776bf8204d77e5d9998e985c00cb Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 16:17:06 +0200 Subject: [PATCH 019/158] Remove stuff about bounds and aux coords as it is not used anyway --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 061f68e665..3cc005109e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -172,11 +172,11 @@ def _set_common_calendar(cubes): months = [cell.point.month for cell in cube.coord('time').cells()] # Reconstruct default calendar - if not 0 in np.diff(years): + if 0 not in np.diff(years): # yearly data dates = [datetime(year, 7, 1) for year in years] - elif not 0 in np.diff(months): + elif 0 not in np.diff(months): # monthly data dates = [datetime(year, month, 15) for year, month in zip(years, months)] From f538fa85cc257aa8772d02446c3f8740b1e1f776 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 16:18:04 +0200 Subject: [PATCH 020/158] Remove stuff about bounds and aux coords as it is not used anyway --- esmvalcore/preprocessor/_multimodel.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 3cc005109e..dbb9729c1c 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -187,13 +187,6 @@ def _set_common_calendar(cubes): # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] cube.coord('time').units = t_unit - # Reset bounds - cube.coord('time').bounds = None - cube.coord('time').guess_bounds() - # Remove aux coords that may differ - for auxcoord in cube.aux_coords: - if auxcoord.long_name in ['day_of_month', 'day_of_year']: - cube.remove_coord(auxcoord) def _get_time_slice(cubes, time): From b0cc1ae4fb439edcaad68b7d83f104017cde650b Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 24 Jun 2020 17:22:48 +0200 Subject: [PATCH 021/158] Clean up tests and add tests for new functions --- esmvalcore/preprocessor/_multimodel.py | 10 +- .../_multimodel/test_multimodel.py | 152 +++++++++--------- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index dbb9729c1c..3ace7e860e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -69,7 +69,7 @@ def _compute_statistic(data, statistic_name): # data is per time point # so we can safely NOT compute stats for single points if data.ndim == 1: - u_datas = [d for d in data] + u_datas = data else: u_datas = [d for d in data if not np.all(d.mask)] if len(u_datas) > 1: @@ -178,8 +178,10 @@ def _set_common_calendar(cubes): elif 0 not in np.diff(months): # monthly data - dates = [datetime(year, month, 15) - for year, month in zip(years, months)] + dates = [ + datetime(year, month, 15) + for year, month in zip(years, months) + ] else: # (sub)daily data raise ValueError("Multimodel only supports yearly or monthly data") @@ -258,11 +260,13 @@ def multi_model_statistics(products, span, statistics, output_products=None): statistics: str statistical measure to be computed. Available options: mean, median, max, min, std + Returns ------- list list of data products or cubes containing the multimodel stats computed. + Raises ------ ValueError diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 78979b13b1..7e98642cdb 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -8,65 +8,33 @@ import tests from esmvalcore.preprocessor import multi_model_statistics -from esmvalcore.preprocessor._multimodel import ( - _assemble_data, _compute_statistic, _set_common_calendar, _plev_fix, _put_in_cube) +from esmvalcore.preprocessor._multimodel import (_assemble_data, + _compute_statistic, + _get_time_slice, _plev_fix, + _put_in_cube, + _set_common_calendar) class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" + def setUp(self): """Prepare tests.""" - coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) - data2 = np.ma.ones((2, 3, 2, 2)) - data3 = np.ma.ones((4, 3, 2, 2)) - mask3 = np.full((4, 3, 2, 2), False) - mask3[0, 0, 0, 0] = True - data3 = np.ma.array(data3, mask=mask3) - - time = iris.coords.DimCoord([14, 45], - standard_name='time', - bounds=[[1., 30.], [30., 60.]], - units=Unit('days since 1850-01-01', - calendar='gregorian')) - time2 = iris.coords.DimCoord([45, 73, 104, 134], - standard_name='time', - bounds=[ - [30., 60.], - [60., 90.], - [90., 120.], - [120., 150.]], - units=Unit( - 'days since 1850-01-01', - calendar='gregorian')) - time3 = iris.coords.DimCoord([104, 134], - standard_name='time', - bounds=[ - [90., 120.], - [120., 150.]], - units=Unit( - 'days since 1850-01-01', - calendar='gregorian')) - day_time = iris.coords.DimCoord([1., 2.], - standard_name='time', - bounds=[[0.5, 1.5], [1.5, 2.5]], - units=Unit( - 'days since 1850-01-01', - calendar='gregorian')) - yr_time = iris.coords.DimCoord([14., 410.], - standard_name='time', - bounds=[[1., 30.], [395., 425.]], - units=Unit('days since 1850-01-01', - calendar='gregorian')) - yr_time2 = iris.coords.DimCoord([1., 367., 733., 1099.], - standard_name='time', - bounds=[ - [0.5, 1.5], - [366, 368], - [732, 734], - [1098, 1100], - ], - units=Unit('days since 1850-01-01', - calendar='gregorian')) + # Make various time arrays + time_args = { + 'standard_name': 'time', + 'units': Unit('days since 1850-01-01', calendar='gregorian') + } + monthly1 = iris.coords.DimCoord([14, 45], **time_args) + monthly2 = iris.coords.DimCoord([45, 73, 104, 134], **time_args) + monthly3 = iris.coords.DimCoord([104, 134], **time_args) + yearly1 = iris.coords.DimCoord([14., 410.], **time_args) + yearly2 = iris.coords.DimCoord([1., 367., 733., 1099.], **time_args) + daily1 = iris.coords.DimCoord([1., 2.], **time_args) + for time in [monthly1, monthly2, monthly3, yearly1, yearly2, daily1]: + time.guess_bounds() + + # Other dimensions are fixed zcoord = iris.coords.DimCoord([0.5, 5., 50.], standard_name='air_pressure', long_name='air_pressure', @@ -74,6 +42,7 @@ def setUp(self): [25., 250.]], units='m', attributes={'positive': 'down'}) + coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) lons = iris.coords.DimCoord([1.5, 2.5], standard_name='longitude', long_name='longitude', @@ -87,26 +56,29 @@ def setUp(self): units='degrees_north', coord_system=coord_sys) - coords_spec4 = [(time, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube1 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec4) + data1 = np.ma.ones((2, 3, 2, 2)) + data2 = np.ma.ones((4, 3, 2, 2)) + mask2 = np.full((4, 3, 2, 2), False) + mask2[0, 0, 0, 0] = True + data2 = np.ma.array(data2, mask=mask2) + + coords_spec1 = [(monthly1, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube1 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec1) - coords_spec5 = [(time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube2 = iris.cube.Cube(data3, dim_coords_and_dims=coords_spec5) + coords_spec2 = [(monthly2, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube2 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec2) - coords_spec6 = [(time3, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube3 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec6) + coords_spec3 = [(monthly3, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube3 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec3) - coords_spec4_yr = [(yr_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube1_yr = iris.cube.Cube(data2, - dim_coords_and_dims=coords_spec4_yr) + coords_spec4 = [(yearly1, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube4 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec4) - coords_spec5_yr = [(yr_time2, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube2_yr = iris.cube.Cube(data3, - dim_coords_and_dims=coords_spec5_yr) - coords_spec_day = [(day_time, 0), (zcoord, 1), (lats, 2), (lons, 3)] - self.cube1_day = iris.cube.Cube(data2, - dim_coords_and_dims=coords_spec_day) + coords_spec5 = [(yearly2, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube5 = iris.cube.Cube(data2, dim_coords_and_dims=coords_spec5) + coords_spec6 = [(daily1, 0), (zcoord, 1), (lats, 2), (lons, 3)] + self.cube6 = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec6) def test_compute_statistic(self): """Test statistic.""" @@ -127,7 +99,7 @@ def test_compute_full_statistic_mon_cube(self): self.assert_array_equal(stats['mean'].data, expected_full_mean) def test_compute_full_statistic_yr_cube(self): - data = [self.cube1_yr, self.cube2_yr] + data = [self.cube4, self.cube5] stats = multi_model_statistics(data, 'full', ['mean']) expected_full_mean = np.ma.ones((4, 3, 2, 2)) expected_full_mean.mask = np.zeros((4, 3, 2, 2)) @@ -141,7 +113,7 @@ def test_compute_overlap_statistic_mon_cube(self): self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) def test_compute_overlap_statistic_yr_cube(self): - data = [self.cube1_yr, self.cube1_yr] + data = [self.cube4, self.cube4] stats = multi_model_statistics(data, 'overlap', ['mean']) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) @@ -172,19 +144,22 @@ def test_compute_min(self): def test_put_in_cube(self): """Test put in cube.""" cube_data = np.ma.ones((2, 3, 2, 2)) - stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=[1,2]) + stat_cube = _put_in_cube(self.cube1, cube_data, "mean", t_axis=[1, 2]) self.assert_array_equal(stat_cube.data, self.cube1.data) def test_assemble_overlap_data(self): """Test overlap data.""" comp_ovlap_mean = _assemble_data([self.cube1, self.cube1], - "mean", span='overlap') + "mean", + span='overlap') expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(comp_ovlap_mean.data, expected_ovlap_mean) def test_assemble_full_data(self): """Test full data.""" - comp_full_mean = _assemble_data([self.cube1, self.cube2], "mean", span='full') + comp_full_mean = _assemble_data([self.cube1, self.cube2], + "mean", + span='full') expected_full_mean = np.ma.ones((5, 3, 2, 2)) expected_full_mean.mask = np.ones((5, 3, 2, 2)) expected_full_mean.mask[1] = False @@ -198,14 +173,39 @@ def test_plev_fix(self): def test_set_common_calendar(self): """Test set common calenar.""" + cube1 = self.cube1 + time1 = cube1.coord('time') + t_unit1 = time1.units + dates = t_unit1.num2date(time1.points) + + t_unit2 = Unit('days since 1850-01-01', calendar='gregorian') + time2 = t_unit2.date2num(dates) + cube2 = self.cube1.copy() + cube2.coord('time').points = time2 + cube2.coord('time').units = t_unit2 + _set_common_calendar([cube1, cube2]) + self.assertEqual(cube1.coord('time'), cube2.coord('time')) + + def test_get_time_slice_all(self): + """Test get time slice if all cubes have data.""" cubes = [self.cube1, self.cube2] - # TODO: complete this test + result = _get_time_slice(cubes, time=45) + expected = [self.cube1[1].data, self.cube2[0].data] + self.assert_array_equal(expected, result) + + def test_get_time_slice_part(self): + """Test get time slice if all cubes have data.""" + cubes = [self.cube1, self.cube2] + result = _get_time_slice(cubes, time=14) + masked = np.ma.empty(list(cubes[0].shape[1:])) + masked.mask = True + expected = [self.cube1[0].data, masked] + self.assert_array_equal(expected, result) def test_raise_daily(self): """Test raise for daily input data.""" with self.assertRaises(ValueError): - _set_common_calendar([self.cube1_day]) - + _set_common_calendar([self.cube6]) if __name__ == '__main__': From aef2578047cca45bb950267cc0618d901ffc65ce Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 29 Jun 2020 17:31:07 +0200 Subject: [PATCH 022/158] Valeriu's suggestions --- doc/recipe/preprocessor.rst | 6 +++++- esmvalcore/preprocessor/_multimodel.py | 11 +++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 0456f12108..56d4159dcf 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -663,6 +663,10 @@ from a statistical point of view, this is needed since weights are not yet implemented; also higher dimensional data is not supported (i.e. anything with dimensionality higher than four: time, vertical axis, two horizontal axes). +Input datasets may have different time coordinates. The multi-model statistics +preprocessor sets a common time coordinate on all datasets. As the number of +days in a year may vary between calendars, (sub-)daily data are not supported. + .. code-block:: yaml preprocessors: @@ -674,7 +678,7 @@ dimensionality higher than four: time, vertical axis, two horizontal axes). see also :func:`esmvalcore.preprocessor.multi_model_statistics`. -When calling the module inside diagnostic scripts, the input must be given +When calling the module inside diagnostic scripts, the input must be given as a list of cubes. The output will be saved in a dictionary where each entry contains the resulting cube with the requested statistic operations. diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 3ace7e860e..fc047e5c05 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -150,9 +150,9 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube -def _set_common_calendar(cubes): +def _unify_time_coordinates(cubes): """ - Make sure all cubes' use the same standard calendar. + Make sure all cubes' use the same standard calendar and time units. Cubes may have different calendars. This function extracts the date information from the cube and re-constructs a default calendar, @@ -189,6 +189,7 @@ def _set_common_calendar(cubes): # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] cube.coord('time').units = t_unit + cube.coord('time').guess_bounds() def _get_time_slice(cubes, time): @@ -271,6 +272,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): ------ ValueError If span is neither overlap nor full. + ValueError + If the time frequency of the input data not yearly or monthly. """ logger.debug('Multimodel statistics: computing: %s', statistics) @@ -284,8 +287,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): cubes = products statistic_products = {} - # Make cubes share the same calendar, so time points are comparable - _set_common_calendar(cubes) + # Reset time coordinates and make cubes share the same calendar + _unify_time_coordinates(cubes) if span == 'overlap': # check if we have any time overlap From 24294dffeb4ca6d91113f0d04dda78ea27741fcd Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 29 Jun 2020 17:37:35 +0200 Subject: [PATCH 023/158] fix tests --- esmvalcore/preprocessor/_multimodel.py | 1 + tests/unit/preprocessor/_multimodel/test_multimodel.py | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fc047e5c05..efd8267d10 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -189,6 +189,7 @@ def _unify_time_coordinates(cubes): # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] cube.coord('time').units = t_unit + cube.coord('time').bounds = None cube.coord('time').guess_bounds() diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 7e98642cdb..e0c01000d4 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -12,12 +12,11 @@ _compute_statistic, _get_time_slice, _plev_fix, _put_in_cube, - _set_common_calendar) + _unify_time_coordinates) class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" - def setUp(self): """Prepare tests.""" # Make various time arrays @@ -171,7 +170,7 @@ def test_plev_fix(self): expected_data = np.ma.ones((3, 2, 2)) self.assert_array_equal(expected_data, fixed_data) - def test_set_common_calendar(self): + def test_unify_time_coordinates(self): """Test set common calenar.""" cube1 = self.cube1 time1 = cube1.coord('time') @@ -183,7 +182,7 @@ def test_set_common_calendar(self): cube2 = self.cube1.copy() cube2.coord('time').points = time2 cube2.coord('time').units = t_unit2 - _set_common_calendar([cube1, cube2]) + _unify_time_coordinates([cube1, cube2]) self.assertEqual(cube1.coord('time'), cube2.coord('time')) def test_get_time_slice_all(self): @@ -205,7 +204,7 @@ def test_get_time_slice_part(self): def test_raise_daily(self): """Test raise for daily input data.""" with self.assertRaises(ValueError): - _set_common_calendar([self.cube6]) + _unify_time_coordinates([self.cube6]) if __name__ == '__main__': From 35210a5c6ba7f135f046a664f56b8f6b90fe3839 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 2 Jul 2020 14:02:54 +0200 Subject: [PATCH 024/158] Realize data before making time slices --- esmvalcore/preprocessor/_multimodel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fccdcc6d4b..67d4b3364d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -237,7 +237,7 @@ def _get_time_slice(cubes, time): idx = int(np.argwhere(cube_time == time)) subset = cube[idx].data else: - subset = np.ma.empty(list(cubes[0].shape[1:])) + subset = np.ma.empty(list(cube.shape[1:])) subset.mask = True time_slice.append(subset) return time_slice @@ -257,6 +257,9 @@ def _assemble_data(cubes, statistic, span='overlap'): new_shape = [n_times] + list(cubes[0].shape[1:]) stats_data = np.ma.zeros(new_shape) + # Realize all cubes at once instead of separately for each time slice + [cube.data for cube in cubes] + # Make time slices and compute stats for i, time in enumerate(new_times): time_data = _get_time_slice(cubes, time) From d2fddeba2199f2c20df8848004a5178737536db0 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 2 Jul 2020 15:19:55 +0200 Subject: [PATCH 025/158] Avoid codacy 'pointless-statement' message --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 67d4b3364d..0a7fddaf5a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -258,7 +258,7 @@ def _assemble_data(cubes, statistic, span='overlap'): stats_data = np.ma.zeros(new_shape) # Realize all cubes at once instead of separately for each time slice - [cube.data for cube in cubes] + _ = [cube.data for cube in cubes] # Make time slices and compute stats for i, time in enumerate(new_times): From cc51e1531a91d195160891e55941a0546711bfb7 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 4 Aug 2020 17:56:22 +0200 Subject: [PATCH 026/158] Address Bouwe's comments --- esmvalcore/preprocessor/_multimodel.py | 14 +++++++------- .../preprocessor/_multimodel/test_multimodel.py | 6 +----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 0a7fddaf5a..d213ec43a3 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -219,7 +219,10 @@ def _unify_time_coordinates(cubes): ] else: # (sub)daily data - raise ValueError("Multimodel only supports yearly or monthly data") + logger.warning( + "Multimodel encountered (sub)daily data. Attempting to" + " continue, but might fail for incompatible calendars.") + break # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] @@ -255,7 +258,7 @@ def _assemble_data(cubes, statistic, span='overlap'): # Target array to populate with computed statistics new_shape = [n_times] + list(cubes[0].shape[1:]) - stats_data = np.ma.zeros(new_shape) + stats_data = np.ma.zeros(new_shape, dtype=np.dtype('float32')) # Realize all cubes at once instead of separately for each time slice _ = [cube.data for cube in cubes] @@ -312,9 +315,6 @@ def multi_model_statistics(products, span, statistics, output_products=None): ------ ValueError If span is neither overlap nor full. - ValueError - If the time frequency of the input data not yearly or monthly. - """ logger.debug('Multimodel statistics: computing: %s', statistics) if len(products) < 2: @@ -350,8 +350,8 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic statistic_cube = _assemble_data(cubes, statistic, span) - statistic_cube.data = np.ma.array(statistic_cube.data, - dtype=np.dtype('float32')) + # statistic_cube.data = np.ma.array(statistic_cube.data, + # dtype=np.dtype('float32')) if output_products: # Add to output product and log provenance diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index d01142dacb..42b28a9525 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -17,6 +17,7 @@ class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" + def setUp(self): """Prepare tests.""" # Make various time arrays @@ -209,11 +210,6 @@ def test_get_time_slice_part(self): expected = [self.cube1[0].data, masked] self.assert_array_equal(expected, result) - def test_raise_daily(self): - """Test raise for daily input data.""" - with self.assertRaises(ValueError): - _unify_time_coordinates([self.cube6]) - if __name__ == '__main__': unittest.main() From 7c182a22605b2f84686357360b7594773f172c5e Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 5 Aug 2020 16:02:43 +0200 Subject: [PATCH 027/158] Update esmvalcore/preprocessor/_multimodel.py Co-authored-by: Bouwe Andela --- esmvalcore/preprocessor/_multimodel.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index d213ec43a3..09481b6c45 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -350,8 +350,6 @@ def multi_model_statistics(products, span, statistics, output_products=None): for statistic in statistics: # Compute statistic statistic_cube = _assemble_data(cubes, statistic, span) - # statistic_cube.data = np.ma.array(statistic_cube.data, - # dtype=np.dtype('float32')) if output_products: # Add to output product and log provenance From a663ea68d0042ee7ad2591ed58ba596b0343737a Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 7 Aug 2020 10:20:30 +0200 Subject: [PATCH 028/158] Don't change the calendar if not necessary --- esmvalcore/preprocessor/_multimodel.py | 37 +++++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index d213ec43a3..86444d2857 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -185,21 +185,30 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): return stats_cube +def _get_consistent_time_unit(cubes): + """Return cubes' time unit if consistent, standard calendar otherwise.""" + t_units = [cube.coord('time').units for cube in cubes] + if len(set(t_units)) == 1: + return t_units[0] + return cf_units.Unit("days since 1850-01-01", calendar="standard") + + def _unify_time_coordinates(cubes): """ - Make sure all cubes' use the same standard calendar and time units. + Make sure all cubes' share the same time coordinate. + + This function extracts the date information from the cube and + reconstructs the time coordinate, resetting the actual dates to the + 15th of the month or 1st of july for yearly data (consistent with + `regrid_time`), so that there are no mismatches in the time arrays. - Cubes may have different calendars. This function extracts the date - information from the cube and re-constructs a default calendar, - resetting the actual dates to the 15th of the month or 1st of july for - yearly data (consistent with `regrid_time`), so that there are no - mismatches in the time arrays. + If cubes have different time units, it will use reset the calendar to + a default gregorian calendar with unit "days since 1850-01-01". - Doesn't work for (sub)daily data, because different calendars may have + Might not work for (sub)daily data, because different calendars may have different number of days in the year. """ - # The default time unit - t_unit = cf_units.Unit("days since 1850-01-01", calendar="standard") + t_unit = _get_consistent_time_unit(cubes) for cube in cubes: # Extract date info from cube @@ -219,10 +228,12 @@ def _unify_time_coordinates(cubes): ] else: # (sub)daily data - logger.warning( - "Multimodel encountered (sub)daily data. Attempting to" - " continue, but might fail for incompatible calendars.") - break + if cube.coord('time').units != t_unit: + logger.warning( + "Multimodel encountered (sub)daily data and inconsistent " + "time units or calendars. Attempting to continue, but " + "might produce unexpected results.") + dates = [cell.point for cell in cube.coord('time').cells()] # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] From c8b5f4b4d1cfe452953313b3f921c2e6111fd756 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 7 Aug 2020 12:09:43 +0200 Subject: [PATCH 029/158] Use template cube's calendar and don't slice by time --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ad478cebb1..82fba4901a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -133,7 +133,7 @@ def _compute_statistic(data, statistic_name): def _put_in_cube(template_cube, cube_data, statistic, t_axis): """Quick cube building and saving.""" - tunits = cf_units.Unit("days since 1850-01-01", calendar="standard") + tunits = template_cube.coord('time').units times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits) coord_names = [c.long_name for c in template_cube.coords()] @@ -279,7 +279,7 @@ def _assemble_data(cubes, statistic, span='overlap'): time_data = _get_time_slice(cubes, time) stats_data[i] = _compute_statistic(time_data, statistic) - template = cubes[0][:n_times] + template = cubes[0] stats_cube = _put_in_cube(template, stats_data, statistic, new_times) return stats_cube From fcb9955d0b3dfdb062444ae803bb9611c45d7060 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 10 Aug 2020 09:36:20 +0200 Subject: [PATCH 030/158] Use num2date rather than cells() method --- esmvalcore/preprocessor/_multimodel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 82fba4901a..6f2f306d73 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -212,8 +212,9 @@ def _unify_time_coordinates(cubes): for cube in cubes: # Extract date info from cube - years = [cell.point.year for cell in cube.coord('time').cells()] - months = [cell.point.month for cell in cube.coord('time').cells()] + coord = cube.coord('time') + years = [p.year for p in coord.units.num2date(coord.points)] + months = [p.year for p in coord.units.num2date(coord.points)] # Reconstruct default calendar if 0 not in np.diff(years): @@ -228,12 +229,13 @@ def _unify_time_coordinates(cubes): ] else: # (sub)daily data - if cube.coord('time').units != t_unit: + coord = cube.coord('time') + if coord.units != t_unit: logger.warning( "Multimodel encountered (sub)daily data and inconsistent " "time units or calendars. Attempting to continue, but " "might produce unexpected results.") - dates = [cell.point for cell in cube.coord('time').cells()] + dates = coord.units.num2date(coord.points) # Update the cubes' time coordinate (both point values and the units!) cube.coord('time').points = [t_unit.date2num(date) for date in dates] From 46337f73ca9184cec7e295137a34f43069fa47fb Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 10 Aug 2020 09:37:49 +0200 Subject: [PATCH 031/158] Apply suggestions from code review Co-authored-by: Bouwe Andela --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 82fba4901a..cc0dc3c68e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -236,7 +236,7 @@ def _unify_time_coordinates(cubes): dates = [cell.point for cell in cube.coord('time').cells()] # Update the cubes' time coordinate (both point values and the units!) - cube.coord('time').points = [t_unit.date2num(date) for date in dates] + cube.coord('time').points = t_unit.date2num(dates) cube.coord('time').units = t_unit cube.coord('time').bounds = None cube.coord('time').guess_bounds() @@ -249,7 +249,7 @@ def _get_time_slice(cubes, time): cube_time = cube.coord('time').points if time in cube_time: idx = int(np.argwhere(cube_time == time)) - subset = cube[idx].data + subset = cube.data[idx] else: subset = np.ma.empty(list(cube.shape[1:])) subset.mask = True From a77b1eb43c92612191267018bb2dbc6f594c7a86 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 11:12:35 +0200 Subject: [PATCH 032/158] Refactor provenance initialization code --- esmvalcore/preprocessor/__init__.py | 37 +++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 0020b70bb7..074d239a00 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -387,7 +387,7 @@ def __init__( debug=None, write_ncl_interface=False, ): - """Initialize""" + """Initialize.""" _check_multi_model_settings(products) super().__init__(ancestors=ancestors, name=name, products=products) self.order = list(order) @@ -396,30 +396,43 @@ def __init__( def _initialize_product_provenance(self): """Initialize product provenance.""" - for product in self.products: - product.initialize_provenance(self.activity) + self._initialize_products(self.products) + self._initialize_multi_model_statistics_provenance() + self._initialize_ensemble_statistics_provenance - # Hacky way to initialize the multi model products as well. + def _initialize_multi_model_statistics_provenance(self): + """Initialize provenance for multi-model statistics.""" step = 'multi_model_statistics' - input_products = [p for p in self.products if step in p.settings] + input_products = self._get_input_products(step) if input_products: statistic_products = input_products[0].settings[step].get( 'output_products', {}).values() - for product in statistic_products: - product.initialize_provenance(self.activity) + self._initialize_products(statistic_products) + + def _initialize_ensemble_statistics_provenance(self): + """Initialize provenance for ensemble statistics.""" step = 'ensemble_statistics' - input_products = [p for p in self.products if step in p.settings] + input_products = self._get_input_products(step) if input_products: statistic_products = set() + for inputs in input_products: - for dataset, prods in inputs.settings[step].get('output_products', {}).items(): - statistic_products.update(prods.values()) - for product in statistic_products: - product.initialize_provenance(self.activity) + items = inputs.settings[step].get('output_products', {}).items() + for dataset, products in items: + statistic_products.update(products.values()) + self._initialize_products(statistic_products) + def _get_input_products(self, step): + """Get input products.""" + return [product for product in self.products if step in product.settings] + + def _initialize_products(self, products): + """Initialize products.""" + for product in products: + product.initialize_provenance(self.activity) def _run(self, _): """Run the preprocessor.""" From 816fbc229d96e2703a9ac7cdeebd58da38034bde Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 12:03:27 +0200 Subject: [PATCH 033/158] Refactor code to update ensemble settings --- esmvalcore/_recipe.py | 47 +++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 2a741d2c6f..0740bce394 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -576,8 +576,8 @@ def _apply_preprocessor_profile(settings, profile_settings): settings[step].update(args) -def _get_statistic_attributes(products): - """Get attributes for the statistic output products.""" +def _get_common_attributes(products): + """Get common attributes for the output products.""" attributes = {} some_product = next(iter(products)) for key, value in some_product.attributes.items(): @@ -629,7 +629,7 @@ def _update_statistic_settings(products, order, preproc_dir): some_product = next(iter(products)) for statistic in some_product.settings[step]['statistics']: - attributes = _get_statistic_attributes(products) + attributes = _get_common_attributes(products) attributes['dataset'] = attributes['alias'] = 'MultiModel{}'.format( statistic.title()) attributes['filename'] = get_statistic_output_file( @@ -642,35 +642,48 @@ def _update_statistic_settings(products, order, preproc_dir): settings['output_products'] = {} settings['output_products'][statistic] = statistic_product + def _update_ensemble_settings(products, order, preproc_dir): step = 'ensemble_statistics' - products = {p for p in products if step in p.settings} + products = {product for product in products if step in product.settings} if not products: return - prods = defaultdict(set) + grouped_products_dict = defaultdict(set) for product in products: try: - dataset = '_'.join([product.attributes['project'], - product.attributes['dataset'], - product.attributes['exp']]) - prods[dataset].add(product) + identifier = '_'.join([product.attributes['project'], + product.attributes['dataset'], + product.attributes['exp']]) except KeyError: continue - for dataset, grouped_products in prods.items(): + else: + grouped_products_dict[identifier].add(product) + + for identifier, grouped_products in grouped_products_dict.items(): some_product = next(iter(grouped_products)) - for statistic in some_product.settings[step]['statistics']: - attributes = _get_statistic_attributes(products) - attributes['dataset'] = '{}_Ensemble{}'.format(dataset, statistic.title()) - attributes['filename'] = get_statistic_output_file( - attributes, preproc_dir) + statistics = some_product.settings[step]['statistics'] + + for statistic in statistics: + common_attributes = _get_common_attributes(products) + + title = statistic.title() + common_attributes['dataset'] = f'{identifier}_Ensemble{title}' + + filename = get_statistic_output_file(common_attributes, preproc_dir) + common_attributes['filename'] = filename + common_settings = _get_remaining_common_settings(step, order, products) - statistic_product = PreprocessorFile(attributes, common_settings) + statistic_product = PreprocessorFile(common_attributes, common_settings) + for product in products: settings = product.settings[step] + if 'output_products' not in settings: + # assume output products is a nested dict settings['output_products'] = defaultdict(lambda: defaultdict(dict)) - settings['output_products'][dataset][statistic] = statistic_product + + settings['output_products'][identifier][statistic] = statistic_product def _update_extract_shape(settings, config_user): From 3f2e126fcaeb499d5f7a6185d9514bcc8f5f737f Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 12:03:58 +0200 Subject: [PATCH 034/158] Avoid double dictionary lookup --- esmvalcore/_recipe.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 0740bce394..5e0508d13f 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -791,8 +791,9 @@ def _get_preprocessor_products(variables, ) products.add(product) - _update_statistic_settings(products, order, config_user['preproc_dir']) - _update_ensemble_settings(products, order, config_user['preproc_dir']) + preproc_dir = config_user['preproc_dir'] + _update_statistic_settings(products, order, preproc_dir) + _update_ensemble_settings(products, order, preproc_dir) for product in products: product.check() From 96866436973de2f0e173a5ca947091c443430d98 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 12:04:12 +0200 Subject: [PATCH 035/158] Clean ensemble statistics code --- esmvalcore/preprocessor/_multimodel.py | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 1fda10c280..a2a11fcb2f 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -396,7 +396,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): statistic_products.add(statistic_product) else: statistic_products[statistic] = statistic_cube - + if output_products: products |= statistic_products return products @@ -404,17 +404,22 @@ def multi_model_statistics(products, span, statistics, output_products=None): def ensemble_statistics(products, output_products, statistics): - prods = defaultdict(set) + product_dict = defaultdict(set) span = 'overlap' - for p in products: - dataset = '_'.join([p.attributes['project'], - p.attributes['dataset'], - p.attributes['exp']]) - - prods[dataset].add(p) - - for dataset, ensemble_products in prods.items(): - statistic_products = multi_model_statistics(ensemble_products, span, statistics, output_products[dataset]) - products |= statistic_products + for product in products: + dataset = '_'.join([product.attributes['project'], + product.attributes['dataset'], + product.attributes['exp']]) + + product_dict[dataset].add(product) + + for dataset, ensemble_products in product_dict.items(): + statistic_products = multi_model_statistics( + ensemble_products, + span, + statistics, + output_products[dataset], + ) + products |= statistic_products # set union return products From 8b7f07634c495f5976a9cc76437775a3e1aba79b Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 13:34:09 +0200 Subject: [PATCH 036/158] Clarify error --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ecdd5f3b60..8a8d42fa0a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -100,7 +100,7 @@ def _compute_statistic(data, statistic_name): quantile = float(statistic_name[1:]) / 100 statistic_function = partial(_quantile, quantile=quantile) else: - raise NotImplementedError + raise ValueError(f'No such statistic: `{statistic_name}`') # no plevs if len(data[0].shape) < 3: From 851e0ec2f3a9d6008dc0b4be4284d185c8f5d545 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 13:34:29 +0200 Subject: [PATCH 037/158] Fix PickleError (lambdas cannot be pickled) --- esmvalcore/_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 50b784ca38..6f512d7237 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -682,7 +682,7 @@ def _update_ensemble_settings(products, order, preproc_dir): if 'output_products' not in settings: # assume output products is a nested dict - settings['output_products'] = defaultdict(lambda: defaultdict(dict)) + settings['output_products'] = defaultdict(dict) settings['output_products'][identifier][statistic] = statistic_product From 1bffc76016e7f84111c0dddb95488f796b8bf7bf Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 13:34:58 +0200 Subject: [PATCH 038/158] Fix crash when exp is a list --- esmvalcore/_recipe.py | 3 ++- esmvalcore/preprocessor/_multimodel.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 6f512d7237..559ffc0bf5 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -655,13 +655,14 @@ def _update_ensemble_settings(products, order, preproc_dir): try: identifier = '_'.join([product.attributes['project'], product.attributes['dataset'], - product.attributes['exp']]) + ''.join(product.attributes['exp'])]) # TODO: clean this except KeyError: continue else: grouped_products_dict[identifier].add(product) for identifier, grouped_products in grouped_products_dict.items(): + some_product = next(iter(grouped_products)) statistics = some_product.settings[step]['statistics'] diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 8a8d42fa0a..86cd56327d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -385,13 +385,21 @@ def multi_model_statistics(products, span, statistics, output_products=None): return statistic_products -def ensemble_statistics(products, output_products, statistics): +def ensemble_statistics(products, output_products, statistics: list): + """Calculate ensemble statistics. + + Parameters + ---------- + statistics: list + List of operators to apply to the data ensemble + """ product_dict = defaultdict(set) span = 'overlap' + for product in products: dataset = '_'.join([product.attributes['project'], product.attributes['dataset'], - product.attributes['exp']]) + ''.join(product.attributes['exp'])]) # TODO: clean this product_dict[dataset].add(product) From 8d319b05668b2b913bf7a24c41c3048e2a7dff17 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 17:34:03 +0200 Subject: [PATCH 039/158] Rename variable --- esmvalcore/preprocessor/_multimodel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 86cd56327d..984299263b 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -397,18 +397,18 @@ def ensemble_statistics(products, output_products, statistics: list): span = 'overlap' for product in products: - dataset = '_'.join([product.attributes['project'], + identifier = '_'.join([product.attributes['project'], product.attributes['dataset'], ''.join(product.attributes['exp'])]) # TODO: clean this - product_dict[dataset].add(product) + product_dict[identifier].add(product) - for dataset, ensemble_products in product_dict.items(): + for identifier, ensemble_products in product_dict.items(): statistic_products = multi_model_statistics( ensemble_products, span, statistics, - output_products[dataset], + output_products[identifier], ) products |= statistic_products # set union From 98c2592405d1b30d01e9442f3405870788757bd4 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 1 Sep 2020 17:34:13 +0200 Subject: [PATCH 040/158] Add missing parentheses --- esmvalcore/preprocessor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 5bfb61bd66..dbe03b3258 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -398,7 +398,7 @@ def _initialize_product_provenance(self): """Initialize product provenance.""" self._initialize_products(self.products) self._initialize_multi_model_statistics_provenance() - self._initialize_ensemble_statistics_provenance + self._initialize_ensemble_statistics_provenance() def _initialize_multi_model_statistics_provenance(self): """Initialize provenance for multi-model statistics.""" From 5be9d9b5821c3073718cc8885bb0c1d02a4d05c5 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 3 Sep 2020 13:47:50 +0200 Subject: [PATCH 041/158] Don't return the union after a multimodel function --- esmvalcore/preprocessor/_multimodel.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 984299263b..79e56fad6c 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -333,6 +333,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): ValueError If span is neither overlap nor full. """ + # breakpoint() logger.debug('Multimodel statistics: computing: %s', statistics) if len(products) < 2: logger.info("Single dataset in list: will not compute statistics.") @@ -379,9 +380,6 @@ def multi_model_statistics(products, span, statistics, output_products=None): else: statistic_products[statistic] = statistic_cube - if output_products: - products |= statistic_products - return products return statistic_products @@ -410,6 +408,5 @@ def ensemble_statistics(products, output_products, statistics: list): statistics, output_products[identifier], ) - products |= statistic_products # set union - return products + return statistic_products From 29123729093a64bd865313d7e7f6d5dfdeb751ac Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 3 Sep 2020 13:49:17 +0200 Subject: [PATCH 042/158] remove breakpoint --- esmvalcore/preprocessor/_multimodel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 79e56fad6c..a229e30357 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -333,7 +333,6 @@ def multi_model_statistics(products, span, statistics, output_products=None): ValueError If span is neither overlap nor full. """ - # breakpoint() logger.debug('Multimodel statistics: computing: %s', statistics) if len(products) < 2: logger.info("Single dataset in list: will not compute statistics.") From 0a587fcca91d95b655e93c7e34ee46b1e5fd4495 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 3 Sep 2020 15:33:03 +0200 Subject: [PATCH 043/158] Return all statistics products --- esmvalcore/preprocessor/_multimodel.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index a229e30357..dbc9619e59 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -335,7 +335,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): """ logger.debug('Multimodel statistics: computing: %s', statistics) if len(products) < 2: - logger.info("Single dataset in list: will not compute statistics.") + logger.warning("Single dataset in list: will not compute statistics.") return products if output_products: cubes = [cube for product in products for cube in product.cubes] @@ -392,7 +392,6 @@ def ensemble_statistics(products, output_products, statistics: list): """ product_dict = defaultdict(set) span = 'overlap' - for product in products: identifier = '_'.join([product.attributes['project'], product.attributes['dataset'], @@ -400,12 +399,14 @@ def ensemble_statistics(products, output_products, statistics: list): product_dict[identifier].add(product) + statistic_products = set() for identifier, ensemble_products in product_dict.items(): - statistic_products = multi_model_statistics( + statistic_product = multi_model_statistics( ensemble_products, span, statistics, output_products[identifier], ) + statistic_products |= statistic_product return statistic_products From d3b1b7f9c94730ddb62739252da0354605df4711 Mon Sep 17 00:00:00 2001 From: Stef Date: Thu, 3 Sep 2020 17:32:13 +0200 Subject: [PATCH 044/158] Clarify code --- esmvalcore/_recipe.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 559ffc0bf5..1b11e3f475 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -630,13 +630,18 @@ def _update_statistic_settings(products, order, preproc_dir): some_product = next(iter(products)) for statistic in some_product.settings[step]['statistics']: check.valid_multimodel_statistic(statistic) - attributes = _get_common_attributes(products) - attributes['dataset'] = attributes['alias'] = 'MultiModel{}'.format( - statistic.title().replace('.', '-')) - attributes['filename'] = get_statistic_output_file( - attributes, preproc_dir) + + common_attributes = _get_common_attributes(products) + + title = 'MultiModel{}'.format(statistic.title().replace('.', '-')) + common_attributes['dataset'] = common_attributes['alias'] = title + + filename = get_statistic_output_file(common_attributes, preproc_dir) + common_attributes['filename'] = filename + common_settings = _get_remaining_common_settings(step, order, products) - statistic_product = PreprocessorFile(attributes, common_settings) + statistic_product = PreprocessorFile(common_attributes, common_settings) + for product in products: settings = product.settings[step] if 'output_products' not in settings: From a3a5f3b30c5a1cdfb043043c9a93328da14ddb8d Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 3 Sep 2020 18:03:51 +0200 Subject: [PATCH 045/158] Identify problem with deepcopy --- esmvalcore/_recipe.py | 10 ++++++++-- esmvalcore/preprocessor/__init__.py | 8 ++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 559ffc0bf5..a3015f859e 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -676,7 +676,10 @@ def _update_ensemble_settings(products, order, preproc_dir): common_attributes['filename'] = filename common_settings = _get_remaining_common_settings(step, order, products) - statistic_product = PreprocessorFile(common_attributes, common_settings) + print(common_settings) # original memory location #0 + + statistic_product = PreprocessorFile(common_attributes, common_settings, avoid_deepcopy=True) + print(statistic_product.settings) # new memory location #1 for product in products: settings = product.settings[step] @@ -686,6 +689,9 @@ def _update_ensemble_settings(products, order, preproc_dir): settings['output_products'] = defaultdict(dict) settings['output_products'][identifier][statistic] = statistic_product + print(statistic_product.settings) # same memory location #1 + print() + # breakpoint(); exit() def _update_extract_shape(settings, config_user): @@ -794,7 +800,7 @@ def _get_preprocessor_products(variables, products.add(product) preproc_dir = config_user['preproc_dir'] - _update_statistic_settings(products, order, preproc_dir) + _update_statistic_settings(products, order, preproc_dir) # order important! _update_ensemble_settings(products, order, preproc_dir) for product in products: diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index dbe03b3258..ba8b8d9a4d 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -194,6 +194,7 @@ def _check_multi_model_settings(products): reference = None for product in products: settings = product.settings.get(step) + print(settings) if settings is None: continue elif reference is None: @@ -272,11 +273,14 @@ def get_step_blocks(steps, order): class PreprocessorFile(TrackedFile): """Preprocessor output file.""" - def __init__(self, attributes, settings, ancestors=None): + def __init__(self, attributes, settings, ancestors=None, avoid_deepcopy=None): super(PreprocessorFile, self).__init__(attributes['filename'], attributes, ancestors) - self.settings = copy.deepcopy(settings) + if not avoid_deepcopy: + self.settings = copy.deepcopy(settings) + else: + self.settings = copy.copy(settings) if 'save' not in self.settings: self.settings['save'] = {} self.settings['save']['filename'] = self.filename From e27954e2e1738561d3a171c9c89c9a4e2dde7430 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 3 Sep 2020 18:04:47 +0200 Subject: [PATCH 046/158] Remove debug statements --- esmvalcore/_recipe.py | 5 ----- esmvalcore/preprocessor/__init__.py | 1 - 2 files changed, 6 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index a3015f859e..28a072c95d 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -676,10 +676,8 @@ def _update_ensemble_settings(products, order, preproc_dir): common_attributes['filename'] = filename common_settings = _get_remaining_common_settings(step, order, products) - print(common_settings) # original memory location #0 statistic_product = PreprocessorFile(common_attributes, common_settings, avoid_deepcopy=True) - print(statistic_product.settings) # new memory location #1 for product in products: settings = product.settings[step] @@ -689,9 +687,6 @@ def _update_ensemble_settings(products, order, preproc_dir): settings['output_products'] = defaultdict(dict) settings['output_products'][identifier][statistic] = statistic_product - print(statistic_product.settings) # same memory location #1 - print() - # breakpoint(); exit() def _update_extract_shape(settings, config_user): diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index ba8b8d9a4d..58d358eaef 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -194,7 +194,6 @@ def _check_multi_model_settings(products): reference = None for product in products: settings = product.settings.get(step) - print(settings) if settings is None: continue elif reference is None: From dcf97f8b4f1b03648deaa8d3b1ddab2b7617154b Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 07:57:44 +0200 Subject: [PATCH 047/158] Refactor multi_model_statistics --- esmvalcore/preprocessor/_multimodel.py | 56 +++++++++++++------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index dbc9619e59..9cd28100b7 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -290,7 +290,7 @@ def _assemble_data(cubes, statistic, span='overlap'): return stats_cube -def multi_model_statistics(products, span, statistics, output_products=None): +def _multi_model_statistics(cubes, span, statistics): """ Compute multi-model statistics. @@ -308,16 +308,12 @@ def multi_model_statistics(products, span, statistics, output_products=None): Parameters ---------- - products: list - list of data products or cubes to be used in multimodel stat - computation; - cube attribute of product is the data cube for computing the stats. + cubes: list of cubes + list of cubes to be used in multimodel stat computation; span: str overlap or full; if overlap, statitsticss are computed on common time- span; if full, statistics are computed on full time spans, ignoring missing data. - output_products: dict - dictionary of output products. statistics: str statistical measure to be computed. Available options: mean, median, max, min, std, or pXX.YY (for percentile XX.YY; decimal part optional). @@ -334,15 +330,9 @@ def multi_model_statistics(products, span, statistics, output_products=None): If span is neither overlap nor full. """ logger.debug('Multimodel statistics: computing: %s', statistics) - if len(products) < 2: + if len(cubes) < 2: logger.warning("Single dataset in list: will not compute statistics.") - return products - if output_products: - cubes = [cube for product in products for cube in product.cubes] - statistic_products = set() - else: - cubes = products - statistic_products = {} + return cubes # Reset time coordinates and make cubes share the same calendar _unify_time_coordinates(cubes) @@ -354,7 +344,7 @@ def multi_model_statistics(products, span, statistics, output_products=None): if len(overlap) <= 1: logger.info("Time overlap between cubes is none or a single point." "check datasets: will not compute statistics.") - return products + return cubes logger.debug("Using common time overlap between " "datasets to compute statistics.") elif span == 'full': @@ -364,22 +354,30 @@ def multi_model_statistics(products, span, statistics, output_products=None): "Unexpected value for span {}, choose from 'overlap', 'full'". format(span)) + statistics_cubes = {} for statistic in statistics: # Compute statistic statistic_cube = _assemble_data(cubes, statistic, span) - - if output_products: - # Add to output product and log provenance - statistic_product = output_products[statistic] - statistic_product.cubes = [statistic_cube] - for product in products: - statistic_product.wasderivedfrom(product) - logger.info("Generated %s", statistic_product) - statistic_products.add(statistic_product) - else: - statistic_products[statistic] = statistic_cube - - return statistic_products + statistics_cubes[statistic] = statistic_cube + + return statistics_cubes + + +def multi_model_statistics(products, span, statistics, output_products): + """Wrap _multimodel_statistics to operate on products instead of cubes.""" + cubes = [cube for product in products for cube in product.cubes] + statistics_cubes = _multi_model_statistics(cubes, span, statistics) + + statistics_products = set() + for statistic, cube in statistics_cubes.items(): + # Add to output product and log provenance + statistics_product = output_products[statistic] + statistics_product.cubes = [cube] + for product in products: + statistics_product.wasderivedfrom(product) + logger.info("Generated %s", statistics_product) + statistics_products.add(statistics_product) + return statistics_products def ensemble_statistics(products, output_products, statistics: list): From e1005bd632f84bbccdb7215e045ff83770f5105d Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 08:20:12 +0200 Subject: [PATCH 048/158] Refactor ensemble_statistics as a 'grouped multi-model_statistics' --- esmvalcore/preprocessor/_multimodel.py | 43 +++++++++++++------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9cd28100b7..ef1649d04d 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -364,7 +364,7 @@ def _multi_model_statistics(cubes, span, statistics): def multi_model_statistics(products, span, statistics, output_products): - """Wrap _multimodel_statistics to operate on products instead of cubes.""" + """Apply _multimodel_statistics on products instead of cubes.""" cubes = [cube for product in products for cube in product.cubes] statistics_cubes = _multi_model_statistics(cubes, span, statistics) @@ -380,31 +380,32 @@ def multi_model_statistics(products, span, statistics, output_products): return statistics_products -def ensemble_statistics(products, output_products, statistics: list): - """Calculate ensemble statistics. - - Parameters - ---------- - statistics: list - List of operators to apply to the data ensemble - """ - product_dict = defaultdict(set) +def _ensemble_statistics(cubes, statistics: list): span = 'overlap' + return _multi_model_statistics(cubes, span, statistics) + + +def ensemble_statistics(products, output_products, statistics: list): + """Apply _ensemble_statistics on grouped products.""" + grouped_products = defaultdict(set) for product in products: identifier = '_'.join([product.attributes['project'], product.attributes['dataset'], ''.join(product.attributes['exp'])]) # TODO: clean this - product_dict[identifier].add(product) + grouped_products[identifier].add(product) + + statistics_products = set() + for identifier, products in grouped_products.items(): + cubes = [cube for product in products for cube in product.cubes] + statistics_cubes = _ensemble_statistics(cubes, statistics) + + for statistic, cube in statistics_cubes.items(): + statistics_product = output_products[identifier][statistic] + statistics_product.cubes = [cube] - statistic_products = set() - for identifier, ensemble_products in product_dict.items(): - statistic_product = multi_model_statistics( - ensemble_products, - span, - statistics, - output_products[identifier], - ) - statistic_products |= statistic_product + for product in products: + statistics_product.wasderivedfrom(product) + statistics_products.add(statistics_product) - return statistic_products + return statistics_products From af9076788acf87ac3f5387f45d7ec74762969fd5 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 09:06:55 +0200 Subject: [PATCH 049/158] First attempt at dedicated ensemble means function using iris instead of mmstats --- esmvalcore/preprocessor/_multimodel.py | 52 +++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ef1649d04d..612ba2299e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -380,9 +380,57 @@ def multi_model_statistics(products, span, statistics, output_products): return statistics_products +def _ensemble_statistics_iris(cubes, statistics: list): + """Use iris merge/collapsed to perform the aggregation.""" + from iris.experimental.equalise_cubes import equalise_attributes + + operators = { + 'COUNT': iris.analysis.COUNT, + 'GMEAN': iris.analysis.GMEAN, + 'HMEAN': iris.analysis.HMEAN, + 'MAX': iris.analysis.MAX, + 'MEAN': iris.analysis.MEAN, + 'MEDIAN': iris.analysis.MEDIAN, + 'MIN': iris.analysis.MIN, + 'PEAK': iris.analysis.PEAK, + # 'PERCENTILE': iris.analysis.PERCENTILE, # requires additional args + # 'PROPORTION': iris.analysis.PROPORTION, # requires additional args + 'RMS': iris.analysis.RMS, + 'STD_DEV': iris.analysis.STD_DEV, + 'SUM': iris.analysis.SUM, + 'VARIANCE': iris.analysis.VARIANCE, + # 'WPERCENTILE': iris.analysis.WPERCENTILE, # requires additional args + } + + for i, cube in enumerate(cubes): + concat_dim = iris.coords.AuxCoord(i, var_name='ens') + cube.add_aux_coord(concat_dim) + + equalise_attributes(cubes) + + cubes = iris.cube.CubeList(cubes) + cube = cubes.merge_cube() + + statistics_cubes = {} + for statistic in statistics: + try: + operator = operators[statistic.upper()] + except KeyError: + logger.error( + 'Statistic %s not supported in ensemble_statistics. ' + 'Choose from %s', + statistic, operators.keys()) + + statistic_cube = cube.collapsed('ens', operator) + statistics_cubes[statistic] = statistic_cube + + return statistics_cubes + + def _ensemble_statistics(cubes, statistics: list): - span = 'overlap' - return _multi_model_statistics(cubes, span, statistics) + # span = 'overlap' + # return _multi_model_statistics(cubes, span, statistics) + return _ensemble_statistics_iris(cubes, statistics) def ensemble_statistics(products, output_products, statistics: list): From 8244c85dc44e86978a299bba5ce8c0c9ad79efe0 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 09:34:32 +0200 Subject: [PATCH 050/158] Lookup of supported operators in iris.analysis --- esmvalcore/preprocessor/_multimodel.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 612ba2299e..73720d3f94 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -383,24 +383,7 @@ def multi_model_statistics(products, span, statistics, output_products): def _ensemble_statistics_iris(cubes, statistics: list): """Use iris merge/collapsed to perform the aggregation.""" from iris.experimental.equalise_cubes import equalise_attributes - - operators = { - 'COUNT': iris.analysis.COUNT, - 'GMEAN': iris.analysis.GMEAN, - 'HMEAN': iris.analysis.HMEAN, - 'MAX': iris.analysis.MAX, - 'MEAN': iris.analysis.MEAN, - 'MEDIAN': iris.analysis.MEDIAN, - 'MIN': iris.analysis.MIN, - 'PEAK': iris.analysis.PEAK, - # 'PERCENTILE': iris.analysis.PERCENTILE, # requires additional args - # 'PROPORTION': iris.analysis.PROPORTION, # requires additional args - 'RMS': iris.analysis.RMS, - 'STD_DEV': iris.analysis.STD_DEV, - 'SUM': iris.analysis.SUM, - 'VARIANCE': iris.analysis.VARIANCE, - # 'WPERCENTILE': iris.analysis.WPERCENTILE, # requires additional args - } + operators = vars(iris.analysis) for i, cube in enumerate(cubes): concat_dim = iris.coords.AuxCoord(i, var_name='ens') @@ -414,12 +397,12 @@ def _ensemble_statistics_iris(cubes, statistics: list): statistics_cubes = {} for statistic in statistics: try: - operator = operators[statistic.upper()] + operator = operators.get(statistic.upper()) except KeyError: logger.error( 'Statistic %s not supported in ensemble_statistics. ' - 'Choose from %s', - statistic, operators.keys()) + 'Choose supported operator from iris.analysis package.', + statistic) statistic_cube = cube.collapsed('ens', operator) statistics_cubes[statistic] = statistic_cube From d78c34e3a5333c51da7eb50dbd34ce2488582610 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 11:20:16 +0200 Subject: [PATCH 051/158] Refactor (grouped) multimodel statistics flow --- esmvalcore/preprocessor/_multimodel.py | 116 ++++++++++++++++--------- 1 file changed, 76 insertions(+), 40 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 73720d3f94..c53c7c8b89 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -12,8 +12,10 @@ """ +import itertools import logging import re +from collections import defaultdict from datetime import datetime from functools import partial, reduce @@ -22,10 +24,6 @@ import numpy as np import scipy -import itertools - -from collections import defaultdict - logger = logging.getLogger(__name__) @@ -290,7 +288,7 @@ def _assemble_data(cubes, statistic, span='overlap'): return stats_cube -def _multi_model_statistics(cubes, span, statistics): +def _multicube_statistics(cubes, span, statistics): """ Compute multi-model statistics. @@ -363,25 +361,14 @@ def _multi_model_statistics(cubes, span, statistics): return statistics_cubes -def multi_model_statistics(products, span, statistics, output_products): - """Apply _multimodel_statistics on products instead of cubes.""" - cubes = [cube for product in products for cube in product.cubes] - statistics_cubes = _multi_model_statistics(cubes, span, statistics) +def _multicube_statistics_iris(cubes, statistics: list): + """Use iris merge/collapsed to perform the aggregation. - statistics_products = set() - for statistic, cube in statistics_cubes.items(): - # Add to output product and log provenance - statistics_product = output_products[statistic] - statistics_product.cubes = [cube] - for product in products: - statistics_product.wasderivedfrom(product) - logger.info("Generated %s", statistics_product) - statistics_products.add(statistics_product) - return statistics_products - - -def _ensemble_statistics_iris(cubes, statistics: list): - """Use iris merge/collapsed to perform the aggregation.""" + Equivalent to _multicube_statistics, but uses iris functions + to perform the aggregation. This only works if the input + cubes are very homogeneous, e.g. for different ensemble members + of the same model/dataset. + """ from iris.experimental.equalise_cubes import equalise_attributes operators = vars(iris.analysis) @@ -410,33 +397,82 @@ def _ensemble_statistics_iris(cubes, statistics: list): return statistics_cubes -def _ensemble_statistics(cubes, statistics: list): - # span = 'overlap' - # return _multi_model_statistics(cubes, span, statistics) - return _ensemble_statistics_iris(cubes, statistics) +def _compute_statistics(products, + statistics, + output_products, + span=None, + use_iris=False): + """Compute statistics on grouped products, using iris or esmvalcore functions.""" + if use_iris: + aggregator = _multicube_statistics_iris + else: + aggregator = partial(_multicube_statistics, span=span) + # Extract cubes from products and compute statistics + cubes = [cube for product in products for cube in product.cubes] + statistics_cubes = aggregator(cubes, span, statistics) -def ensemble_statistics(products, output_products, statistics: list): - """Apply _ensemble_statistics on grouped products.""" + # Add statistics to output_products + statistics_products = set() + for statistic, cube in statistics_cubes.items(): + # Add to output product and log provenance + statistics_product = output_products[statistic] + statistics_product.cubes = [cube] + for product in products: + statistics_product.wasderivedfrom(product) + logger.info("Generated %s", statistics_product) + statistics_products.add(statistics_product) + + return statistics_products + + +def _group(products, groupby=None): + """Group products.""" grouped_products = defaultdict(set) for product in products: - identifier = '_'.join([product.attributes['project'], - product.attributes['dataset'], - ''.join(product.attributes['exp'])]) # TODO: clean this + if groupby == 'ensemble': + identifier = '_'.join([ + product.attributes['project'], + product.attributes['dataset'], + *product.attributes['exp'] + ]) grouped_products[identifier].add(product) + return grouped_products + + +def _compute_grouped_statistics(products, + output_products, + statistics: list, + groupby, + use_iris=False): + """Apply _compute_statistics on grouped products.""" + grouped_products = _group(products, groupby) statistics_products = set() for identifier, products in grouped_products.items(): - cubes = [cube for product in products for cube in product.cubes] - statistics_cubes = _ensemble_statistics(cubes, statistics) + sub_output_products = output_products[identifier] - for statistic, cube in statistics_cubes.items(): - statistics_product = output_products[identifier][statistic] - statistics_product.cubes = [cube] + statistics_product = _compute_statistic(products, + sub_output_products, + statistics, + use_iris=use_iris) - for product in products: - statistics_product.wasderivedfrom(product) - statistics_products.add(statistics_product) + statistics_products.add(statistics_product) return statistics_products + + +def multi_model_statistics(products, span, statistics, output_products): + return _compute_statistics(products, + statistics, + output_products, + span=span) + + +def ensemble_statistics(products, statistics, output_products): + return _compute_grouped_statistics(products, + statistics, + output_products, + groupby='ensemble', + use_iris=True) From cc2ebe0e78d83e59cff5f7fd5632b7987033b5c8 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 12:00:22 +0200 Subject: [PATCH 052/158] Implement group function on PreprocessorFile --- esmvalcore/_recipe.py | 10 ++-------- esmvalcore/preprocessor/__init__.py | 12 ++++++++++++ esmvalcore/preprocessor/_multimodel.py | 6 +----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 9f9b47b596..069f2609cc 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -657,14 +657,8 @@ def _update_ensemble_settings(products, order, preproc_dir): grouped_products_dict = defaultdict(set) for product in products: - try: - identifier = '_'.join([product.attributes['project'], - product.attributes['dataset'], - ''.join(product.attributes['exp'])]) # TODO: clean this - except KeyError: - continue - else: - grouped_products_dict[identifier].add(product) + identifier = product.group('project', 'dataset', 'exp') + grouped_products_dict[identifier].add(product) for identifier, grouped_products in grouped_products_dict.items(): diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 58d358eaef..06db3d595a 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -354,6 +354,18 @@ def _initialize_entity(self): } self.entity.add_attributes(settings) + def group(self, *keys: str): + from collections.abc import Iterable + + identifier = [] + for key in keys: + attribute = self.attributes[key] + if isinstance(attribute, Iterable): + '-'.join(attribute) + identifier.append(attribute) + + return '_'.join(identifier) + # TODO: use a custom ProductSet that raises an exception if you try to # add the same Product twice diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index c53c7c8b89..ae7b3e14f4 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -431,11 +431,7 @@ def _group(products, groupby=None): grouped_products = defaultdict(set) for product in products: if groupby == 'ensemble': - identifier = '_'.join([ - product.attributes['project'], - product.attributes['dataset'], - *product.attributes['exp'] - ]) + identifier = product.group('project', 'dataset', 'exp') grouped_products[identifier].add(product) From 8a2b628d245ebb89d241ca236269e75506bf472b Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 12:05:50 +0200 Subject: [PATCH 053/158] Fix and clean up refactored _multimodel.py --- esmvalcore/preprocessor/_multimodel.py | 70 ++++++++++++++------------ 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index c53c7c8b89..cd03962a8c 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -288,7 +288,7 @@ def _assemble_data(cubes, statistic, span='overlap'): return stats_cube -def _multicube_statistics(cubes, span, statistics): +def _multicube_statistics(cubes, statistics, span): """ Compute multi-model statistics. @@ -397,12 +397,12 @@ def _multicube_statistics_iris(cubes, statistics: list): return statistics_cubes -def _compute_statistics(products, - statistics, - output_products, - span=None, - use_iris=False): - """Compute statistics on grouped products, using iris or esmvalcore functions.""" +def _multiproduct_statistics(products, + statistics, + output_products, + span=None, + use_iris=False): + """Compute statistics on (grouped) products, using iris or esmvalcore functions.""" if use_iris: aggregator = _multicube_statistics_iris else: @@ -410,7 +410,7 @@ def _compute_statistics(products, # Extract cubes from products and compute statistics cubes = [cube for product in products for cube in product.cubes] - statistics_cubes = aggregator(cubes, span, statistics) + statistics_cubes = aggregator(cubes=cubes, statistics=statistics) # Add statistics to output_products statistics_products = set() @@ -432,9 +432,8 @@ def _group(products, groupby=None): for product in products: if groupby == 'ensemble': identifier = '_'.join([ - product.attributes['project'], - product.attributes['dataset'], - *product.attributes['exp'] + product.attributes['project'], product.attributes['dataset'], + ''.join(product.attributes['exp']) ]) grouped_products[identifier].add(product) @@ -442,37 +441,44 @@ def _group(products, groupby=None): return grouped_products -def _compute_grouped_statistics(products, - output_products, - statistics: list, - groupby, - use_iris=False): - """Apply _compute_statistics on grouped products.""" - grouped_products = _group(products, groupby) +def _grouped_multiproduct_statistics(products, + statistics: list, + output_products, + groupby, + use_iris=False): + """Apply _multiproduct_statistics on grouped products.""" + grouped_products = _group(products, groupby=groupby) statistics_products = set() + # breakpoint() for identifier, products in grouped_products.items(): sub_output_products = output_products[identifier] - statistics_product = _compute_statistic(products, - sub_output_products, - statistics, - use_iris=use_iris) + statistics_product = _multiproduct_statistics( + products=products, + statistics=statistics, + output_products=sub_output_products, + use_iris=use_iris, + ) - statistics_products.add(statistics_product) + statistics_products |= statistics_product return statistics_products def multi_model_statistics(products, span, statistics, output_products): - return _compute_statistics(products, - statistics, - output_products, - span=span) + return _multiproduct_statistics( + products=products, + statistics=statistics, + output_products=output_products, + span=span, + ) def ensemble_statistics(products, statistics, output_products): - return _compute_grouped_statistics(products, - statistics, - output_products, - groupby='ensemble', - use_iris=True) + return _grouped_multiproduct_statistics( + products=products, + statistics=statistics, + output_products=output_products, + groupby='ensemble', + use_iris=True, + ) From cc5f61cd7f4367cba063ca06d9cf6f192fa395ce Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 12:15:18 +0200 Subject: [PATCH 054/158] Move group keywords to ensemble_stats function --- esmvalcore/preprocessor/_multimodel.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 108be13132..3d086fb8ec 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -430,8 +430,7 @@ def _group(products, groupby=None): """Group products.""" grouped_products = defaultdict(set) for product in products: - if groupby == 'ensemble': - identifier = product.group('project', 'dataset', 'exp') + identifier = product.group(groupby) grouped_products[identifier].add(product) @@ -446,7 +445,6 @@ def _grouped_multiproduct_statistics(products, """Apply _multiproduct_statistics on grouped products.""" grouped_products = _group(products, groupby=groupby) statistics_products = set() - # breakpoint() for identifier, products in grouped_products.items(): sub_output_products = output_products[identifier] @@ -472,10 +470,11 @@ def multi_model_statistics(products, span, statistics, output_products): def ensemble_statistics(products, statistics, output_products): + ensemble_grouping = ['project', 'dataset', 'exp'] return _grouped_multiproduct_statistics( products=products, statistics=statistics, output_products=output_products, - groupby='ensemble', + groupby=ensemble_grouping, use_iris=True, ) From 4b3114730e2f50f2df12715b3c69c6e242cfaeef Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 12:17:34 +0200 Subject: [PATCH 055/158] Tweak group method --- esmvalcore/_recipe.py | 3 ++- esmvalcore/preprocessor/__init__.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 069f2609cc..dd7180ffe3 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -651,13 +651,14 @@ def _update_statistic_settings(products, order, preproc_dir): def _update_ensemble_settings(products, order, preproc_dir): step = 'ensemble_statistics' + ensemble_grouping = 'project', 'dataset', 'exp' products = {product for product in products if step in product.settings} if not products: return grouped_products_dict = defaultdict(set) for product in products: - identifier = product.group('project', 'dataset', 'exp') + identifier = product.group(ensemble_grouping) grouped_products_dict[identifier].add(product) for identifier, grouped_products in grouped_products_dict.items(): diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 06db3d595a..2768669c03 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -354,9 +354,12 @@ def _initialize_entity(self): } self.entity.add_attributes(settings) - def group(self, *keys: str): + def group(self, keys: list): from collections.abc import Iterable + if isinstance(keys, str): + keys = [keys] + identifier = [] for key in keys: attribute = self.attributes[key] From 6523e05a790e9571495c273d14cb8cb533266280 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 13:56:34 +0200 Subject: [PATCH 056/158] Rename _update_statistic_settings -> _update_multi_model_settings --- esmvalcore/_recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index dd7180ffe3..7d716a1df1 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -617,7 +617,7 @@ def _update_multi_dataset_settings(variable, settings): _exclude_dataset(settings, variable, step) -def _update_statistic_settings(products, order, preproc_dir): +def _update_multi_model_settings(products, order, preproc_dir): """Define statistic output products.""" # TODO: move this to multi model statistics function? # But how to check, with a dry-run option? @@ -795,7 +795,7 @@ def _get_preprocessor_products(variables, products.add(product) preproc_dir = config_user['preproc_dir'] - _update_statistic_settings(products, order, preproc_dir) # order important! + _update_multi_model_settings(products, order, preproc_dir) # order important! _update_ensemble_settings(products, order, preproc_dir) for product in products: From 21989a40a25b43b5f897957e227f44e5800aa92a Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 13:57:02 +0200 Subject: [PATCH 057/158] Refactor product grouping (groupby func) --- esmvalcore/_recipe.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 7d716a1df1..320b3e52cc 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -649,6 +649,15 @@ def _update_multi_model_settings(products, order, preproc_dir): settings['output_products'][statistic] = statistic_product +def groupby(iterable, keyfunc: callable) -> dict: + grouped = defaultdict(set) + for item in iterable: + key = keyfunc(item) + grouped[key].add(item) + + return grouped + + def _update_ensemble_settings(products, order, preproc_dir): step = 'ensemble_statistics' ensemble_grouping = 'project', 'dataset', 'exp' @@ -656,10 +665,7 @@ def _update_ensemble_settings(products, order, preproc_dir): if not products: return - grouped_products_dict = defaultdict(set) - for product in products: - identifier = product.group(ensemble_grouping) - grouped_products_dict[identifier].add(product) + grouped_products_dict = groupby(products, keyfunc=lambda p: p.group(ensemble_grouping)) for identifier, grouped_products in grouped_products_dict.items(): From 73d5afd3907976e2a80a44040bc1800fa507b2a5 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 14:36:17 +0200 Subject: [PATCH 058/158] Homogenize multi_model/ensemble recipe functions --- esmvalcore/_recipe.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 320b3e52cc..ef936ea232 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -619,34 +619,37 @@ def _update_multi_dataset_settings(variable, settings): def _update_multi_model_settings(products, order, preproc_dir): """Define statistic output products.""" - # TODO: move this to multi model statistics function? - # But how to check, with a dry-run option? step = 'multi_model_statistics' products = {p for p in products if step in p.settings} if not products: return + identifier = 'multi_model' + some_product = next(iter(products)) for statistic in some_product.settings[step]['statistics']: check.valid_multimodel_statistic(statistic) common_attributes = _get_common_attributes(products) - title = 'MultiModel{}'.format(statistic.title().replace('.', '-')) + statistic_str = statistic.title().replace('.', '-') + title = f'MultiModel_{statistic_str}' common_attributes['dataset'] = common_attributes['alias'] = title filename = get_statistic_output_file(common_attributes, preproc_dir) common_attributes['filename'] = filename common_settings = _get_remaining_common_settings(step, order, products) + statistic_product = PreprocessorFile(common_attributes, common_settings) for product in products: settings = product.settings[step] if 'output_products' not in settings: - settings['output_products'] = {} - settings['output_products'][statistic] = statistic_product + settings['output_products'] = defaultdict(dict) + settings['output_products'][identifier][statistic] = statistic_product + settings['groupby'] = None def groupby(iterable, keyfunc: callable) -> dict: @@ -660,8 +663,8 @@ def groupby(iterable, keyfunc: callable) -> dict: def _update_ensemble_settings(products, order, preproc_dir): step = 'ensemble_statistics' - ensemble_grouping = 'project', 'dataset', 'exp' - products = {product for product in products if step in product.settings} + ensemble_grouping = ('project', 'dataset', 'exp') + products = {p for p in products if step in p.settings} if not products: return @@ -675,8 +678,9 @@ def _update_ensemble_settings(products, order, preproc_dir): for statistic in statistics: common_attributes = _get_common_attributes(products) - title = statistic.title() - common_attributes['dataset'] = f'{identifier}_Ensemble{title}' + statistic_str = statistic.title().replace('.', '-') + title = f'Ensemble_{identifier}_{statistic_str}' + common_attributes['dataset'] = common_attributes['alias'] = title filename = get_statistic_output_file(common_attributes, preproc_dir) common_attributes['filename'] = filename @@ -693,6 +697,7 @@ def _update_ensemble_settings(products, order, preproc_dir): settings['output_products'] = defaultdict(dict) settings['output_products'][identifier][statistic] = statistic_product + settings['groupby'] = ensemble_grouping def _update_extract_shape(settings, config_user): From 42a52b19780609d58c11ee87f3661bbae1a76d55 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 14:55:25 +0200 Subject: [PATCH 059/158] Make multimodel-statistics accept a groupby keyword as well --- esmvalcore/preprocessor/__init__.py | 71 +++++++++++++------------- esmvalcore/preprocessor/_multimodel.py | 19 +++++-- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 2768669c03..72093522b1 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -136,9 +136,7 @@ FINAL_STEPS = DEFAULT_ORDER[DEFAULT_ORDER.index('save'):] MULTI_MODEL_FUNCTIONS = { - 'multi_model_statistics', - 'mask_fillvalues', - 'ensemble_statistics' + 'multi_model_statistics', 'mask_fillvalues', 'ensemble_statistics' } @@ -201,9 +199,10 @@ def _check_multi_model_settings(products): elif reference.settings[step] != settings: raise ValueError( "Unable to combine differing multi-dataset settings for " - "{} and {}, {} and {}".format( - reference.filename, product.filename, - reference.settings[step], settings)) + "{} and {}, {} and {}".format(reference.filename, + product.filename, + reference.settings[step], + settings)) def _get_multi_model_settings(products, step): @@ -245,8 +244,7 @@ def preprocess(items, step, **settings): items = [] for item in result: - if isinstance(item, - (PreprocessorFile, Cube, str)): + if isinstance(item, (PreprocessorFile, Cube, str)): items.append(item) else: items.extend(item) @@ -271,8 +269,11 @@ def get_step_blocks(steps, order): class PreprocessorFile(TrackedFile): """Preprocessor output file.""" - - def __init__(self, attributes, settings, ancestors=None, avoid_deepcopy=None): + def __init__(self, + attributes, + settings, + ancestors=None, + avoid_deepcopy=None): super(PreprocessorFile, self).__init__(attributes['filename'], attributes, ancestors) @@ -378,8 +379,8 @@ def _apply_multimodel(products, step, debug): """Apply multi model step to products.""" settings, exclude = _get_multi_model_settings(products, step) - logger.debug("Applying %s to\n%s", step, '\n'.join( - str(p) for p in products - exclude)) + logger.debug("Applying %s to\n%s", step, + '\n'.join(str(p) for p in products - exclude)) result = preprocess(products - exclude, step, **settings) products = set(result) | exclude @@ -395,15 +396,14 @@ def _apply_multimodel(products, step, debug): class PreprocessingTask(BaseTask): """Task for running the preprocessor""" - def __init__( - self, - products, - ancestors=None, - name='', - order=DEFAULT_ORDER, - debug=None, - write_ncl_interface=False, + self, + products, + ancestors=None, + name='', + order=DEFAULT_ORDER, + debug=None, + write_ncl_interface=False, ): """Initialize.""" _check_multi_model_settings(products) @@ -418,34 +418,35 @@ def _initialize_product_provenance(self): self._initialize_multi_model_statistics_provenance() self._initialize_ensemble_statistics_provenance() - def _initialize_multi_model_statistics_provenance(self): - """Initialize provenance for multi-model statistics.""" - step = 'multi_model_statistics' - input_products = self._get_input_products(step) - if input_products: - statistic_products = input_products[0].settings[step].get( - 'output_products', {}).values() - - self._initialize_products(statistic_products) - - def _initialize_ensemble_statistics_provenance(self): - """Initialize provenance for ensemble statistics.""" - step = 'ensemble_statistics' + def _initialize_multiproduct_provenance(self, step): input_products = self._get_input_products(step) if input_products: statistic_products = set() for inputs in input_products: - items = inputs.settings[step].get('output_products', {}).items() + items = input_products[0].settings[step].get( + 'output_products', {}).items() for dataset, products in items: statistic_products.update(products.values()) self._initialize_products(statistic_products) + def _initialize_multi_model_statistics_provenance(self): + """Initialize provenance for multi-model statistics.""" + step = 'multi_model_statistics' + self._initialize_multiproduct_provenance(step) + + def _initialize_ensemble_statistics_provenance(self): + """Initialize provenance for ensemble statistics.""" + step = 'ensemble_statistics' + self._initialize_multiproduct_provenance(step) + def _get_input_products(self, step): """Get input products.""" - return [product for product in self.products if step in product.settings] + return [ + product for product in self.products if step in product.settings + ] def _initialize_products(self, products): """Initialize products.""" diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 3d086fb8ec..bc20ed24f8 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -441,9 +441,14 @@ def _grouped_multiproduct_statistics(products, statistics: list, output_products, groupby, + span=None, use_iris=False): """Apply _multiproduct_statistics on grouped products.""" - grouped_products = _group(products, groupby=groupby) + if groupby is None: + grouped_products = {'multi_model': products} + else: + grouped_products = _group(products, groupby=groupby) + statistics_products = set() for identifier, products in grouped_products.items(): sub_output_products = output_products[identifier] @@ -452,6 +457,7 @@ def _grouped_multiproduct_statistics(products, products=products, statistics=statistics, output_products=sub_output_products, + span=span, use_iris=use_iris, ) @@ -460,16 +466,21 @@ def _grouped_multiproduct_statistics(products, return statistics_products -def multi_model_statistics(products, span, statistics, output_products): - return _multiproduct_statistics( +def multi_model_statistics(products, + statistics, + output_products, + span, + groupby=None): + return _grouped_multiproduct_statistics( products=products, statistics=statistics, output_products=output_products, + groupby=groupby, span=span, ) -def ensemble_statistics(products, statistics, output_products): +def ensemble_statistics(products, statistics, output_products, groupby=None): ensemble_grouping = ['project', 'dataset', 'exp'] return _grouped_multiproduct_statistics( products=products, From 62689bf324b91a6bd096dfc511f19d5507a86639 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 14:58:25 +0200 Subject: [PATCH 060/158] Refactor multi model/ensemble settings --- esmvalcore/_recipe.py | 48 ++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index ef936ea232..47a0cd9573 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -617,6 +617,15 @@ def _update_multi_dataset_settings(variable, settings): _exclude_dataset(settings, variable, step) +def groupby(iterable, keyfunc: callable) -> dict: + grouped = defaultdict(set) + for item in iterable: + key = keyfunc(item) + grouped[key].add(item) + + return grouped + + def _update_multi_model_settings(products, order, preproc_dir): """Define statistic output products.""" step = 'multi_model_statistics' @@ -652,23 +661,17 @@ def _update_multi_model_settings(products, order, preproc_dir): settings['groupby'] = None -def groupby(iterable, keyfunc: callable) -> dict: - grouped = defaultdict(set) - for item in iterable: - key = keyfunc(item) - grouped[key].add(item) - - return grouped - - -def _update_ensemble_settings(products, order, preproc_dir): - step = 'ensemble_statistics' - ensemble_grouping = ('project', 'dataset', 'exp') +def _update_multi_product_settings(products, order, preproc_dir, step, grouping): + # TODO: avoid deep copy? + # TODO: title -> identifier.title()? products = {p for p in products if step in p.settings} if not products: return - grouped_products_dict = groupby(products, keyfunc=lambda p: p.group(ensemble_grouping)) + if grouping: + grouped_products_dict = groupby(products, keyfunc=lambda p: p.group(grouping)) + else: + grouped_products_dict = {'multi_model': products} for identifier, grouped_products in grouped_products_dict.items(): @@ -678,8 +681,8 @@ def _update_ensemble_settings(products, order, preproc_dir): for statistic in statistics: common_attributes = _get_common_attributes(products) - statistic_str = statistic.title().replace('.', '-') - title = f'Ensemble_{identifier}_{statistic_str}' + statistic_str = statistic.replace('.', '-') + title = f'{identifier}_{statistic_str}' common_attributes['dataset'] = common_attributes['alias'] = title filename = get_statistic_output_file(common_attributes, preproc_dir) @@ -697,7 +700,20 @@ def _update_ensemble_settings(products, order, preproc_dir): settings['output_products'] = defaultdict(dict) settings['output_products'][identifier][statistic] = statistic_product - settings['groupby'] = ensemble_grouping + settings['groupby'] = grouping + + +def _update_ensemble_settings(products, order, preproc_dir): + step = 'ensemble_statistics' + ensemble_grouping = ('project', 'dataset', 'exp') + + _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping) + +def _update_multi_model_settings(products, order, preproc_dir): + step = 'multi_model_statistics' + grouping = None + + _update_multi_product_settings(products, order, preproc_dir, step, grouping=None) def _update_extract_shape(settings, config_user): From aee20ea31666671cc64eaa388492e36f34128802 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 14:59:01 +0200 Subject: [PATCH 061/158] Return default value for None argument --- esmvalcore/preprocessor/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 72093522b1..0c148f61d2 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -358,6 +358,9 @@ def _initialize_entity(self): def group(self, keys: list): from collections.abc import Iterable + if not keys: + return '' + if isinstance(keys, str): keys = [keys] From 559fcf3e1240408b29f3f1ff6bd66e98f8450a7a Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 15:00:40 +0200 Subject: [PATCH 062/158] Remove redundant function --- esmvalcore/_recipe.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 47a0cd9573..e406a20c0a 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -626,41 +626,6 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def _update_multi_model_settings(products, order, preproc_dir): - """Define statistic output products.""" - step = 'multi_model_statistics' - - products = {p for p in products if step in p.settings} - if not products: - return - - identifier = 'multi_model' - - some_product = next(iter(products)) - for statistic in some_product.settings[step]['statistics']: - check.valid_multimodel_statistic(statistic) - - common_attributes = _get_common_attributes(products) - - statistic_str = statistic.title().replace('.', '-') - title = f'MultiModel_{statistic_str}' - common_attributes['dataset'] = common_attributes['alias'] = title - - filename = get_statistic_output_file(common_attributes, preproc_dir) - common_attributes['filename'] = filename - - common_settings = _get_remaining_common_settings(step, order, products) - - statistic_product = PreprocessorFile(common_attributes, common_settings) - - for product in products: - settings = product.settings[step] - if 'output_products' not in settings: - settings['output_products'] = defaultdict(dict) - settings['output_products'][identifier][statistic] = statistic_product - settings['groupby'] = None - - def _update_multi_product_settings(products, order, preproc_dir, step, grouping): # TODO: avoid deep copy? # TODO: title -> identifier.title()? @@ -713,7 +678,7 @@ def _update_multi_model_settings(products, order, preproc_dir): step = 'multi_model_statistics' grouping = None - _update_multi_product_settings(products, order, preproc_dir, step, grouping=None) + _update_multi_product_settings(products, order, preproc_dir, step, grouping=gruping) def _update_extract_shape(settings, config_user): From 5689953a22ff7dc77b96b84d295ab4bb96ee9607 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 15:04:49 +0200 Subject: [PATCH 063/158] Fix typo --- esmvalcore/_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index e406a20c0a..591075ceb3 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -678,7 +678,7 @@ def _update_multi_model_settings(products, order, preproc_dir): step = 'multi_model_statistics' grouping = None - _update_multi_product_settings(products, order, preproc_dir, step, grouping=gruping) + _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping) def _update_extract_shape(settings, config_user): From 9d20217738f12eba86b3950b319b05d1ee915644 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 4 Sep 2020 15:06:44 +0200 Subject: [PATCH 064/158] Use groupby keyword from product settings for ensemble_statistics --- esmvalcore/preprocessor/_multimodel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index bc20ed24f8..5bc0cc6b43 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -471,6 +471,7 @@ def multi_model_statistics(products, output_products, span, groupby=None): + """Apply multimodel statistics to esmvalcore preprocessor products.""" return _grouped_multiproduct_statistics( products=products, statistics=statistics, @@ -481,11 +482,11 @@ def multi_model_statistics(products, def ensemble_statistics(products, statistics, output_products, groupby=None): - ensemble_grouping = ['project', 'dataset', 'exp'] + """Apply ensemble statistics to esmvalcore preprocessor products.""" return _grouped_multiproduct_statistics( products=products, statistics=statistics, output_products=output_products, - groupby=ensemble_grouping, + groupby=groupby, use_iris=True, ) From 74c55f32bb00e2adcf67d31e9ff46180f0d89cb4 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 4 Sep 2020 15:23:21 +0200 Subject: [PATCH 065/158] Add docstrings --- esmvalcore/_recipe.py | 6 +++++- esmvalcore/preprocessor/__init__.py | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 591075ceb3..f83f310407 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -618,6 +618,7 @@ def _update_multi_dataset_settings(variable, settings): def groupby(iterable, keyfunc: callable) -> dict: + """Group iterable by key function.""" grouped = defaultdict(set) for item in iterable: key = keyfunc(item) @@ -626,7 +627,8 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def _update_multi_product_settings(products, order, preproc_dir, step, grouping): +def _update_multi_product_settings(products, order, preproc_dir, step, grouping=None): + """Define output settings for generic multi-product products.""" # TODO: avoid deep copy? # TODO: title -> identifier.title()? products = {p for p in products if step in p.settings} @@ -669,12 +671,14 @@ def _update_multi_product_settings(products, order, preproc_dir, step, grouping) def _update_ensemble_settings(products, order, preproc_dir): + """Define output settings for ensemble products.""" step = 'ensemble_statistics' ensemble_grouping = ('project', 'dataset', 'exp') _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping) def _update_multi_model_settings(products, order, preproc_dir): + """Define output settings for multi model products.""" step = 'multi_model_statistics' grouping = None diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 0c148f61d2..212cce88f9 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -355,7 +355,12 @@ def _initialize_entity(self): } self.entity.add_attributes(settings) - def group(self, keys: list): + def group(self, keys: list) -> str: + """Generate group keyword. + + Returns a string that identifies a group. + Concatenates a list of values from .attributes + """ from collections.abc import Iterable if not keys: From e0f3c5a62f8e11e256af48ec01fdbe53ecaebd82 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 7 Sep 2020 13:30:44 +0200 Subject: [PATCH 066/158] idea for bypassing update order issue --- esmvalcore/_recipe.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index f83f310407..f0cb0f78ad 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -627,19 +627,21 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def _update_multi_product_settings(products, order, preproc_dir, step, grouping=None): +def _update_multi_product_settings(input_products, order, preproc_dir, step, grouping=None): """Define output settings for generic multi-product products.""" # TODO: avoid deep copy? # TODO: title -> identifier.title()? - products = {p for p in products if step in p.settings} + + products = {p for p in input_products if step in p.settings} if not products: - return + return input_products if grouping: grouped_products_dict = groupby(products, keyfunc=lambda p: p.group(grouping)) else: grouped_products_dict = {'multi_model': products} + output_products = set() for identifier, grouped_products in grouped_products_dict.items(): some_product = next(iter(grouped_products)) @@ -659,6 +661,8 @@ def _update_multi_product_settings(products, order, preproc_dir, step, grouping= statistic_product = PreprocessorFile(common_attributes, common_settings, avoid_deepcopy=True) + output_products.add(statistic_product) + for product in products: settings = product.settings[step] @@ -669,20 +673,22 @@ def _update_multi_product_settings(products, order, preproc_dir, step, grouping= settings['output_products'][identifier][statistic] = statistic_product settings['groupby'] = grouping + return output_products + def _update_ensemble_settings(products, order, preproc_dir): """Define output settings for ensemble products.""" step = 'ensemble_statistics' ensemble_grouping = ('project', 'dataset', 'exp') - _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping) + return _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping,) def _update_multi_model_settings(products, order, preproc_dir): """Define output settings for multi model products.""" step = 'multi_model_statistics' grouping = None - _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping) + return _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping,) def _update_extract_shape(settings, config_user): @@ -791,8 +797,9 @@ def _get_preprocessor_products(variables, products.add(product) preproc_dir = config_user['preproc_dir'] - _update_multi_model_settings(products, order, preproc_dir) # order important! - _update_ensemble_settings(products, order, preproc_dir) + + products = _update_ensemble_settings(products, order, preproc_dir) + products = _update_multi_model_settings(products, order, preproc_dir) # order important! for product in products: product.check() From 5d7585c265078e08b1746b703ed1d0f4dcd93491 Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 7 Sep 2020 16:15:01 +0200 Subject: [PATCH 067/158] Fix duplicate filenames when multi-model/ensemble statistics are both used --- esmvalcore/_recipe.py | 58 ++++++++++++++++---------- esmvalcore/preprocessor/_multimodel.py | 2 +- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index f0cb0f78ad..786427ad68 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -627,22 +627,40 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def _update_multi_product_settings(input_products, order, preproc_dir, step, grouping=None): +def _patch_multi_model_filename(settings, prefix, tags, statistic_str): + """Patch multi-model file name. + + This is necessary to avoid overwriting existing files.""" + from pathlib import Path + multi_model_tag = tags['multi_model_statistics'] + for multi_model_product in settings['multi_model_statistics']['output_products'][multi_model_tag].values(): + p = Path(multi_model_product._filename) + name = p.name + if not name.startswith('Ensemble'): + fn = p.with_name(prefix + '_' + name) + multi_model_product.settings['save']['filename'] = str(fn) + multi_model_product._filename = str(fn) + + +def _update_multi_product_settings(products, order, preproc_dir, step, grouping=None): """Define output settings for generic multi-product products.""" # TODO: avoid deep copy? # TODO: title -> identifier.title()? - - products = {p for p in input_products if step in p.settings} + products = {p for p in products if step in p.settings} if not products: - return input_products + return - if grouping: - grouped_products_dict = groupby(products, keyfunc=lambda p: p.group(grouping)) - else: - grouped_products_dict = {'multi_model': products} + tags = { + 'multi_model_statistics': 'MultiModel', + 'ensemble_statistics': 'Ensemble' + } + tag = tags[step] + + grouped_products_dict = groupby(products, keyfunc=lambda p: p.group(grouping)) - output_products = set() for identifier, grouped_products in grouped_products_dict.items(): + if not identifier: + identifier = tag some_product = next(iter(grouped_products)) statistics = some_product.settings[step]['statistics'] @@ -651,18 +669,18 @@ def _update_multi_product_settings(input_products, order, preproc_dir, step, gro common_attributes = _get_common_attributes(products) statistic_str = statistic.replace('.', '-') - title = f'{identifier}_{statistic_str}' + title = f'{identifier}-{statistic_str}' common_attributes['dataset'] = common_attributes['alias'] = title filename = get_statistic_output_file(common_attributes, preproc_dir) common_attributes['filename'] = filename common_settings = _get_remaining_common_settings(step, order, products) + if 'multi_model_statistics' in common_settings: + _patch_multi_model_filename(common_settings, f'{tag}-{statistic_str}', tags, statistic_str) statistic_product = PreprocessorFile(common_attributes, common_settings, avoid_deepcopy=True) - output_products.add(statistic_product) - for product in products: settings = product.settings[step] @@ -673,22 +691,20 @@ def _update_multi_product_settings(input_products, order, preproc_dir, step, gro settings['output_products'][identifier][statistic] = statistic_product settings['groupby'] = grouping - return output_products - def _update_ensemble_settings(products, order, preproc_dir): """Define output settings for ensemble products.""" step = 'ensemble_statistics' ensemble_grouping = ('project', 'dataset', 'exp') - return _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping,) + _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping) def _update_multi_model_settings(products, order, preproc_dir): """Define output settings for multi model products.""" step = 'multi_model_statistics' grouping = None - return _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping,) + _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping) def _update_extract_shape(settings, config_user): @@ -748,9 +764,11 @@ def _get_preprocessor_products(variables, and sets the correct ancestry. """ products = set() + preproc_dir = config_user['preproc_dir'] + for variable in variables: variable['filename'] = get_output_file(variable, - config_user['preproc_dir']) + preproc_dir) if ancestor_products: grouped_ancestors = _match_products(ancestor_products, variables) @@ -796,10 +814,8 @@ def _get_preprocessor_products(variables, ) products.add(product) - preproc_dir = config_user['preproc_dir'] - - products = _update_ensemble_settings(products, order, preproc_dir) - products = _update_multi_model_settings(products, order, preproc_dir) # order important! + _update_multi_model_settings(products, order, preproc_dir) # order important! + _update_ensemble_settings(products, order, preproc_dir) for product in products: product.check() diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 5bc0cc6b43..bff3d6de7b 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -445,7 +445,7 @@ def _grouped_multiproduct_statistics(products, use_iris=False): """Apply _multiproduct_statistics on grouped products.""" if groupby is None: - grouped_products = {'multi_model': products} + grouped_products = {'MultiModel': products} else: grouped_products = _group(products, groupby=groupby) From a643241679e61f4453cb812d9c6dc788c3905e8d Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 7 Sep 2020 16:29:03 +0200 Subject: [PATCH 068/158] Remove unused argument and tweak codestyle --- esmvalcore/_recipe.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 786427ad68..cd3d65b5be 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -627,17 +627,17 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def _patch_multi_model_filename(settings, prefix, tags, statistic_str): +def _patch_multi_model_filename(settings, prefix, tags): """Patch multi-model file name. - This is necessary to avoid overwriting existing files.""" + This is necessary to avoid filename collisions.""" from pathlib import Path multi_model_tag = tags['multi_model_statistics'] for multi_model_product in settings['multi_model_statistics']['output_products'][multi_model_tag].values(): - p = Path(multi_model_product._filename) - name = p.name + path = Path(multi_model_product._filename) + name = path.name if not name.startswith('Ensemble'): - fn = p.with_name(prefix + '_' + name) + fn = path.with_name(prefix + '_' + name) multi_model_product.settings['save']['filename'] = str(fn) multi_model_product._filename = str(fn) @@ -669,7 +669,7 @@ def _update_multi_product_settings(products, order, preproc_dir, step, grouping= common_attributes = _get_common_attributes(products) statistic_str = statistic.replace('.', '-') - title = f'{identifier}-{statistic_str}' + title = f'{identifier}{statistic_str.title()}' common_attributes['dataset'] = common_attributes['alias'] = title filename = get_statistic_output_file(common_attributes, preproc_dir) @@ -677,7 +677,11 @@ def _update_multi_product_settings(products, order, preproc_dir, step, grouping= common_settings = _get_remaining_common_settings(step, order, products) if 'multi_model_statistics' in common_settings: - _patch_multi_model_filename(common_settings, f'{tag}-{statistic_str}', tags, statistic_str) + _patch_multi_model_filename( + common_settings, + f'{tag}{statistic_str.title()}', + tags, + ) statistic_product = PreprocessorFile(common_attributes, common_settings, avoid_deepcopy=True) From 339ec1e202b52e58fddcd96d7260e69d24512c43 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 09:11:55 +0200 Subject: [PATCH 069/158] Change order, improve ancestry updating, better filenaming --- esmvalcore/_data_finder.py | 7 +- esmvalcore/_recipe.py | 127 +++++++++++++++---------- esmvalcore/preprocessor/__init__.py | 8 +- esmvalcore/preprocessor/_multimodel.py | 20 ++-- 4 files changed, 92 insertions(+), 70 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index 7a27dee4f6..f0081cd653 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -270,6 +270,11 @@ def get_statistic_output_file(variable, preproc_dir): '{dataset}_{mip}_{short_name}_{start_year}-{end_year}.nc', ) - outfile = template.format(**variable) + # if ensemble in variable: + # template = ...{'ensemble'} + # if multimodel in variable: + # template = ... + + outfile = template.format(**variable) return outfile diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 786427ad68..35b0c4800e 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -596,8 +596,8 @@ def _get_common_attributes(products): return attributes -def _get_remaining_common_settings(step, order, products): - """Get preprocessor settings that are shared between products.""" +def _get_downstream_settings(step, order, products): + """Get downstream preprocessor settings that are shared between products.""" settings = {} remaining_steps = order[order.index(step) + 1:] some_product = next(iter(products)) @@ -627,28 +627,38 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def _patch_multi_model_filename(settings, prefix, tags, statistic_str): - """Patch multi-model file name. +def get_multi_output_file(attributes, preproc_dir): + """Get multi model statistic filename depending on settings.""" - This is necessary to avoid overwriting existing files.""" - from pathlib import Path - multi_model_tag = tags['multi_model_statistics'] - for multi_model_product in settings['multi_model_statistics']['output_products'][multi_model_tag].values(): - p = Path(multi_model_product._filename) - name = p.name - if not name.startswith('Ensemble'): - fn = p.with_name(prefix + '_' + name) - multi_model_product.settings['save']['filename'] = str(fn) - multi_model_product._filename = str(fn) + template = os.path.join( + preproc_dir, + '{attrs[diagnostic]}', + '{attrs[variable_group]}', + '{attrs[project]}_{attrs[dataset]}_{attrs[exp]}' + '{attrs[ensemble_statistics]}' + '{attrs[multi_model_statistics]}' + '_{attrs[mip]}_{attrs[short_name]}' + '_{attrs[start_year]}-{attrs[end_year]}.nc', + ) + + outfile = template.format(attrs=defaultdict(str, **attributes)) + return outfile + + +def _update_multi_product_settings(input_products, order, preproc_dir, step, grouping=None): + """Return new products that are aggregated over multiple datasets. + These new products will replace the original products at runtime. Therefore, they + need to have all the settings for the remaining steps. -def _update_multi_product_settings(products, order, preproc_dir, step, grouping=None): - """Define output settings for generic multi-product products.""" - # TODO: avoid deep copy? - # TODO: title -> identifier.title()? - products = {p for p in products if step in p.settings} + The functions in _multimodel.py take output_products as function arguments. These are + the output_products created here. But since those functions are called from the + input products, the products that are created here need to be added to their + ancestors products' settings (). + """ + products = {p for p in input_products if step in p.settings} if not products: - return + return input_products tags = { 'multi_model_statistics': 'MultiModel', @@ -656,55 +666,56 @@ def _update_multi_product_settings(products, order, preproc_dir, step, grouping= } tag = tags[step] - grouped_products_dict = groupby(products, keyfunc=lambda p: p.group(grouping)) + settings = list(products)[0].settings[step] + downstream_settings = _get_downstream_settings(step, order, products) - for identifier, grouped_products in grouped_products_dict.items(): - if not identifier: - identifier = tag + grouped_products = groupby(products, keyfunc=lambda p: p.group(grouping)) - some_product = next(iter(grouped_products)) - statistics = some_product.settings[step]['statistics'] + relevant_settings = {'output_products': defaultdict(dict)} # pass to ancestors - for statistic in statistics: - common_attributes = _get_common_attributes(products) + output_products = set() + for identifier, products in grouped_products.items(): + common_attributes = _get_common_attributes(products) - statistic_str = statistic.replace('.', '-') - title = f'{identifier}-{statistic_str}' - common_attributes['dataset'] = common_attributes['alias'] = title - - filename = get_statistic_output_file(common_attributes, preproc_dir) - common_attributes['filename'] = filename + for statistic in settings.get('statistics'): - common_settings = _get_remaining_common_settings(step, order, products) - if 'multi_model_statistics' in common_settings: - _patch_multi_model_filename(common_settings, f'{tag}-{statistic_str}', tags, statistic_str) + statistic_str = statistic.replace('.', '-') # avoid . in filename for percentiles + step_tag = f'{tag}{statistic_str.title()}' + common_attributes[step] = step_tag - statistic_product = PreprocessorFile(common_attributes, common_settings, avoid_deepcopy=True) + filename = get_multi_output_file(common_attributes, preproc_dir) + common_attributes['filename'] = filename - for product in products: - settings = product.settings[step] + statistic_product = PreprocessorFile(common_attributes, downstream_settings) + output_products.add(statistic_product) - if 'output_products' not in settings: - # assume output products is a nested dict - settings['output_products'] = defaultdict(dict) + relevant_settings['output_products'][identifier][statistic] = statistic_product - settings['output_products'][identifier][statistic] = statistic_product - settings['groupby'] = grouping + return output_products, relevant_settings -def _update_ensemble_settings(products, order, preproc_dir): +def _update_ensemble(products, order, preproc_dir): """Define output settings for ensemble products.""" step = 'ensemble_statistics' ensemble_grouping = ('project', 'dataset', 'exp') - _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping) + return _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping) + -def _update_multi_model_settings(products, order, preproc_dir): +def _update_multimodel(products, order, preproc_dir): """Define output settings for multi model products.""" step = 'multi_model_statistics' grouping = None - _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping) + return _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping) + + +def update_ancestors(ancestors, step, downstream_settings): + """Retroactively add settings to ancestor products.""" + for product in ancestors: + settings = product.settings[step] + for key, value in downstream_settings.items(): + settings[key] = value def _update_extract_shape(settings, config_user): @@ -814,12 +825,24 @@ def _get_preprocessor_products(variables, ) products.add(product) - _update_multi_model_settings(products, order, preproc_dir) # order important! - _update_ensemble_settings(products, order, preproc_dir) - for product in products: + ensemble_products, ensemble_settings = _update_ensemble(products, order, preproc_dir) + multimodel_products, multimodel_settings = _update_multimodel(ensemble_products, order, preproc_dir) + + # Update multi-product settings (workaround for lack of better ancestry tracking) + update_ancestors(ancestors=products, step='ensemble_statistics', downstream_settings=ensemble_settings,) + update_ancestors(ancestors=products, step='multi_model_statistics', downstream_settings=multimodel_settings,) + update_ancestors(ancestors=ensemble_products, step='multi_model_statistics', downstream_settings=multimodel_settings,) + + for product in products | ensemble_products | multimodel_products: product.check() + # TODO make groupby keyword work for multi-model statistics + # TODO correct naming of identifiers for grouped multimodel + # TODO fix (underscores in) filenames + # TODO get back the pretty printing of PreprocessorFile objects + # TODO check if get_statistics_filename in _data_finder.py can be removed/replaced by the function above (get_multi_output_file) + return products diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 212cce88f9..de51a38531 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -272,15 +272,11 @@ class PreprocessorFile(TrackedFile): def __init__(self, attributes, settings, - ancestors=None, - avoid_deepcopy=None): + ancestors=None,): super(PreprocessorFile, self).__init__(attributes['filename'], attributes, ancestors) - if not avoid_deepcopy: - self.settings = copy.deepcopy(settings) - else: - self.settings = copy.copy(settings) + self.settings = copy.deepcopy(settings) if 'save' not in self.settings: self.settings['save'] = {} self.settings['save']['filename'] = self.filename diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index bff3d6de7b..0fcecb692c 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -401,9 +401,9 @@ def _multiproduct_statistics(products, statistics, output_products, span=None, - use_iris=False): + engine='esmvalcore'): """Compute statistics on (grouped) products, using iris or esmvalcore functions.""" - if use_iris: + if engine == 'iris': aggregator = _multicube_statistics_iris else: aggregator = partial(_multicube_statistics, span=span) @@ -442,12 +442,9 @@ def _grouped_multiproduct_statistics(products, output_products, groupby, span=None, - use_iris=False): + engine='esmvalcore'): """Apply _multiproduct_statistics on grouped products.""" - if groupby is None: - grouped_products = {'MultiModel': products} - else: - grouped_products = _group(products, groupby=groupby) + grouped_products = _group(products, groupby=groupby) statistics_products = set() for identifier, products in grouped_products.items(): @@ -458,7 +455,7 @@ def _grouped_multiproduct_statistics(products, statistics=statistics, output_products=sub_output_products, span=span, - use_iris=use_iris, + engine=engine, ) statistics_products |= statistics_product @@ -481,12 +478,13 @@ def multi_model_statistics(products, ) -def ensemble_statistics(products, statistics, output_products, groupby=None): +def ensemble_statistics(products, statistics, output_products): """Apply ensemble statistics to esmvalcore preprocessor products.""" + ensemble_grouping = ('project', 'dataset', 'exp') return _grouped_multiproduct_statistics( products=products, statistics=statistics, output_products=output_products, - groupby=groupby, - use_iris=True, + groupby=ensemble_grouping, + engine='iris', ) From 85d0bb84dc81c1873801b9fff538fb8722500739 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 13:34:24 +0200 Subject: [PATCH 070/158] Fix output filenames and pretty print PreprocessorFile --- esmvalcore/_data_finder.py | 29 ++++++++++++++++------------- esmvalcore/_recipe.py | 25 ++----------------------- esmvalcore/preprocessor/__init__.py | 4 ++++ 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index f0081cd653..6101e9bb75 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -261,20 +261,23 @@ def get_output_file(variable, preproc_dir): return outfile -def get_statistic_output_file(variable, preproc_dir): - """Get multi model statistic filename depending on settings.""" - template = os.path.join( - preproc_dir, - '{diagnostic}', - '{variable_group}', - '{dataset}_{mip}_{short_name}_{start_year}-{end_year}.nc', - ) +def get_multiproduct_filename(attributes, preproc_dir): + """Get ensemble/multi-model filename depending on settings.""" + relevant_keys = ['project', 'dataset', 'exp', 'ensemble_statistics', + 'multi_model_statistics', 'mip', 'short_name'] + filename_segments = [] + for key in relevant_keys: + if key in attributes: + filename_segments.append(attributes[key]) - # if ensemble in variable: - # template = ...{'ensemble'} - # if multimodel in variable: - # template = ... + # Add period and extension + segments.append(f"{attributes['start_year']}-{attributes['end_year']}.nc") - outfile = template.format(**variable) + outfile = os.path.join( + preproc_dir, + attributes['diagnostic'], + attributes['variable_group'], + '_'.join(filename_segments), + ) return outfile diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 35b0c4800e..323e7c10c5 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -16,7 +16,7 @@ from ._config import (TAGS, get_activity, get_institutes, get_project_config, replace_tags) from ._data_finder import (get_input_filelist, get_output_file, - get_statistic_output_file) + get_multiproduct_filename) from ._provenance import TrackedFile, get_recipe_provenance from ._recipe_checks import RecipeError from ._task import (DiagnosticTask, get_flattened_tasks, get_independent_tasks, @@ -627,24 +627,6 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def get_multi_output_file(attributes, preproc_dir): - """Get multi model statistic filename depending on settings.""" - - template = os.path.join( - preproc_dir, - '{attrs[diagnostic]}', - '{attrs[variable_group]}', - '{attrs[project]}_{attrs[dataset]}_{attrs[exp]}' - '{attrs[ensemble_statistics]}' - '{attrs[multi_model_statistics]}' - '_{attrs[mip]}_{attrs[short_name]}' - '_{attrs[start_year]}-{attrs[end_year]}.nc', - ) - - outfile = template.format(attrs=defaultdict(str, **attributes)) - return outfile - - def _update_multi_product_settings(input_products, order, preproc_dir, step, grouping=None): """Return new products that are aggregated over multiple datasets. @@ -683,7 +665,7 @@ def _update_multi_product_settings(input_products, order, preproc_dir, step, gro step_tag = f'{tag}{statistic_str.title()}' common_attributes[step] = step_tag - filename = get_multi_output_file(common_attributes, preproc_dir) + filename = get_multiproduct_filename(common_attributes, preproc_dir) common_attributes['filename'] = filename statistic_product = PreprocessorFile(common_attributes, downstream_settings) @@ -839,9 +821,6 @@ def _get_preprocessor_products(variables, # TODO make groupby keyword work for multi-model statistics # TODO correct naming of identifiers for grouped multimodel - # TODO fix (underscores in) filenames - # TODO get back the pretty printing of PreprocessorFile objects - # TODO check if get_statistics_filename in _data_finder.py can be removed/replaced by the function above (get_multi_output_file) return products diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index de51a38531..187e873abe 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -286,6 +286,10 @@ def __init__(self, self._cubes = None self._prepared = False + def __repr__(self): + name = Path(self._filename).name + return f'{self.__class__.__name__}({repr(name)})' + def check(self): """Check preprocessor settings.""" check_preprocessor_settings(self.settings) From d8ff56aa2c5add452f49020f40a9a441c5226ee9 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 13:38:27 +0200 Subject: [PATCH 071/158] two typos --- esmvalcore/_data_finder.py | 2 +- esmvalcore/preprocessor/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index 6101e9bb75..d7d4575c2d 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -272,7 +272,7 @@ def get_multiproduct_filename(attributes, preproc_dir): filename_segments.append(attributes[key]) # Add period and extension - segments.append(f"{attributes['start_year']}-{attributes['end_year']}.nc") + filename_segments.append(f"{attributes['start_year']}-{attributes['end_year']}.nc") outfile = os.path.join( preproc_dir, diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 187e873abe..96dd6e134c 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -2,6 +2,7 @@ import copy import inspect import logging +from pathlib import Path from pprint import pformat from iris.cube import Cube From 09929cdb433797614be43fd88559a8c28736f46d Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 13:55:59 +0200 Subject: [PATCH 072/158] Make groupby argument work for multi-model statistics preprocessor --- esmvalcore/_recipe.py | 8 ++++---- esmvalcore/preprocessor/_multimodel.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 323e7c10c5..1b12b55370 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -679,17 +679,17 @@ def _update_multi_product_settings(input_products, order, preproc_dir, step, gro def _update_ensemble(products, order, preproc_dir): """Define output settings for ensemble products.""" step = 'ensemble_statistics' - ensemble_grouping = ('project', 'dataset', 'exp') + groupby = ('project', 'dataset', 'exp') - return _update_multi_product_settings(products, order, preproc_dir, step, grouping=ensemble_grouping) + return _update_multi_product_settings(products, order, preproc_dir, step, grouping=groupby) def _update_multimodel(products, order, preproc_dir): """Define output settings for multi model products.""" step = 'multi_model_statistics' - grouping = None + groupby = list(products)[0].settings[step].get('groupby', None) - return _update_multi_product_settings(products, order, preproc_dir, step, grouping=grouping) + return _update_multi_product_settings(products, order, preproc_dir, step, grouping=groupby) def update_ancestors(ancestors, step, downstream_settings): diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 0fcecb692c..9ac847612e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -444,6 +444,8 @@ def _grouped_multiproduct_statistics(products, span=None, engine='esmvalcore'): """Apply _multiproduct_statistics on grouped products.""" + breakpoint() + grouped_products = _group(products, groupby=groupby) statistics_products = set() From 1e8762a46a6c6726ffa28b5128fd2b5f348385af Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 13:58:13 +0200 Subject: [PATCH 073/158] Some code formatting --- esmvalcore/_recipe.py | 104 +++++++++++++++---------- esmvalcore/preprocessor/__init__.py | 12 +-- esmvalcore/preprocessor/_multimodel.py | 2 - 3 files changed, 71 insertions(+), 47 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 1b12b55370..d6f33800fe 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1,12 +1,12 @@ """Recipe parser.""" import fnmatch +import itertools import logging import os import re from collections import OrderedDict, defaultdict from copy import deepcopy from pprint import pformat -import itertools import yaml from netCDF4 import Dataset @@ -15,8 +15,8 @@ from . import _recipe_checks as check from ._config import (TAGS, get_activity, get_institutes, get_project_config, replace_tags) -from ._data_finder import (get_input_filelist, get_output_file, - get_multiproduct_filename) +from ._data_finder import (get_input_filelist, get_multiproduct_filename, + get_output_file) from ._provenance import TrackedFile, get_recipe_provenance from ._recipe_checks import RecipeError from ._task import (DiagnosticTask, get_flattened_tasks, get_independent_tasks, @@ -40,6 +40,7 @@ def ordered_safe_load(stream): """Load a YAML file using OrderedDict instead of dict.""" class OrderedSafeLoader(yaml.SafeLoader): """Loader class that uses OrderedDict to load a map.""" + def construct_mapping(loader, node): """Load a map as an OrderedDict.""" loader.flatten_mapping(node) @@ -468,21 +469,18 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars): if not fx_vars: return - fx_vars = [ - _get_fx_file(variable, fxvar, config_user) - for fxvar in fx_vars - ] + fx_vars = [_get_fx_file(variable, fxvar, config_user) for fxvar in fx_vars] fx_dict = {fx_var[1]['short_name']: fx_var[0] for fx_var in fx_vars} settings['fx_variables'] = fx_dict logger.info('Using fx_files: %s for variable %s during step %s', - pformat(settings['fx_variables']), - variable['short_name'], + pformat(settings['fx_variables']), variable['short_name'], step_name) def _update_fx_settings(settings, variable, config_user): """Update fx settings depending on the needed method.""" + # get fx variables either from user defined attribute or fixed def _get_fx_vars_from_attribute(step_settings, step_name): user_fx_vars = step_settings.get('fx_variables') @@ -493,8 +491,8 @@ def _get_fx_vars_from_attribute(step_settings, step_name): user_fx_vars.append('sftof') elif step_name == 'mask_landseaice': user_fx_vars = ['sftgif'] - elif step_name in ('area_statistics', - 'volume_statistics', 'zonal_statistics'): + elif step_name in ('area_statistics', 'volume_statistics', + 'zonal_statistics'): user_fx_vars = [] return user_fx_vars @@ -506,8 +504,8 @@ def _get_fx_vars_from_attribute(step_settings, step_name): for step_name, step_settings in settings.items(): if step_name in fx_steps: fx_vars = _get_fx_vars_from_attribute(step_settings, step_name) - _update_fx_files(step_name, step_settings, - variable, config_user, fx_vars) + _update_fx_files(step_name, step_settings, variable, config_user, + fx_vars) def _read_attributes(filename): @@ -627,7 +625,11 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped -def _update_multi_product_settings(input_products, order, preproc_dir, step, grouping=None): +def _update_multi_product_settings(input_products, + order, + preproc_dir, + step, + grouping=None): """Return new products that are aggregated over multiple datasets. These new products will replace the original products at runtime. Therefore, they @@ -653,7 +655,9 @@ def _update_multi_product_settings(input_products, order, preproc_dir, step, gro grouped_products = groupby(products, keyfunc=lambda p: p.group(grouping)) - relevant_settings = {'output_products': defaultdict(dict)} # pass to ancestors + relevant_settings = { + 'output_products': defaultdict(dict) + } # pass to ancestors output_products = set() for identifier, products in grouped_products.items(): @@ -661,17 +665,21 @@ def _update_multi_product_settings(input_products, order, preproc_dir, step, gro for statistic in settings.get('statistics'): - statistic_str = statistic.replace('.', '-') # avoid . in filename for percentiles + statistic_str = statistic.replace( + '.', '-') # avoid . in filename for percentiles step_tag = f'{tag}{statistic_str.title()}' common_attributes[step] = step_tag - filename = get_multiproduct_filename(common_attributes, preproc_dir) + filename = get_multiproduct_filename(common_attributes, + preproc_dir) common_attributes['filename'] = filename - statistic_product = PreprocessorFile(common_attributes, downstream_settings) + statistic_product = PreprocessorFile(common_attributes, + downstream_settings) output_products.add(statistic_product) - relevant_settings['output_products'][identifier][statistic] = statistic_product + relevant_settings['output_products'][identifier][ + statistic] = statistic_product return output_products, relevant_settings @@ -679,9 +687,13 @@ def _update_multi_product_settings(input_products, order, preproc_dir, step, gro def _update_ensemble(products, order, preproc_dir): """Define output settings for ensemble products.""" step = 'ensemble_statistics' - groupby = ('project', 'dataset', 'exp') + groupby = ['project', 'dataset', 'exp'] - return _update_multi_product_settings(products, order, preproc_dir, step, grouping=groupby) + return _update_multi_product_settings(products, + order, + preproc_dir, + step, + grouping=groupby) def _update_multimodel(products, order, preproc_dir): @@ -689,7 +701,11 @@ def _update_multimodel(products, order, preproc_dir): step = 'multi_model_statistics' groupby = list(products)[0].settings[step].get('groupby', None) - return _update_multi_product_settings(products, order, preproc_dir, step, grouping=groupby) + return _update_multi_product_settings(products, + order, + preproc_dir, + step, + grouping=groupby) def update_ancestors(ancestors, step, downstream_settings): @@ -745,10 +761,7 @@ def get_matching(attributes): return grouped_products -def _get_preprocessor_products(variables, - profile, - order, - ancestor_products, +def _get_preprocessor_products(variables, profile, order, ancestor_products, config_user): """ Get preprocessor product definitions for a set of datasets. @@ -760,8 +773,7 @@ def _get_preprocessor_products(variables, preproc_dir = config_user['preproc_dir'] for variable in variables: - variable['filename'] = get_output_file(variable, - preproc_dir) + variable['filename'] = get_output_file(variable, preproc_dir) if ancestor_products: grouped_ancestors = _match_products(ancestor_products, variables) @@ -807,14 +819,27 @@ def _get_preprocessor_products(variables, ) products.add(product) - - ensemble_products, ensemble_settings = _update_ensemble(products, order, preproc_dir) - multimodel_products, multimodel_settings = _update_multimodel(ensemble_products, order, preproc_dir) + ensemble_products, ensemble_settings = _update_ensemble( + products, order, preproc_dir) + multimodel_products, multimodel_settings = _update_multimodel( + ensemble_products, order, preproc_dir) # Update multi-product settings (workaround for lack of better ancestry tracking) - update_ancestors(ancestors=products, step='ensemble_statistics', downstream_settings=ensemble_settings,) - update_ancestors(ancestors=products, step='multi_model_statistics', downstream_settings=multimodel_settings,) - update_ancestors(ancestors=ensemble_products, step='multi_model_statistics', downstream_settings=multimodel_settings,) + update_ancestors( + ancestors=products, + step='ensemble_statistics', + downstream_settings=ensemble_settings, + ) + update_ancestors( + ancestors=products, + step='multi_model_statistics', + downstream_settings=multimodel_settings, + ) + update_ancestors( + ancestors=ensemble_products, + step='multi_model_statistics', + downstream_settings=multimodel_settings, + ) for product in products | ensemble_products | multimodel_products: product.check() @@ -840,12 +865,11 @@ def _get_single_preprocessor_task(variables, check.check_for_temporal_preprocs(profile) ancestor_products = None - products = _get_preprocessor_products( - variables=variables, - profile=profile, - order=order, - ancestor_products=ancestor_products, - config_user=config_user) + products = _get_preprocessor_products(variables=variables, + profile=profile, + order=order, + ancestor_products=ancestor_products, + config_user=config_user) if not products: raise RecipeError( diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 96dd6e134c..edc5da1c41 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -22,7 +22,7 @@ from ._mask import (mask_above_threshold, mask_below_threshold, mask_fillvalues, mask_glaciated, mask_inside_range, mask_landsea, mask_landseaice, mask_outside_range) -from ._multimodel import multi_model_statistics, ensemble_statistics +from ._multimodel import ensemble_statistics, multi_model_statistics from ._other import clip from ._regrid import extract_levels, extract_point, regrid from ._time import (annual_statistics, anomalies, climate_statistics, @@ -270,10 +270,12 @@ def get_step_blocks(steps, order): class PreprocessorFile(TrackedFile): """Preprocessor output file.""" - def __init__(self, - attributes, - settings, - ancestors=None,): + def __init__( + self, + attributes, + settings, + ancestors=None, + ): super(PreprocessorFile, self).__init__(attributes['filename'], attributes, ancestors) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9ac847612e..0fcecb692c 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -444,8 +444,6 @@ def _grouped_multiproduct_statistics(products, span=None, engine='esmvalcore'): """Apply _multiproduct_statistics on grouped products.""" - breakpoint() - grouped_products = _group(products, groupby=groupby) statistics_products = set() From f4728d16396760e7e66f0ac44ab251aaf194d39e Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 8 Sep 2020 14:41:50 +0200 Subject: [PATCH 074/158] Make clearer string representation for TrackedFile/PreprocessorFile --- esmvalcore/_provenance.py | 4 +++- esmvalcore/preprocessor/__init__.py | 4 ---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index bf675fae0b..59b51cdc16 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -2,6 +2,7 @@ import copy import logging import os +from pathlib import Path from netCDF4 import Dataset from PIL import Image @@ -122,7 +123,8 @@ def __init__(self, filename, attributes, ancestors=None): def __str__(self): """Return summary string.""" - return "{}: {}".format(self.__class__.__name__, self.filename) + name = Path(self._filename).name + return f'{self.__class__.__name__}({repr(name)})' def copy_provenance(self, target=None): """Create a copy with identical provenance information.""" diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index edc5da1c41..8515fb0a4b 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -289,10 +289,6 @@ def __init__( self._cubes = None self._prepared = False - def __repr__(self): - name = Path(self._filename).name - return f'{self.__class__.__name__}({repr(name)})' - def check(self): """Check preprocessor settings.""" check_preprocessor_settings(self.settings) From 1c3d52ceaa21490b210579303ac4f30fae4addef Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 8 Sep 2020 14:42:06 +0200 Subject: [PATCH 075/158] Fix crash when `exp` is a list --- esmvalcore/_data_finder.py | 5 ++++- esmvalcore/preprocessor/__init__.py | 6 ++---- esmvalcore/preprocessor/_multimodel.py | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index d7d4575c2d..cce4f8d34a 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -269,7 +269,10 @@ def get_multiproduct_filename(attributes, preproc_dir): filename_segments = [] for key in relevant_keys: if key in attributes: - filename_segments.append(attributes[key]) + attribute = attributes[key] + if isinstance(attribute, (list, tuple)): + attribute = '-'.join(attribute) + filename_segments.append(attribute) # Add period and extension filename_segments.append(f"{attributes['start_year']}-{attributes['end_year']}.nc") diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 8515fb0a4b..838786183b 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -360,8 +360,6 @@ def group(self, keys: list) -> str: Returns a string that identifies a group. Concatenates a list of values from .attributes """ - from collections.abc import Iterable - if not keys: return '' @@ -371,8 +369,8 @@ def group(self, keys: list) -> str: identifier = [] for key in keys: attribute = self.attributes[key] - if isinstance(attribute, Iterable): - '-'.join(attribute) + if isinstance(attribute, (list, tuple)): + attribute = '-'.join(attribute) identifier.append(attribute) return '_'.join(identifier) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 0fcecb692c..6a51414462 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -420,6 +420,7 @@ def _multiproduct_statistics(products, statistics_product.cubes = [cube] for product in products: statistics_product.wasderivedfrom(product) + logger.info("Generated %s", statistics_product) statistics_products.add(statistics_product) From 7ac082739beccc8a98de3b96cbfaa38834ac8621 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 16:23:53 +0200 Subject: [PATCH 076/158] Make multimodel groupby work for difficult combinations of groups; add tag option as dataset keys; fix filenames --- esmvalcore/_data_finder.py | 6 +++++- esmvalcore/_recipe.py | 27 ++++++++++++++++---------- esmvalcore/preprocessor/_multimodel.py | 23 +++++++++++++++++----- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index d7d4575c2d..9cb0e3c7ef 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -269,7 +269,10 @@ def get_multiproduct_filename(attributes, preproc_dir): filename_segments = [] for key in relevant_keys: if key in attributes: - filename_segments.append(attributes[key]) + filename_segments.extend(attributes[key].split('_')) + + # Remove duplicate segments: + filename_segments = list(dict.fromkeys(filename_segments)) # Add period and extension filename_segments.append(f"{attributes['start_year']}-{attributes['end_year']}.nc") @@ -280,4 +283,5 @@ def get_multiproduct_filename(attributes, preproc_dir): attributes['variable_group'], '_'.join(filename_segments), ) + return outfile diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index d6f33800fe..c4bc79a5cc 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -625,6 +625,20 @@ def groupby(iterable, keyfunc: callable) -> dict: return grouped +def get_tag(step, identifier, statistic): + # Avoid . in filename for percentiles + statistic = statistic.replace('.', '-') + + if step == 'ensemble_statistics': + tag = 'Ensemble' + statistic.title() + elif identifier == '': + tag = 'MultiModel' + statistic.title() + else: + tag = identifier + statistic.title() + + return tag + + def _update_multi_product_settings(input_products, order, preproc_dir, @@ -644,12 +658,6 @@ def _update_multi_product_settings(input_products, if not products: return input_products - tags = { - 'multi_model_statistics': 'MultiModel', - 'ensemble_statistics': 'Ensemble' - } - tag = tags[step] - settings = list(products)[0].settings[step] downstream_settings = _get_downstream_settings(step, order, products) @@ -665,10 +673,9 @@ def _update_multi_product_settings(input_products, for statistic in settings.get('statistics'): - statistic_str = statistic.replace( - '.', '-') # avoid . in filename for percentiles - step_tag = f'{tag}{statistic_str.title()}' - common_attributes[step] = step_tag + tag = get_tag(step, identifier, statistic) + + common_attributes[step] = tag filename = get_multiproduct_filename(common_attributes, preproc_dir) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 0fcecb692c..0647b1893a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -328,9 +328,6 @@ def _multicube_statistics(cubes, statistics, span): If span is neither overlap nor full. """ logger.debug('Multimodel statistics: computing: %s', statistics) - if len(cubes) < 2: - logger.warning("Single dataset in list: will not compute statistics.") - return cubes # Reset time coordinates and make cubes share the same calendar _unify_time_coordinates(cubes) @@ -396,6 +393,14 @@ def _multicube_statistics_iris(cubes, statistics: list): return statistics_cubes +def flatten(lst): + """Return individual elements from a mixed/nested list.""" + for element in lst: + if isinstance(element, (list, tuple)): + yield from flatten(element) + else: + yield element + def _multiproduct_statistics(products, statistics, @@ -410,17 +415,25 @@ def _multiproduct_statistics(products, # Extract cubes from products and compute statistics cubes = [cube for product in products for cube in product.cubes] - statistics_cubes = aggregator(cubes=cubes, statistics=statistics) + cubes = list(flatten(cubes)) + + if len(cubes) < 2: + logger.info('Found only 1 cube; no statistics computed for %r', list(products)[0]) + statistics_cubes = {statistic: cubes[0] for statistic in statistics} + else: + statistics_cubes = aggregator(cubes=cubes, statistics=statistics) # Add statistics to output_products statistics_products = set() + if span: + breakpoint() for statistic, cube in statistics_cubes.items(): # Add to output product and log provenance statistics_product = output_products[statistic] statistics_product.cubes = [cube] for product in products: statistics_product.wasderivedfrom(product) - logger.info("Generated %s", statistics_product) + logger.info("Generated %r", statistics_product) statistics_products.add(statistics_product) return statistics_products From 1541dd3b2ec288a6dbb9b6dc1b8bf6676f3b97ac Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 16:33:23 +0200 Subject: [PATCH 077/158] tick off todo's --- esmvalcore/_recipe.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index c4bc79a5cc..bc025ef680 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -851,9 +851,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, for product in products | ensemble_products | multimodel_products: product.check() - # TODO make groupby keyword work for multi-model statistics - # TODO correct naming of identifiers for grouped multimodel - return products From 1b081cc3343839486ed3b73f4940e714f6c95558 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 8 Sep 2020 16:38:26 +0200 Subject: [PATCH 078/158] fix bug for monthly data in unify time --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 7556e48f4a..72b181e791 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -211,7 +211,7 @@ def _unify_time_coordinates(cubes): # Extract date info from cube coord = cube.coord('time') years = [p.year for p in coord.units.num2date(coord.points)] - months = [p.year for p in coord.units.num2date(coord.points)] + months = [p.month for p in coord.units.num2date(coord.points)] # Reconstruct default calendar if 0 not in np.diff(years): From c35c530a62e7862d179513e7478b4250b147a6fc Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 8 Sep 2020 17:26:02 +0200 Subject: [PATCH 079/158] Remove invalid pytest command --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index dc7036dc6f..9dc53e190b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,6 @@ addopts = --cov-report=xml:test-reports/coverage.xml --cov-report=html:test-reports/coverage_html --html=test-reports/report.html - --numprocesses auto env = MPLBACKEND = Agg flake8-ignore = From 7d9d8553f02f60d32dc6146be614d1ea4288e271 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 8 Sep 2020 17:26:26 +0200 Subject: [PATCH 080/158] Pass pytest->test_simple_recipe --- esmvalcore/_recipe.py | 61 +++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index c4bc79a5cc..55cb31d329 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -656,7 +656,7 @@ def _update_multi_product_settings(input_products, """ products = {p for p in input_products if step in p.settings} if not products: - return input_products + return input_products, dict() settings = list(products)[0].settings[step] downstream_settings = _get_downstream_settings(step, order, products) @@ -683,6 +683,7 @@ def _update_multi_product_settings(input_products, statistic_product = PreprocessorFile(common_attributes, downstream_settings) + output_products.add(statistic_product) relevant_settings['output_products'][identifier][ @@ -824,35 +825,45 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, settings=settings, ancestors=ancestors, ) + products.add(product) - ensemble_products, ensemble_settings = _update_ensemble( - products, order, preproc_dir) - multimodel_products, multimodel_settings = _update_multimodel( - ensemble_products, order, preproc_dir) + if 'ensemble_statistics' in profile: + ensemble_products, ensemble_settings = _update_ensemble( + products, order, preproc_dir) - # Update multi-product settings (workaround for lack of better ancestry tracking) - update_ancestors( - ancestors=products, - step='ensemble_statistics', - downstream_settings=ensemble_settings, - ) - update_ancestors( - ancestors=products, - step='multi_model_statistics', - downstream_settings=multimodel_settings, - ) - update_ancestors( - ancestors=ensemble_products, - step='multi_model_statistics', - downstream_settings=multimodel_settings, - ) + # check for ensemble_settings to bypass tests + update_ancestors( + ancestors=products, + step='ensemble_statistics', + downstream_settings=ensemble_settings, + ) + else: + ensemble_products = set() - for product in products | ensemble_products | multimodel_products: - product.check() + if 'multi_model_statistics' in profile: + multimodel_products, multimodel_settings = _update_multimodel( + ensemble_products, order, preproc_dir) + + # check for multi_model_settings to bypass tests + update_ancestors( + ancestors=products, + step='multi_model_statistics', + downstream_settings=multimodel_settings, + ) - # TODO make groupby keyword work for multi-model statistics - # TODO correct naming of identifiers for grouped multimodel + if 'ensemble_statistics' in profile: + # Update multi-product settings (workaround for lack of better ancestry tracking) + update_ancestors( + ancestors=ensemble_products, + step='multi_model_statistics', + downstream_settings=multimodel_settings, + ) + else: + multimodel_products = set() + + for product in products | multimodel_products | ensemble_products: + product.check() return products From 2f2e354296e4a919f7897d5fc914d93af895ac11 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 8 Sep 2020 17:46:43 +0200 Subject: [PATCH 081/158] Pass pytest->test_custom_preproc_order --- esmvalcore/_recipe.py | 58 ++++++++++++-------------------- tests/integration/test_recipe.py | 2 +- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 55cb31d329..6aa345abd0 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -642,8 +642,7 @@ def get_tag(step, identifier, statistic): def _update_multi_product_settings(input_products, order, preproc_dir, - step, - grouping=None): + step): """Return new products that are aggregated over multiple datasets. These new products will replace the original products at runtime. Therefore, they @@ -659,6 +658,12 @@ def _update_multi_product_settings(input_products, return input_products, dict() settings = list(products)[0].settings[step] + + if step == 'ensemble_statistics': + grouping = ['project', 'dataset', 'exp'] + else: + grouping = settings.get('groupby', None) + downstream_settings = _get_downstream_settings(step, order, products) grouped_products = groupby(products, keyfunc=lambda p: p.group(grouping)) @@ -692,30 +697,6 @@ def _update_multi_product_settings(input_products, return output_products, relevant_settings -def _update_ensemble(products, order, preproc_dir): - """Define output settings for ensemble products.""" - step = 'ensemble_statistics' - groupby = ['project', 'dataset', 'exp'] - - return _update_multi_product_settings(products, - order, - preproc_dir, - step, - grouping=groupby) - - -def _update_multimodel(products, order, preproc_dir): - """Define output settings for multi model products.""" - step = 'multi_model_statistics' - groupby = list(products)[0].settings[step].get('groupby', None) - - return _update_multi_product_settings(products, - order, - preproc_dir, - step, - grouping=groupby) - - def update_ancestors(ancestors, step, downstream_settings): """Retroactively add settings to ancestor products.""" for product in ancestors: @@ -828,35 +809,38 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, products.add(product) - if 'ensemble_statistics' in profile: - ensemble_products, ensemble_settings = _update_ensemble( - products, order, preproc_dir) + ensemble_step = 'ensemble_statistics' + multi_model_step = 'multi_model_statistics' + + if ensemble_step in profile: + ensemble_products, ensemble_settings = _update_multi_product_settings( + products, order, preproc_dir, ensemble_step) # check for ensemble_settings to bypass tests update_ancestors( ancestors=products, - step='ensemble_statistics', + step=ensemble_step, downstream_settings=ensemble_settings, ) else: - ensemble_products = set() + ensemble_products = products - if 'multi_model_statistics' in profile: - multimodel_products, multimodel_settings = _update_multimodel( - ensemble_products, order, preproc_dir) + if multi_model_step in profile: + multimodel_products, multimodel_settings = _update_multi_product_settings( + ensemble_products, order, preproc_dir, multi_model_step) # check for multi_model_settings to bypass tests update_ancestors( ancestors=products, - step='multi_model_statistics', + step=multi_model_step, downstream_settings=multimodel_settings, ) - if 'ensemble_statistics' in profile: + if ensemble_step in profile: # Update multi-product settings (workaround for lack of better ancestry tracking) update_ancestors( ancestors=ensemble_products, - step='multi_model_statistics', + step=multi_model_step, downstream_settings=multimodel_settings, ) else: diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 425571ff73..afd2b52ff0 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -821,7 +821,7 @@ def test_custom_preproc_order(tmp_path, patched_datafinder, config_user): operator: mean multi_model_statistics: span: overlap - statistics: [mean ] + statistics: [mean] custom: custom_order: true <<: *default From 068a628ca2dd4ee6d25dc9fea8dfc52991128151 Mon Sep 17 00:00:00 2001 From: Stef Date: Wed, 9 Sep 2020 11:00:07 +0200 Subject: [PATCH 082/158] Fix pytest multimodel errors --- esmvalcore/_recipe.py | 38 +++++++++---------- esmvalcore/preprocessor/_multimodel.py | 6 +-- .../_multimodel/test_multimodel.py | 20 +++++++--- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 6aa345abd0..c7896d2310 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1,6 +1,5 @@ """Recipe parser.""" import fnmatch -import itertools import logging import os import re @@ -595,7 +594,8 @@ def _get_common_attributes(products): def _get_downstream_settings(step, order, products): - """Get downstream preprocessor settings that are shared between products.""" + """Get downstream preprocessor settings that are shared between + products.""" settings = {} remaining_steps = order[order.index(step) + 1:] some_product = next(iter(products)) @@ -639,19 +639,16 @@ def get_tag(step, identifier, statistic): return tag -def _update_multi_product_settings(input_products, - order, - preproc_dir, - step): +def _update_multiproduct(input_products, order, preproc_dir, step): """Return new products that are aggregated over multiple datasets. - These new products will replace the original products at runtime. Therefore, they - need to have all the settings for the remaining steps. + These new products will replace the original products at runtime. + Therefore, they need to have all the settings for the remaining steps. - The functions in _multimodel.py take output_products as function arguments. These are - the output_products created here. But since those functions are called from the - input products, the products that are created here need to be added to their - ancestors products' settings (). + The functions in _multimodel.py take output_products as function arguments. + These are the output_products created here. But since those functions are + called from the input products, the products that are created here need to + be added to their ancestors products' settings (). """ products = {p for p in input_products if step in p.settings} if not products: @@ -752,11 +749,10 @@ def get_matching(attributes): def _get_preprocessor_products(variables, profile, order, ancestor_products, config_user): - """ - Get preprocessor product definitions for a set of datasets. + """Get preprocessor product definitions for a set of datasets. - It updates recipe settings as needed by various preprocessors - and sets the correct ancestry. + It updates recipe settings as needed by various preprocessors and + sets the correct ancestry. """ products = set() preproc_dir = config_user['preproc_dir'] @@ -813,7 +809,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, multi_model_step = 'multi_model_statistics' if ensemble_step in profile: - ensemble_products, ensemble_settings = _update_multi_product_settings( + ensemble_products, ensemble_settings = _update_multiproduct( products, order, preproc_dir, ensemble_step) # check for ensemble_settings to bypass tests @@ -826,7 +822,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ensemble_products = products if multi_model_step in profile: - multimodel_products, multimodel_settings = _update_multi_product_settings( + multimodel_products, multimodel_settings = _update_multiproduct( ensemble_products, order, preproc_dir, multi_model_step) # check for multi_model_settings to bypass tests @@ -837,7 +833,8 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ) if ensemble_step in profile: - # Update multi-product settings (workaround for lack of better ancestry tracking) + # Update multi-product settings (workaround for lack of better + # ancestry tracking) update_ancestors( ancestors=ensemble_products, step=multi_model_step, @@ -1104,8 +1101,7 @@ def _initialize_datasets(raw_datasets): @staticmethod def _expand_ensemble(variables): - """ - Expand ensemble members to multiple datasets. + """Expand ensemble members to multiple datasets. Expansion only supports ensembles defined as strings, not lists. """ diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 72b181e791..99087537fb 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -400,7 +400,7 @@ def flatten(lst): def _multiproduct_statistics(products, statistics, output_products, - span=None, + span='overlap', engine='esmvalcore'): """Compute statistics on (grouped) products, using iris or esmvalcore functions.""" @@ -451,7 +451,7 @@ def _grouped_multiproduct_statistics(products, statistics: list, output_products, groupby, - span=None, + span='overlap', engine='esmvalcore'): """Apply _multiproduct_statistics on grouped products.""" grouped_products = _group(products, groupby=groupby) @@ -476,7 +476,7 @@ def _grouped_multiproduct_statistics(products, def multi_model_statistics(products, statistics, output_products, - span, + span='overlap', groupby=None): """Apply multimodel statistics to esmvalcore preprocessor products.""" return _grouped_multiproduct_statistics( diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 42b28a9525..6bb6b78efe 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -7,12 +7,12 @@ from cf_units import Unit import tests -from esmvalcore.preprocessor import multi_model_statistics from esmvalcore.preprocessor._multimodel import (_assemble_data, _compute_statistic, _get_time_slice, _plev_fix, _put_in_cube, - _unify_time_coordinates) + _unify_time_coordinates, + _multicube_statistics) class Test(tests.Test): @@ -92,7 +92,7 @@ def test_compute_statistic(self): def test_compute_full_statistic_mon_cube(self): data = [self.cube1, self.cube2] - stats = multi_model_statistics(data, 'full', ['mean']) + stats = _multicube_statistics(data, span='full', statistics=['mean']) expected_full_mean = np.ma.ones((5, 3, 2, 2)) expected_full_mean.mask = np.ones((5, 3, 2, 2)) expected_full_mean.mask[1] = False @@ -100,7 +100,7 @@ def test_compute_full_statistic_mon_cube(self): def test_compute_full_statistic_yr_cube(self): data = [self.cube4, self.cube5] - stats = multi_model_statistics(data, 'full', ['mean']) + stats = _multicube_statistics(data, span='full', statistics=['mean']) expected_full_mean = np.ma.ones((4, 3, 2, 2)) expected_full_mean.mask = np.zeros((4, 3, 2, 2)) expected_full_mean.mask[2:4] = True @@ -108,13 +108,21 @@ def test_compute_full_statistic_yr_cube(self): def test_compute_overlap_statistic_mon_cube(self): data = [self.cube1, self.cube1] - stats = multi_model_statistics(data, 'overlap', ['mean']) + stats = _multicube_statistics( + data, + span='overlap', + statistics=['mean'] + ) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) def test_compute_overlap_statistic_yr_cube(self): data = [self.cube4, self.cube4] - stats = multi_model_statistics(data, 'overlap', ['mean']) + stats = _multicube_statistics( + data, + span='overlap', + statistics=['mean'] + ) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) From 276112a0ccbfac76c6953ed8fd85165d916ed82a Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 9 Sep 2020 16:41:56 +0200 Subject: [PATCH 083/158] Remove flatten, it is no longer necessary since the nested loop has been fixed --- esmvalcore/preprocessor/_multimodel.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 99087537fb..ea74c2052a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -388,15 +388,6 @@ def _multicube_statistics_iris(cubes, statistics: list): return statistics_cubes -def flatten(lst): - """Return individual elements from a mixed/nested list.""" - for element in lst: - if isinstance(element, (list, tuple)): - yield from flatten(element) - else: - yield element - - def _multiproduct_statistics(products, statistics, output_products, @@ -411,7 +402,6 @@ def _multiproduct_statistics(products, # Extract cubes from products and compute statistics cubes = [cube for product in products for cube in product.cubes] - cubes = list(flatten(cubes)) if len(cubes) < 2: logger.info('Found only 1 cube; no statistics computed for %r', From d262f0ba67be1aaa9c26ed4614c3cc9c3e99575b Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 9 Sep 2020 17:26:28 +0200 Subject: [PATCH 084/158] Add documentation to _multimodel.py; expose multicube functions to public API --- esmvalcore/preprocessor/_multimodel.py | 158 ++++++++++++------ .../_multimodel/test_multimodel.py | 10 +- 2 files changed, 116 insertions(+), 52 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ea74c2052a..c0114735a3 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -1,14 +1,15 @@ -"""multimodel statistics. +"""Statistics across cubes. -Functions for multi-model operations -supports a multitude of multimodel statistics -computations; the only requisite is the ingested -cubes have (TIME-LAT-LON) or (TIME-PLEV-LAT-LON) -dimensions; and obviously consistent units. +This module contains functions to compute statistics +across multiple cubes or products. -It operates on different (time) spans: -- full: computes stats on full dataset time; -- overlap: computes common time overlap between datasets; +Ensemble statistics uses iris built in functions and +support lazy evaluation. Multi-model statistics uses +custom functions that operate directly on numpy arrays. + +Wrapper functions separate esmvalcore internals, operating on products, +from generalized functions that operate on iris cubes. +These wrappers support grouped execution by passing a groupby keyword. """ import logging @@ -21,6 +22,7 @@ import iris import numpy as np import scipy +from iris.experimental.equalise_cubes import equalise_attributes logger = logging.getLogger(__name__) @@ -283,38 +285,38 @@ def _assemble_data(cubes, statistic, span='overlap'): return stats_cube -def _multicube_statistics(cubes, statistics, span): - """Compute multi-model statistics. +def multicube_statistics(cubes, statistics, span): + """Compute statistics across input cubes. + + This function deals with non-homogeneous cubes by + taking the time union (span='full') or intersection + (span='overlap'), and extending or subsetting the + cubes as necessary. Apart from the time coordinate, + cubes must have consistent shapes. - Multimodel statistics computed along the time axis. Can be - computed across a common overlap in time (set span: overlap) - or across the full length in time of each model (set span: full). - Restrictive computation is also available by excluding any set of - models that the user will not want to include in the statistics - (set exclude: [excluded models list]). + This function operates directly on numpy (masked) arrays + and rebuilds the resulting cubes from scratch. Therefore, + it is not suitable for lazy evaluation. - Restrictions needed by the input data: - - model datasets must have consistent shapes, - - higher dimensional data is not supported (ie dims higher than four: - time, vertical axis, two horizontal axes). + This function is restricted to maximum four-dimensional data: + time, vertical axis, two horizontal axes. Parameters ---------- cubes: list of cubes list of cubes to be used in multimodel stat computation; + statistics: list of strings + statistical measures to be computed. Available options: mean, median, + max, min, std, or pXX.YY (for percentile XX.YY; decimal part optional). span: str overlap or full; if overlap, statitsticss are computed on common time- span; if full, statistics are computed on full time spans, ignoring missing data. - statistics: str - statistical measure to be computed. Available options: mean, median, - max, min, std, or pXX.YY (for percentile XX.YY; decimal part optional). Returns ------- - list - list of data products or cubes containing the multimodel stats - computed. + dict + dictionary of statistics cubes with statistics' names as keys. Raises ------ @@ -352,15 +354,34 @@ def _multicube_statistics(cubes, statistics, span): return statistics_cubes -def _multicube_statistics_iris(cubes, statistics: list): - """Use iris merge/collapsed to perform the aggregation. +def multicube_statistics_iris(cubes, statistics): + """Compute statistics across input cubes. + + Cubes are merged and subsequently collapsed along a new auxiliary + coordinate. Inconsistent attributes will be removed. + + This method uses iris' built in functions, exposing the operators + in iris.analysis and supporting lazy evaluation. Input cubes must + have consistent shapes. + + Note: some of the operators in iris.analysis require additional + arguments, such as percentiles or weights. These operators + are currently not supported. + + Parameters + ---------- + cubes: list of cubes + list of cubes to be used in multimodel stat computation; + statistics: list of strings + statistical measures to be computed. Choose from the operators + listed in the iris.analysis package. + + Returns + ------- + dict + dictionary of statistics cubes with statistics' names as keys. - Equivalent to _multicube_statistics, but uses iris functions to - perform the aggregation. This only works if the input cubes are very - homogeneous, e.g. for different ensemble members of the same - model/dataset. """ - from iris.experimental.equalise_cubes import equalise_attributes operators = vars(iris.analysis) for i, cube in enumerate(cubes): @@ -393,16 +414,43 @@ def _multiproduct_statistics(products, output_products, span='overlap', engine='esmvalcore'): - """Compute statistics on (grouped) products, using iris or esmvalcore - functions.""" + """Compute statistics across products. + + Extract cubes from products, calculate the statistics across cubes + and assign the resulting output cubes to the output_products. + + Parameters + ---------- + products: list + list of PreprocessorFile's + statistics: list + list of strings describing the statistics that will be computed + output_products: dict + dict of PreprocessorFile's with statistic names as keys. + span: str + 'full' or 'overlap', whether to calculate the statistics on the + time union ('full') or time intersection ('overlap') of the cubes. + engine: str + 'iris' or 'esmvalcore', which function to use to compute the + statistics. Iris is more efficient and exposes more operators, + but requires highly homogeneous/compatible cubes. + + Returns + ------- + set + set of PreprocessorFiles containing the computed + statistics cubes. + + """ if engine == 'iris': - aggregator = _multicube_statistics_iris + aggregator = multicube_statistics_iris else: - aggregator = partial(_multicube_statistics, span=span) + aggregator = partial(multicube_statistics, span=span) - # Extract cubes from products and compute statistics + # Extract cubes from products cubes = [cube for product in products for cube in product.cubes] + # Compute statistics if len(cubes) < 2: logger.info('Found only 1 cube; no statistics computed for %r', list(products)[0]) @@ -410,13 +458,12 @@ def _multiproduct_statistics(products, else: statistics_cubes = aggregator(cubes=cubes, statistics=statistics) - # Add statistics to output_products + # Add cubes to output products and log provenance statistics_products = set() - for statistic, cube in statistics_cubes.items(): - # Add to output product and log provenance statistics_product = output_products[statistic] statistics_product.cubes = [cube] + for product in products: statistics_product.wasderivedfrom(product) @@ -427,7 +474,10 @@ def _multiproduct_statistics(products, def _group(products, groupby=None): - """Group products.""" + """Group products. + + Returns a dict of product sets with identifiers as keys. + """ grouped_products = defaultdict(set) for product in products: identifier = product.group(groupby) @@ -450,7 +500,8 @@ def _grouped_multiproduct_statistics(products, for identifier, products in grouped_products.items(): sub_output_products = output_products[identifier] - statistics_product = _multiproduct_statistics( + # Compute statistics on a single group + group_statistics = _multiproduct_statistics( products=products, statistics=statistics, output_products=sub_output_products, @@ -458,7 +509,7 @@ def _grouped_multiproduct_statistics(products, engine=engine, ) - statistics_products |= statistics_product + statistics_products |= group_statistics return statistics_products @@ -468,7 +519,14 @@ def multi_model_statistics(products, output_products, span='overlap', groupby=None): - """Apply multimodel statistics to esmvalcore preprocessor products.""" + """ESMValCore entry point for multi model statistics. + + The products are grouped (if groupby argument is specified) and the + cubes are extracted from their products. Resulting cubes are added to + their corresponding `output_products`. + + Multi-model statistics are computed in `multicube_statistics`. + """ return _grouped_multiproduct_statistics( products=products, statistics=statistics, @@ -479,7 +537,13 @@ def multi_model_statistics(products, def ensemble_statistics(products, statistics, output_products): - """Apply ensemble statistics to esmvalcore preprocessor products.""" + """ESMValCore entry point for ensemble statistics. + + The products are grouped and the cubes are extracted from + the products. Resulting cubes are assigned to `output_products`. + + Ensemble statistics are computed in `multicube_statistics_iris`. + """ ensemble_grouping = ('project', 'dataset', 'exp') return _grouped_multiproduct_statistics( products=products, diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 6bb6b78efe..10c80fdf02 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -12,7 +12,7 @@ _get_time_slice, _plev_fix, _put_in_cube, _unify_time_coordinates, - _multicube_statistics) + multicube_statistics) class Test(tests.Test): @@ -92,7 +92,7 @@ def test_compute_statistic(self): def test_compute_full_statistic_mon_cube(self): data = [self.cube1, self.cube2] - stats = _multicube_statistics(data, span='full', statistics=['mean']) + stats = multicube_statistics(data, span='full', statistics=['mean']) expected_full_mean = np.ma.ones((5, 3, 2, 2)) expected_full_mean.mask = np.ones((5, 3, 2, 2)) expected_full_mean.mask[1] = False @@ -100,7 +100,7 @@ def test_compute_full_statistic_mon_cube(self): def test_compute_full_statistic_yr_cube(self): data = [self.cube4, self.cube5] - stats = _multicube_statistics(data, span='full', statistics=['mean']) + stats = multicube_statistics(data, span='full', statistics=['mean']) expected_full_mean = np.ma.ones((4, 3, 2, 2)) expected_full_mean.mask = np.zeros((4, 3, 2, 2)) expected_full_mean.mask[2:4] = True @@ -108,7 +108,7 @@ def test_compute_full_statistic_yr_cube(self): def test_compute_overlap_statistic_mon_cube(self): data = [self.cube1, self.cube1] - stats = _multicube_statistics( + stats = multicube_statistics( data, span='overlap', statistics=['mean'] @@ -118,7 +118,7 @@ def test_compute_overlap_statistic_mon_cube(self): def test_compute_overlap_statistic_yr_cube(self): data = [self.cube4, self.cube4] - stats = _multicube_statistics( + stats = multicube_statistics( data, span='overlap', statistics=['mean'] From 3d531b9f3efcd30e5124a0400619a465c87a6ba8 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 10 Sep 2020 13:10:45 +0200 Subject: [PATCH 085/158] Add to RTD, improve docstrings, add multicube stats to public API --- doc/recipe/preprocessor.rst | 121 ++++++++++++++++--------- esmvalcore/preprocessor/__init__.py | 6 +- esmvalcore/preprocessor/_multimodel.py | 86 +++++++++--------- 3 files changed, 128 insertions(+), 85 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index cfc47fadf5..1928505da8 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -14,6 +14,7 @@ roughly following the default order in which preprocessor functions are applied: * :ref:`Land/Sea/Ice masking` * :ref:`Horizontal regridding` * :ref:`Masking of missing values` +* :ref:`Ensemble statistics` * :ref:`Multi-model statistics` * :ref:`Time operations` * :ref:`Area operations` @@ -448,17 +449,17 @@ Missing values masks -------------------- Missing (masked) values can be a nuisance especially when dealing with -multimodel ensembles and having to compute multimodel statistics; different +multi-model ensembles and having to compute multi-model statistics; different numbers of missing data from dataset to dataset may introduce biases and artificially assign more weight to the datasets that have less missing data. This is handled in ESMValTool via the missing values masks: two types of -such masks are available, one for the multimodel case and another for the +such masks are available, one for the multi-model case and another for the single model case. -The multimodel missing values mask (``mask_fillvalues``) is a preprocessor step +The multi-model missing values mask (``mask_fillvalues``) is a preprocessor step that usually comes after all the single-model steps (regridding, area selection etc) have been performed; in a nutshell, it combines missing values masks from -individual models into a multimodel missing values mask; the individual model +individual models into a multi-model missing values mask; the individual model masks are built according to common criteria: the user chooses a time window in which missing data points are counted, and if the number of missing data points relative to the number of total data points in a window is less than a chosen @@ -484,7 +485,7 @@ See also :func:`esmvalcore.preprocessor.mask_fillvalues`. Common mask for multiple models ------------------------------- -It is possible to use ``mask_fillvalues`` to create a combined multimodel +It is possible to use ``mask_fillvalues`` to create a combined multi-model mask (all the masks from all the analyzed models combined into a single mask); for that purpose setting the ``threshold_fraction`` to 0 will not discard any time windows, essentially keeping the original model masks and @@ -522,7 +523,7 @@ Horizontal regridding Regridding is necessary when various datasets are available on a variety of `lat-lon` grids and they need to be brought together on a common grid (for -various statistical operations e.g. multimodel statistics or for e.g. direct +various statistical operations e.g. multi-model statistics or for e.g. direct inter-comparison or comparison with observational datasets). Regridding is conceptually a very similar process to interpolation (in fact, the regridder engine uses interpolation and extrapolation, with various schemes). The primary @@ -636,6 +637,37 @@ See also :func:`esmvalcore.preprocessor.regrid` for resolutions of ``< 0.5`` degrees the regridding becomes very slow and will use a lot of memory. +.. _ensemble statistics: + +Ensemble statistics +=================== +For certain use cases it may be desirable to compute ensemble statistics. For +example to prevent models with many ensemble member getting excessive weight in +the multi-model statistics functions. + +Theoretically, ensemble statistics are a special case (grouped) multi-model +statistics. However, they should typically be computed earlier in the workflow. +Moreover, because multiple ensemble members of the same model are typically more +consistent/homogeneous than datasets from different models, the implementation +is more straigtforward and can benefit from lazy evaluation and more efficient +computation. + +The preprocessor takes a list of statistics as input: + +.. code-block:: yaml + + preprocessors: + example_preprocessor: + ensemble_statistics: + statistics: [mean, median] + +This preprocessor function exposes the iris analysis package, and works with all +(capitalized) statistics from the `iris' analysis package +`_ +that can be executed without additional arguments (e.g. percentiles are not +supported because it requires additional keywords: percentile.). + +see also :func:`esmvalcore.preprocessor.multiproduct_statistics_iris`. .. _multi-model statistics: @@ -653,22 +685,14 @@ multi-model ``mean``, ``median``, ``max``, ``min``, ``std``, and / or ``multi_model_statistics``. Percentiles can be specified like ``p1.5`` or ``p95``. The decimal point will be replaced by a dash in the output file. -Note that current multimodel statistics in ESMValTool are local (not global), -and are computed along the time axis. As such, can be computed across a common -overlap in time (by specifying ``span: overlap`` argument) or across the full -length in time of each model (by specifying ``span: full`` argument). - Restrictive computation is also available by excluding any set of models that the user will not want to include in the statistics (by setting ``exclude: [excluded models list]`` argument). The implementation has a few restrictions -that apply to the input data: model datasets must have consistent shapes, and -from a statistical point of view, this is needed since weights are not yet -implemented; also higher dimensional data is not supported (i.e. anything with -dimensionality higher than four: time, vertical axis, two horizontal axes). +that apply to the input data: model datasets must have consistent shapes, apart from +the time dimension; and cubes with more than four dimensions (time, vertical axis, two horizontal axes) are not supported. -Input datasets may have different time coordinates. The multi-model statistics -preprocessor sets a common time coordinate on all datasets. As the number of -days in a year may vary between calendars, (sub-)daily data are not supported. +Input datasets may have different time coordinates. Statistics can be computed across overlapping times only (``span: overlap``) or across the full time span of the combined +models (``span: full``). The preprocessor sets a common time coordinate on all datasets. As the number of days in a year may vary between calendars, (sub-)daily data are not supported. .. code-block:: yaml @@ -679,29 +703,42 @@ days in a year may vary between calendars, (sub-)daily data are not supported. statistics: [mean, median] exclude: [NCEP] -see also :func:`esmvalcore.preprocessor.multi_model_statistics`. +Multi-model statistics now also supports a ``groupby`` argument. You can group by +any dataset key (``project``, ``experiment``, etc.) or a combination of keys. You can +also add an arbitrary 'tag' to a dataset definition and then groupby that tag. When +using this preprocessor in conjunction with `ensemble statistics`_ preprocessor, you +can groupby ``ensemble_statistics`` as well. For example: + +.. code-block:: yaml + + datasets: + - {dataset: CanESM2, exp: historical, ensemble: "r(1:2)i1p1", tag: 'group1'} + - {dataset: CCSM4, exp: historical, ensemble: "r(1:2)i1p1", tag: 'group2'} + + preprocessors: + example_preprocessor: + ensemble_statistics: + statistics: [median, mean] + multi_model_statistics: + span: overlap + statistics: [min, max] + groupby: [ensemble_statistics, tag] + exclude: [NCEP] -When calling the module inside diagnostic scripts, the input must be given -as a list of cubes. The output will be saved in a dictionary where each -entry contains the resulting cube with the requested statistic operations. +This will first compute ensemble mean and median, and then compute the multi-model +min and max separately for the ensemble means and medians. -.. code-block:: - from esmvalcore.preprocessor import multi_model_statistics - statistics = multi_model_statistics([cube1,...,cubeN], 'overlap', ['mean', 'median']) - mean_cube = statistics['mean'] - median_cube = statistics['median'] +see also :func:`esmvalcore.preprocessor.multiproduct_statistics`. .. note:: - Note that the multimodel array operations, albeit performed in - per-time/per-horizontal level loops to save memory, could, however, be - rather memory-intensive (since they are not performed lazily as - yet). The Section on :ref:`Memory use` details the memory intake - for different run scenarios, but as a thumb rule, for the multimodel - preprocessor, the expected maximum memory intake could be approximated as - the number of datasets multiplied by the average size in memory for one - dataset. + The multi-model array operations can be rather memory-intensive (since they are not + performed lazily as yet). The Section on :ref:`Memory use` details the memory + intake for different run scenarios, but as a thumb rule, for the multi-model + preprocessor, the expected maximum memory intake could be approximated as the + number of datasets multiplied by the average size in memory + for one dataset. .. _time operations: @@ -1176,7 +1213,7 @@ as a CMOR variable can permit): fx_variables: [{'short_name': 'areacello', 'mip': 'Omon'}, {'short_name': 'volcello, mip': 'fx'}] -The recipe parser wil automatically find the data files that are associated with these +The recipe parser will automatically find the data files that are associated with these variables and pass them to the function for loading and processing. See also :func:`esmvalcore.preprocessor.area_statistics`. @@ -1236,7 +1273,7 @@ as a CMOR variable can permit): fx_variables: [{'short_name': 'areacello', 'mip': 'Omon'}, {'short_name': 'volcello, mip': 'fx'}] -The recipe parser wil automatically find the data files that are associated with these +The recipe parser will automatically find the data files that are associated with these variables and pass them to the function for loading and processing. See also :func:`esmvalcore.preprocessor.volume_statistics`. @@ -1369,14 +1406,14 @@ In the most general case, we can set upper limits on the maximum memory the analysis will require: -``Ms = (R + N) x F_eff - F_eff`` - when no multimodel analysis is performed; +``Ms = (R + N) x F_eff - F_eff`` - when no multi-model analysis is performed; -``Mm = (2R + N) x F_eff - 2F_eff`` - when multimodel analysis is performed; +``Mm = (2R + N) x F_eff - 2F_eff`` - when multi-model analysis is performed; where -* ``Ms``: maximum memory for non-multimodel module -* ``Mm``: maximum memory for multimodel module +* ``Ms``: maximum memory for non-multi-model module +* ``Mm``: maximum memory for multi-model module * ``R``: computational efficiency of module; `R` is typically 2-3 * ``N``: number of datasets * ``F_eff``: average size of data per dataset where ``F_eff = e x f x F`` @@ -1395,7 +1432,7 @@ where ``Mm = 1.5 x (N - 2)`` GB As a rule of thumb, the maximum required memory at a certain time for -multimodel analysis could be estimated by multiplying the number of datasets by +multi-model analysis could be estimated by multiplying the number of datasets by the average file size of all the datasets; this memory intake is high but also assumes that all data is fully realized in memory; this aspect will gradually change and the amount of realized data will decrease with the increase of diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 9c647efe02..e2002c72df 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -21,7 +21,8 @@ from ._mask import (mask_above_threshold, mask_below_threshold, mask_fillvalues, mask_glaciated, mask_inside_range, mask_landsea, mask_landseaice, mask_outside_range) -from ._multimodel import ensemble_statistics, multi_model_statistics +from ._multimodel import (ensemble_statistics, multi_model_statistics, + multicube_statistics, multicube_statistics_iris) from ._other import clip from ._regrid import extract_levels, extract_point, regrid from ._time import (annual_statistics, anomalies, climate_statistics, @@ -67,6 +68,7 @@ 'mask_landseaice', # Ensemble statistics 'ensemble_statistics', + 'multicube_statistics_iris', # Regridding 'regrid', # Point interpolation @@ -88,7 +90,9 @@ # 'average_zone': average_zone, # 'cross_section': cross_section, 'detrend', + # Multi-model statistics 'multi_model_statistics', + 'multicube_statistics', # Grid-point operations 'extract_named_regions', 'depth_integration', diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index c0114735a3..50217d5732 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -1,15 +1,15 @@ """Statistics across cubes. -This module contains functions to compute statistics -across multiple cubes or products. +This module contains functions to compute statistics across multiple cubes or +products. -Ensemble statistics uses iris built in functions and -support lazy evaluation. Multi-model statistics uses -custom functions that operate directly on numpy arrays. +Ensemble statistics uses iris built in functions and support lazy evaluation. +Multi-model statistics uses custom functions that operate directly on numpy +arrays. -Wrapper functions separate esmvalcore internals, operating on products, -from generalized functions that operate on iris cubes. -These wrappers support grouped execution by passing a groupby keyword. +Wrapper functions separate esmvalcore internals, operating on products, from +generalized functions that operate on iris cubes. These wrappers support +grouped execution by passing a groupby keyword. """ import logging @@ -196,13 +196,13 @@ def _get_consistent_time_unit(cubes): def _unify_time_coordinates(cubes): """Make sure all cubes' share the same time coordinate. - This function extracts the date information from the cube and - reconstructs the time coordinate, resetting the actual dates to the - 15th of the month or 1st of july for yearly data (consistent with - `regrid_time`), so that there are no mismatches in the time arrays. + This function extracts the date information from the cube and reconstructs + the time coordinate, resetting the actual dates to the 15th of the month or + 1st of july for yearly data (consistent with `regrid_time`), so that there + are no mismatches in the time arrays. - If cubes have different time units, it will use reset the calendar to - a default gregorian calendar with unit "days since 1850-01-01". + If cubes have different time units, it will use reset the calendar to a + default gregorian calendar with unit "days since 1850-01-01". Might not work for (sub)daily data, because different calendars may have different number of days in the year. @@ -288,24 +288,23 @@ def _assemble_data(cubes, statistic, span='overlap'): def multicube_statistics(cubes, statistics, span): """Compute statistics across input cubes. - This function deals with non-homogeneous cubes by - taking the time union (span='full') or intersection - (span='overlap'), and extending or subsetting the - cubes as necessary. Apart from the time coordinate, - cubes must have consistent shapes. + This function deals with non-homogeneous cubes by taking the time union + (span='full') or intersection (span='overlap'), and extending or subsetting + the cubes as necessary. Apart from the time coordinate, cubes must have + consistent shapes. - This function operates directly on numpy (masked) arrays - and rebuilds the resulting cubes from scratch. Therefore, - it is not suitable for lazy evaluation. + This function operates directly on numpy (masked) arrays and rebuilds the + resulting cubes from scratch. Therefore, it is not suitable for lazy + evaluation. - This function is restricted to maximum four-dimensional data: - time, vertical axis, two horizontal axes. + This function is restricted to maximum four-dimensional data: time, + vertical axis, two horizontal axes. Parameters ---------- - cubes: list of cubes + cubes: list list of cubes to be used in multimodel stat computation; - statistics: list of strings + statistics: list statistical measures to be computed. Available options: mean, median, max, min, std, or pXX.YY (for percentile XX.YY; decimal part optional). span: str @@ -357,22 +356,24 @@ def multicube_statistics(cubes, statistics, span): def multicube_statistics_iris(cubes, statistics): """Compute statistics across input cubes. + Like multicube_statistics, but operates directly on Iris cubes. + Cubes are merged and subsequently collapsed along a new auxiliary coordinate. Inconsistent attributes will be removed. - This method uses iris' built in functions, exposing the operators - in iris.analysis and supporting lazy evaluation. Input cubes must - have consistent shapes. + This method uses iris' built in functions, exposing the operators in + iris.analysis and supporting lazy evaluation. Input cubes must have + consistent shapes. - Note: some of the operators in iris.analysis require additional - arguments, such as percentiles or weights. These operators - are currently not supported. + Note: some of the operators in iris.analysis require additional arguments, + such as percentiles or weights. These operators are currently not + supported. Parameters ---------- - cubes: list of cubes + cubes: list list of cubes to be used in multimodel stat computation; - statistics: list of strings + statistics: list statistical measures to be computed. Choose from the operators listed in the iris.analysis package. @@ -380,7 +381,6 @@ def multicube_statistics_iris(cubes, statistics): ------- dict dictionary of statistics cubes with statistics' names as keys. - """ operators = vars(iris.analysis) @@ -416,8 +416,8 @@ def _multiproduct_statistics(products, engine='esmvalcore'): """Compute statistics across products. - Extract cubes from products, calculate the statistics across cubes - and assign the resulting output cubes to the output_products. + Extract cubes from products, calculate the statistics across cubes and + assign the resulting output cubes to the output_products. Parameters ---------- @@ -438,9 +438,7 @@ def _multiproduct_statistics(products, Returns ------- set - set of PreprocessorFiles containing the computed - statistics cubes. - + set of PreprocessorFiles containing the computed statistics cubes. """ if engine == 'iris': aggregator = multicube_statistics_iris @@ -525,7 +523,9 @@ def multi_model_statistics(products, cubes are extracted from their products. Resulting cubes are added to their corresponding `output_products`. - Multi-model statistics are computed in `multicube_statistics`. + See also + -------- + multicube_statistics : core statistics function. """ return _grouped_multiproduct_statistics( products=products, @@ -542,7 +542,9 @@ def ensemble_statistics(products, statistics, output_products): The products are grouped and the cubes are extracted from the products. Resulting cubes are assigned to `output_products`. - Ensemble statistics are computed in `multicube_statistics_iris`. + See also + -------- + multicube_statistics_iris : core statistics function. """ ensemble_grouping = ('project', 'dataset', 'exp') return _grouped_multiproduct_statistics( From 3a72d58fb13f66a8fe29f5c3983d41542c62a80b Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 10:39:16 +0200 Subject: [PATCH 086/158] Raise Exception instead of logging it This will expose the error if the function is called directly --- esmvalcore/preprocessor/_multimodel.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 50217d5732..2a540ffcf9 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -396,13 +396,14 @@ def multicube_statistics_iris(cubes, statistics): statistics_cubes = {} for statistic in statistics: try: - operator = operators.get(statistic.upper()) + operator = operators[statistic.upper()] except KeyError: - logger.error( - 'Statistic %s not supported in ensemble_statistics. ' - 'Choose supported operator from iris.analysis package.', - statistic) + raise KeyError( + f'Statistic {statistic} not supported in', + '`ensemble_statistics`.', + 'Choose supported operator from `iris.analysis package`.') + # this will always return a masked array statistic_cube = cube.collapsed('ens', operator) statistics_cubes[statistic] = statistic_cube From d8f467588fb5d2c7fbf048a5f99da4c6fd1152de Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 10:40:16 +0200 Subject: [PATCH 087/158] Avoid cubes being updated in-place Calling the function in succession causes an exception, because the attributes have already been equalized --- esmvalcore/preprocessor/_multimodel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 2a540ffcf9..285960bd31 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -14,6 +14,7 @@ import logging import re +import copy from collections import defaultdict from datetime import datetime from functools import partial, reduce @@ -384,6 +385,8 @@ def multicube_statistics_iris(cubes, statistics): """ operators = vars(iris.analysis) + cubes = copy.deepcopy(cubes) + for i, cube in enumerate(cubes): concat_dim = iris.coords.AuxCoord(i, var_name='ens') cube.add_aux_coord(concat_dim) From 92f8c7a72f7fadc70ef4336d37c1911158632ff0 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 10:41:03 +0200 Subject: [PATCH 088/158] Add tests for multi-model statistics --- .../_multimodel/test_multimodel.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 10c80fdf02..9f4ddf63a0 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -5,6 +5,7 @@ import iris import numpy as np from cf_units import Unit +import pytest import tests from esmvalcore.preprocessor._multimodel import (_assemble_data, @@ -12,7 +13,9 @@ _get_time_slice, _plev_fix, _put_in_cube, _unify_time_coordinates, - multicube_statistics) + multicube_statistics, + multicube_statistics_iris, + ) class Test(tests.Test): @@ -126,6 +129,31 @@ def test_compute_overlap_statistic_yr_cube(self): expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) + def test_multicube_statistics_fail(self): + data = [self.cube1, self.cube1*2.0] + with pytest.raises(ValueError): + stats = multicube_statistics( + data, + span='overlap', + statistics=['non-existant'] + ) + + def test_multicube_statistics_iris(self): + data = [self.cube1, self.cube1*2.0] + statistics = ['mean', 'min', 'max'] + stats = multicube_statistics_iris(data, statistics=statistics) + expected_mean = np.ma.ones((2, 3, 2, 2)) * 1.5 + expected_min = np.ma.ones((2, 3, 2, 2)) * 1.0 + expected_max = np.ma.ones((2, 3, 2, 2)) * 2.0 + self.assert_array_equal(stats['mean'].data, expected_mean) + self.assert_array_equal(stats['min'].data, expected_min) + self.assert_array_equal(stats['max'].data, expected_max) + + def test_multicube_statistics_iris_fail(self): + data = [self.cube1, self.cube1*2.0] + with pytest.raises(KeyError): + multicube_statistics_iris(data, statistics=['non-existent']) + def test_compute_std(self): """Test statistic.""" data = [self.cube1.data[0], self.cube2.data[0] * 2] From 14c1baa629f4b9a3a273fd506247d836e25f7824 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 11:54:35 +0200 Subject: [PATCH 089/158] Add integration test for ensemble statistics --- tests/integration/test_recipe.py | 57 ++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index afd2b52ff0..ae5f0a2fdd 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1500,6 +1500,63 @@ def test_extract_shape_raises(tmp_path, patched_datafinder, config_user, assert invalid_arg in str(exc.value) +def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): + from collections import defaultdict + + content = dedent(""" + preprocessors: + default: &default + custom_order: true + area_statistics: + operator: mean + ensemble_statistics: + statistics: ['mean', 'median'] + + diagnostics: + diagnostic_name: + variables: + pr: + project: CMIP5 + mip: Amon + start_year: 2000 + end_year: 2002 + preprocessor: default + additional_datasets: + - {dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"} + - {dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"} + scripts: null + """) + + recipe = get_recipe(tmp_path, content, config_user) + variable = recipe.diagnostics['diagnostic_name']['preprocessor_output']['pr'] + datasets = set([var['dataset'] for var in variable]) + + assert len(recipe.tasks) == 1 + products = next(iter(recipe.tasks)).products + assert len(products) == 4 + + statistics = ['mean', 'median'] + product_out = defaultdict(list) + + for i, product in enumerate(products): + settings = product.settings['ensemble_statistics'] + assert settings['statistics'] == statistics + output_products = settings['output_products'] + assert isinstance(output_products, dict) + + for identifier, statistic_out in output_products.items(): + assert isinstance(statistic_out, dict) + assert all(s in statistics for s in statistic_out) + for statistic, preproc_file in statistic_out.items(): + product_out[identifier, statistic].append(preproc_file) + + # Make sure that output products are consistent + for key, value in product_out.items(): + assert len(set(value)) == 1, 'Output products are not equal' + + assert len(product_out) == len(datasets) * len(statistics) + + def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): content = dedent(""" preprocessors: From 40fabbb396bb709a531934f774e5f47f66eab326 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 14:25:19 +0200 Subject: [PATCH 090/158] Add integration test for multi model statistics --- tests/integration/test_recipe.py | 66 +++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index ae5f0a2fdd..0993813394 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1,4 +1,5 @@ import os +from collections import defaultdict from pathlib import Path from pprint import pformat from textwrap import dedent @@ -1500,9 +1501,21 @@ def test_extract_shape_raises(tmp_path, patched_datafinder, config_user, assert invalid_arg in str(exc.value) -def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): - from collections import defaultdict +def _get_output_preproc_files(products, preprocessor): + product_out = defaultdict(list) + + for i, product in enumerate(products): + settings = product.settings[preprocessor] + output_products = settings['output_products'] + + for identifier, statistic_out in output_products.items(): + for statistic, preproc_file in statistic_out.items(): + product_out[identifier, statistic].append(preproc_file) + + return product_out + +def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): content = dedent(""" preprocessors: default: &default @@ -1557,6 +1570,55 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): assert len(product_out) == len(datasets) * len(statistics) +def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): + statistics = ['mean', 'max'] + diagnostic = 'diagnostic_name' + variable = 'pr' + + content = dedent(f""" + preprocessors: + default: &default + custom_order: true + area_statistics: + operator: mean + multi_model_statistics: + span: overlap + statistics: {statistics} + + diagnostics: + {diagnostic}: + variables: + {variable}: + project: CMIP5 + mip: Amon + start_year: 2000 + end_year: 2002 + preprocessor: default + additional_datasets: + - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} + - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} + scripts: null + """) + + recipe = _get_recipe(tmp_path, content, config_user) + variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] + datasets = set([var['dataset'] for var in variable]) + + assert len(recipe.tasks) == 1 + products = next(iter(recipe.tasks)).products + assert len(products) == 4 + + product_out = get_output_preproc_files(products, 'multi_model_statistics') + + # Make sure that output products are consistent + for (identifier, statistic), value in product_out.items(): + assert identifier == '' + assert statistic in statistics + assert len(set(value)) == 1, 'Output products are not equal' + + assert len(product_out) == len(statistics) + + def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): content = dedent(""" preprocessors: From e2db41639461dabea032885aa46c419f2e32bcd1 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 14:33:12 +0200 Subject: [PATCH 091/158] Refactor multi_model/ensemble statistics --- tests/integration/test_recipe.py | 91 ++++++++++++++------------------ 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 0993813394..afe8a71692 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1515,56 +1515,47 @@ def _get_output_preproc_files(products, preprocessor): return product_out + def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): - content = dedent(""" + statistics = ['mean', 'max'] + diagnostic = 'diagnostic_name' + variable = 'pr' + preprocessor = 'ensemble_statistics' + + content = dedent(f""" preprocessors: default: &default custom_order: true area_statistics: operator: mean - ensemble_statistics: - statistics: ['mean', 'median'] + {preprocessor}: + statistics: {statistics} diagnostics: - diagnostic_name: + {diagnostic}: variables: - pr: + {variable}: project: CMIP5 mip: Amon start_year: 2000 end_year: 2002 preprocessor: default additional_datasets: - - {dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"} - - {dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"} + - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} + - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} scripts: null """) recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['diagnostic_name']['preprocessor_output']['pr'] + variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] datasets = set([var['dataset'] for var in variable]) - assert len(recipe.tasks) == 1 products = next(iter(recipe.tasks)).products - assert len(products) == 4 - - statistics = ['mean', 'median'] - product_out = defaultdict(list) - - for i, product in enumerate(products): - settings = product.settings['ensemble_statistics'] - assert settings['statistics'] == statistics - output_products = settings['output_products'] - assert isinstance(output_products, dict) - - for identifier, statistic_out in output_products.items(): - assert isinstance(statistic_out, dict) - assert all(s in statistics for s in statistic_out) - for statistic, preproc_file in statistic_out.items(): - product_out[identifier, statistic].append(preproc_file) + product_out = get_output_preproc_files(products, preprocessor) # Make sure that output products are consistent - for key, value in product_out.items(): + for (identifier, statistic), value in product_out.items(): + assert statistic in statistics assert len(set(value)) == 1, 'Output products are not equal' assert len(product_out) == len(datasets) * len(statistics) @@ -1574,41 +1565,39 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): statistics = ['mean', 'max'] diagnostic = 'diagnostic_name' variable = 'pr' + preprocessor = ['multi_model_statistics'] content = dedent(f""" - preprocessors: - default: &default - custom_order: true - area_statistics: - operator: mean - multi_model_statistics: - span: overlap - statistics: {statistics} + preprocessors: + default: &default + custom_order: true + area_statistics: + operator: mean + {preprocessor}: + span: overlap + statistics: {statistics} - diagnostics: - {diagnostic}: - variables: - {variable}: - project: CMIP5 - mip: Amon - start_year: 2000 - end_year: 2002 - preprocessor: default - additional_datasets: - - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} - - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} - scripts: null + diagnostics: + {diagnostic}: + variables: + {variable}: + project: CMIP5 + mip: Amon + start_year: 2000 + end_year: 2002 + preprocessor: default + additional_datasets: + - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} + - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} + scripts: null """) recipe = _get_recipe(tmp_path, content, config_user) variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] datasets = set([var['dataset'] for var in variable]) - assert len(recipe.tasks) == 1 products = next(iter(recipe.tasks)).products - assert len(products) == 4 - - product_out = get_output_preproc_files(products, 'multi_model_statistics') + product_out = get_output_preproc_files(products, preprocessor) # Make sure that output products are consistent for (identifier, statistic), value in product_out.items(): From 9a6bfc279727d2b70618bccb4449261b6333d325 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 14:44:24 +0200 Subject: [PATCH 092/158] Fix bugs in tests --- tests/integration/test_recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index afe8a71692..5ed0d16a37 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1551,7 +1551,7 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): datasets = set([var['dataset'] for var in variable]) products = next(iter(recipe.tasks)).products - product_out = get_output_preproc_files(products, preprocessor) + product_out = _get_output_preproc_files(products, preprocessor) # Make sure that output products are consistent for (identifier, statistic), value in product_out.items(): @@ -1565,7 +1565,7 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): statistics = ['mean', 'max'] diagnostic = 'diagnostic_name' variable = 'pr' - preprocessor = ['multi_model_statistics'] + preprocessor = 'multi_model_statistics' content = dedent(f""" preprocessors: From 14531d760b9b351e1e91367ff13ce1a4ac44bb63 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 14:55:58 +0200 Subject: [PATCH 093/158] Refactor output consistency check --- tests/integration/test_recipe.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 5ed0d16a37..b5c689a508 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1501,7 +1501,7 @@ def test_extract_shape_raises(tmp_path, patched_datafinder, config_user, assert invalid_arg in str(exc.value) -def _get_output_preproc_files(products, preprocessor): +def _test_output_product_consistency(products, preprocessor, statistics): product_out = defaultdict(list) for i, product in enumerate(products): @@ -1512,6 +1512,11 @@ def _get_output_preproc_files(products, preprocessor): for statistic, preproc_file in statistic_out.items(): product_out[identifier, statistic].append(preproc_file) + # Make sure that output products are consistent + for (identifier, statistic), value in product_out.items(): + assert statistic in statistics + assert len(set(value)) == 1, 'Output products are not equal' + return product_out @@ -1551,12 +1556,7 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): datasets = set([var['dataset'] for var in variable]) products = next(iter(recipe.tasks)).products - product_out = _get_output_preproc_files(products, preprocessor) - - # Make sure that output products are consistent - for (identifier, statistic), value in product_out.items(): - assert statistic in statistics - assert len(set(value)) == 1, 'Output products are not equal' + product_out = _test_output_product_consistency(products, preprocessor, statistics) assert len(product_out) == len(datasets) * len(statistics) @@ -1597,13 +1597,7 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): datasets = set([var['dataset'] for var in variable]) products = next(iter(recipe.tasks)).products - product_out = get_output_preproc_files(products, preprocessor) - - # Make sure that output products are consistent - for (identifier, statistic), value in product_out.items(): - assert identifier == '' - assert statistic in statistics - assert len(set(value)) == 1, 'Output products are not equal' + product_out = _test_output_product_consistency(products, preprocessor, statistics) assert len(product_out) == len(statistics) From c8a208b6262c6fce873d1576909e7fb65b0099d8 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 15:05:37 +0200 Subject: [PATCH 094/158] Fix typo --- tests/integration/test_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index b5c689a508..1c97c60417 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1592,7 +1592,7 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = _get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, config_user) variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] datasets = set([var['dataset'] for var in variable]) From 72f5cc973f8a3580f120fafd5862eb1c36fa214f Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 15:06:09 +0200 Subject: [PATCH 095/158] Add multi model integration test using groupby --- tests/integration/test_recipe.py | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 1c97c60417..438ca00f24 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1602,6 +1602,67 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): assert len(product_out) == len(statistics) +def test_groupby_ensemble_statistics(tmp_path, patched_datafinder, config_user): + diagnostic = 'diagnostic_name' + variable = 'pr' + + mm_statistics = ['mean', 'max'] + mm_preprocessor = 'multi_model_statistics' + ens_statistics = ['mean', 'median'] + ens_preprocessor = 'ensemble_statistics' + + groupby = [ens_preprocessor, 'tag'] + + content = dedent(f""" + preprocessors: + default: &default + custom_order: true + area_statistics: + operator: mean + {ens_preprocessor}: + statistics: {ens_statistics} + {mm_preprocessor}: + span: overlap + groupby: {groupby} + statistics: {mm_statistics} + + diagnostics: + {diagnostic}: + variables: + {variable}: + project: CMIP5 + mip: Amon + start_year: 2000 + end_year: 2002 + preprocessor: default + additional_datasets: + - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1", tag: group1}} + - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1", tag: group2}} + scripts: null + """) + + recipe = get_recipe(tmp_path, content, config_user) + variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] + datasets = set([var['dataset'] for var in variable]) + + products = next(iter(recipe.tasks)).products + + ens_products = _test_output_product_consistency( + products, + ens_preprocessor, + ens_statistics, + ) + + mm_products = _test_output_product_consistency( + products, + mm_preprocessor, + mm_statistics, + ) + + assert len(ens_products) == len(datasets) * len(ens_statistics) + assert len(mm_products) == len(mm_statistics) * len(ens_statistics) * len(groupby) + + def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): content = dedent(""" preprocessors: From 3d1b7b40ac75e75edb8f97a90e924e8b89a5faa0 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 15:31:57 +0200 Subject: [PATCH 096/158] Remove redundant function --- esmvalcore/preprocessor/_multimodel.py | 49 +++++++++----------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 285960bd31..8b5f6ea6fa 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -489,13 +489,22 @@ def _group(products, groupby=None): return grouped_products -def _grouped_multiproduct_statistics(products, - statistics: list, - output_products, - groupby, - span='overlap', - engine='esmvalcore'): - """Apply _multiproduct_statistics on grouped products.""" +def multi_model_statistics(products, + statistics: list, + output_products, + groupby=None, + span='overlap', + engine='esmvalcore'): + """ESMValCore entry point for multi model statistics. + + The products are grouped (if groupby argument is specified) and the + cubes are extracted from their products. Resulting cubes are added to + their corresponding `output_products`. + + See also + -------- + multicube_statistics : core statistics function. + """ grouped_products = _group(products, groupby=groupby) statistics_products = set() @@ -516,30 +525,6 @@ def _grouped_multiproduct_statistics(products, return statistics_products -def multi_model_statistics(products, - statistics, - output_products, - span='overlap', - groupby=None): - """ESMValCore entry point for multi model statistics. - - The products are grouped (if groupby argument is specified) and the - cubes are extracted from their products. Resulting cubes are added to - their corresponding `output_products`. - - See also - -------- - multicube_statistics : core statistics function. - """ - return _grouped_multiproduct_statistics( - products=products, - statistics=statistics, - output_products=output_products, - groupby=groupby, - span=span, - ) - - def ensemble_statistics(products, statistics, output_products): """ESMValCore entry point for ensemble statistics. @@ -551,7 +536,7 @@ def ensemble_statistics(products, statistics, output_products): multicube_statistics_iris : core statistics function. """ ensemble_grouping = ('project', 'dataset', 'exp') - return _grouped_multiproduct_statistics( + return multi_model_statistics( products=products, statistics=statistics, output_products=output_products, From f7e7b7b8d0be49baf41a868611df87f6d05db5e4 Mon Sep 17 00:00:00 2001 From: Stef Date: Fri, 11 Sep 2020 16:01:04 +0200 Subject: [PATCH 097/158] Refactor groupby function --- esmvalcore/_recipe.py | 17 +++---------- esmvalcore/preprocessor/_multimodel.py | 21 +++------------- esmvalcore/preprocessor/_other.py | 34 ++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index c7896d2310..9302e11cff 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -27,6 +27,7 @@ from .preprocessor._derive import get_required from .preprocessor._download import synda_search from .preprocessor._io import DATASET_KEYS, concatenate_callback +from .preprocessor._other import _groupby from .preprocessor._regrid import (get_cmor_levels, get_reference_levels, parse_cell_spec) @@ -615,17 +616,7 @@ def _update_multi_dataset_settings(variable, settings): _exclude_dataset(settings, variable, step) -def groupby(iterable, keyfunc: callable) -> dict: - """Group iterable by key function.""" - grouped = defaultdict(set) - for item in iterable: - key = keyfunc(item) - grouped[key].add(item) - - return grouped - - -def get_tag(step, identifier, statistic): +def _get_tag(step, identifier, statistic): # Avoid . in filename for percentiles statistic = statistic.replace('.', '-') @@ -663,7 +654,7 @@ def _update_multiproduct(input_products, order, preproc_dir, step): downstream_settings = _get_downstream_settings(step, order, products) - grouped_products = groupby(products, keyfunc=lambda p: p.group(grouping)) + grouped_products = _groupby(products, keyfunc=lambda p: p.group(grouping)) relevant_settings = { 'output_products': defaultdict(dict) @@ -675,7 +666,7 @@ def _update_multiproduct(input_products, order, preproc_dir, step): for statistic in settings.get('statistics'): - tag = get_tag(step, identifier, statistic) + tag = _get_tag(step, identifier, statistic) common_attributes[step] = tag diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 8b5f6ea6fa..ed75bae779 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -12,10 +12,9 @@ grouped execution by passing a groupby keyword. """ +import copy import logging import re -import copy -from collections import defaultdict from datetime import datetime from functools import partial, reduce @@ -25,6 +24,8 @@ import scipy from iris.experimental.equalise_cubes import equalise_attributes +from ._other import _groupby + logger = logging.getLogger(__name__) @@ -475,20 +476,6 @@ def _multiproduct_statistics(products, return statistics_products -def _group(products, groupby=None): - """Group products. - - Returns a dict of product sets with identifiers as keys. - """ - grouped_products = defaultdict(set) - for product in products: - identifier = product.group(groupby) - - grouped_products[identifier].add(product) - - return grouped_products - - def multi_model_statistics(products, statistics: list, output_products, @@ -505,7 +492,7 @@ def multi_model_statistics(products, -------- multicube_statistics : core statistics function. """ - grouped_products = _group(products, groupby=groupby) + grouped_products = _groupby(products, keyfunc=lambda p: p.group(groupby)) statistics_products = set() for identifier, products in grouped_products.items(): diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 697e8b3347..0fed73dfaf 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -1,8 +1,7 @@ -""" -Preprocessor functions that do not fit into any of the categories. -""" +"""Preprocessor functions that do not fit into any of the categories.""" import logging +from collections import defaultdict import dask.array as da @@ -10,8 +9,7 @@ def clip(cube, minimum=None, maximum=None): - """ - Clip values at a specified minimum and/or maximum value + """Clip values at a specified minimum and/or maximum value. Values lower than minimum are set to minimum and values higher than maximum are set to maximum. @@ -38,3 +36,29 @@ def clip(cube, minimum=None, maximum=None): raise ValueError("Maximum should be equal or larger than minimum.") cube.data = da.clip(cube.core_data(), minimum, maximum) return cube + + +def _groupby(iterable, keyfunc: callable) -> dict: + """Group iterable by key function. + + The items are grouped by the value that is returned by the `keyfunc` + + Parameters + ---------- + iterable : list, tuple or iterable + List of items to group + keyfunc : callable + Used to determine the group of each item. These become the keys + of the returned dictionary + + Returns + ------- + dict + Returns a dictionary with the grouped values. + """ + grouped = defaultdict(set) + for item in iterable: + key = keyfunc(item) + grouped[key].add(item) + + return grouped From 8e21e06aed10a42f9180f7b846220b0f9fe32da3 Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 11:21:16 +0200 Subject: [PATCH 098/158] Rename function --- tests/integration/test_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 438ca00f24..9b8f5c8937 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1602,7 +1602,7 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): assert len(product_out) == len(statistics) -def test_groupby_ensemble_statistics(tmp_path, patched_datafinder, config_user): +def test_groupby_combined_statistics(tmp_path, patched_datafinder, config_user): diagnostic = 'diagnostic_name' variable = 'pr' From 96988c0ba92927a60583049b7872a1fedcb4f74e Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 12:14:11 +0200 Subject: [PATCH 099/158] Clean function --- esmvalcore/preprocessor/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index e2002c72df..e5937557ba 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -431,12 +431,12 @@ def _initialize_multiproduct_provenance(self, step): if input_products: statistic_products = set() - for inputs in input_products: - items = input_products[0].settings[step].get( - 'output_products', {}).items() + for input_product in input_products: + step_settings = input_product.settings[step] + output_products = step_settings.get('output_products', {}) - for dataset, products in items: - statistic_products.update(products.values()) + for product in output_products.values(): + statistic_products.update(product.values()) self._initialize_products(statistic_products) From 8ea0bce7920212a77f290b03ee625ac087456b16 Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 12:14:30 +0200 Subject: [PATCH 100/158] Fix Codacy issues --- esmvalcore/_provenance.py | 1 + esmvalcore/_recipe.py | 4 +-- esmvalcore/preprocessor/__init__.py | 42 +++++++++++++------------- esmvalcore/preprocessor/_multimodel.py | 30 +++++++++--------- tests/integration/test_recipe.py | 32 ++++++++++++-------- 5 files changed, 59 insertions(+), 50 deletions(-) diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index e4ce3f8d51..db1f4bf582 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -109,6 +109,7 @@ def get_task_provenance(task, recipe_entity): class TrackedFile(object): """File with provenance tracking.""" + def __init__(self, filename, attributes, ancestors=None): """Create an instance of a file with provenance tracking.""" self._filename = filename diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 9302e11cff..ad18204b78 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -480,7 +480,6 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars): def _update_fx_settings(settings, variable, config_user): """Update fx settings depending on the needed method.""" - # get fx variables either from user defined attribute or fixed def _get_fx_vars_from_attribute(step_settings, step_name): user_fx_vars = step_settings.get('fx_variables') @@ -595,8 +594,7 @@ def _get_common_attributes(products): def _get_downstream_settings(step, order, products): - """Get downstream preprocessor settings that are shared between - products.""" + """Get downstream preprocessor settings shared between products.""" settings = {} remaining_steps = order[order.index(step) + 1:] some_product = next(iter(products)) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index e5937557ba..cd8413bb34 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -273,14 +273,14 @@ def get_step_blocks(steps, order): class PreprocessorFile(TrackedFile): """Preprocessor output file.""" - def __init__( - self, - attributes, - settings, - ancestors=None, + + def __init__(self, + attributes, + settings, + ancestors=None, ): - super(PreprocessorFile, self).__init__(attributes['filename'], - attributes, ancestors) + super().__init__(attributes['filename'], + attributes, ancestors) self.settings = copy.deepcopy(settings) if 'save' not in self.settings: @@ -350,7 +350,7 @@ def is_closed(self): def _initialize_entity(self): """Initialize the entity representing the file.""" - super(PreprocessorFile, self)._initialize_entity() + super()._initialize_entity() settings = { 'preprocessor:' + k: str(v) for k, v in self.settings.items() @@ -404,14 +404,14 @@ def _apply_multimodel(products, step, debug): class PreprocessingTask(BaseTask): """Task for running the preprocessor.""" - def __init__( - self, - products, - ancestors=None, - name='', - order=DEFAULT_ORDER, - debug=None, - write_ncl_interface=False, + + def __init__(self, + products, + ancestors=None, + name='', + order=DEFAULT_ORDER, + debug=None, + write_ncl_interface=False, ): """Initialize.""" _check_multi_model_settings(products) @@ -423,8 +423,8 @@ def __init__( def _initialize_product_provenance(self): """Initialize product provenance.""" self._initialize_products(self.products) - self._initialize_multi_model_statistics_provenance() - self._initialize_ensemble_statistics_provenance() + self._initialize_multi_model_provenance() + self._initialize_ensemble_provenance() def _initialize_multiproduct_provenance(self, step): input_products = self._get_input_products(step) @@ -440,12 +440,12 @@ def _initialize_multiproduct_provenance(self, step): self._initialize_products(statistic_products) - def _initialize_multi_model_statistics_provenance(self): + def _initialize_multi_model_provenance(self): """Initialize provenance for multi-model statistics.""" step = 'multi_model_statistics' self._initialize_multiproduct_provenance(step) - def _initialize_ensemble_statistics_provenance(self): + def _initialize_ensemble_provenance(self): """Initialize provenance for ensemble statistics.""" step = 'ensemble_statistics' self._initialize_multiproduct_provenance(step) @@ -503,6 +503,6 @@ def __str__(self): self.__class__.__name__, order, products, - super(PreprocessingTask, self).str(), + super().str(), ) return txt diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index ed75bae779..1518a9590a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -401,11 +401,11 @@ def multicube_statistics_iris(cubes, statistics): for statistic in statistics: try: operator = operators[statistic.upper()] - except KeyError: - raise KeyError( + except KeyError as e: + raise ValueError( f'Statistic {statistic} not supported in', - '`ensemble_statistics`.', - 'Choose supported operator from `iris.analysis package`.') + '`ensemble_statistics`. Choose supported operator from', + '`iris.analysis package`.') from e # this will always return a masked array statistic_cube = cube.collapsed('ens', operator) @@ -476,13 +476,13 @@ def _multiproduct_statistics(products, return statistics_products -def multi_model_statistics(products, +def multi_model_statistics(input_products: set, statistics: list, - output_products, - groupby=None, - span='overlap', - engine='esmvalcore'): - """ESMValCore entry point for multi model statistics. + output_products: set, + groupby: str=None, + span: str='overlap', + engine: str='esmvalcore'): + """Entry point for multi model statistics. The products are grouped (if groupby argument is specified) and the cubes are extracted from their products. Resulting cubes are added to @@ -492,7 +492,7 @@ def multi_model_statistics(products, -------- multicube_statistics : core statistics function. """ - grouped_products = _groupby(products, keyfunc=lambda p: p.group(groupby)) + grouped_products = _groupby(input_products, keyfunc=lambda p: p.group(groupby)) statistics_products = set() for identifier, products in grouped_products.items(): @@ -512,8 +512,10 @@ def multi_model_statistics(products, return statistics_products -def ensemble_statistics(products, statistics, output_products): - """ESMValCore entry point for ensemble statistics. +def ensemble_statistics(input_products: set, + statistics: list, + output_products: set): + """Entry point for ensemble statistics. The products are grouped and the cubes are extracted from the products. Resulting cubes are assigned to `output_products`. @@ -524,7 +526,7 @@ def ensemble_statistics(products, statistics, output_products): """ ensemble_grouping = ('project', 'dataset', 'exp') return multi_model_statistics( - products=products, + input_products=input_products, statistics=statistics, output_products=output_products, groupby=ensemble_grouping, diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 9b8f5c8937..8cbe1cd23b 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1520,7 +1520,6 @@ def _test_output_product_consistency(products, preprocessor, statistics): return product_out - def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): statistics = ['mean', 'max'] diagnostic = 'diagnostic_name' @@ -1546,8 +1545,10 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): end_year: 2002 preprocessor: default additional_datasets: - - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} - - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} + - {{dataset: CanESM2, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1"}} + - {{dataset: CCSM4, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1"}} scripts: null """) @@ -1556,7 +1557,8 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): datasets = set([var['dataset'] for var in variable]) products = next(iter(recipe.tasks)).products - product_out = _test_output_product_consistency(products, preprocessor, statistics) + product_out = _test_output_product_consistency( + products, preprocessor, statistics) assert len(product_out) == len(datasets) * len(statistics) @@ -1587,22 +1589,25 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): end_year: 2002 preprocessor: default additional_datasets: - - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} - - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1"}} + - {{dataset: CanESM2, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1"}} + - {{dataset: CCSM4, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1"}} scripts: null """) recipe = get_recipe(tmp_path, content, config_user) variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] - datasets = set([var['dataset'] for var in variable]) products = next(iter(recipe.tasks)).products - product_out = _test_output_product_consistency(products, preprocessor, statistics) + product_out = _test_output_product_consistency( + products, preprocessor, statistics) assert len(product_out) == len(statistics) -def test_groupby_combined_statistics(tmp_path, patched_datafinder, config_user): +def test_groupby_combined_statistics( + tmp_path, patched_datafinder, config_user): diagnostic = 'diagnostic_name' variable = 'pr' @@ -1636,8 +1641,10 @@ def test_groupby_combined_statistics(tmp_path, patched_datafinder, config_user): end_year: 2002 preprocessor: default additional_datasets: - - {{dataset: CanESM2, exp: [historical, rcp45], ensemble: "r(1:2)i1p1", tag: group1}} - - {{dataset: CCSM4, exp: [historical, rcp45], ensemble: "r(1:2)i1p1", tag: group2}} + - {{dataset: CanESM2, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1", tag: group1}} + - {{dataset: CCSM4, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1", tag: group2}} scripts: null """) @@ -1660,7 +1667,8 @@ def test_groupby_combined_statistics(tmp_path, patched_datafinder, config_user): ) assert len(ens_products) == len(datasets) * len(ens_statistics) - assert len(mm_products) == len(mm_statistics) * len(ens_statistics) * len(groupby) + assert len(mm_products) == len( + mm_statistics) * len(ens_statistics) * len(groupby) def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): From d16836d3c8e5fa3b2074ec58c97d2b649f6e32b0 Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 13:04:12 +0200 Subject: [PATCH 101/158] Fix Codacy issues --- esmvalcore/preprocessor/__init__.py | 3 +-- esmvalcore/preprocessor/_multimodel.py | 13 +++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index cd8413bb34..94458dcd6c 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -277,8 +277,7 @@ class PreprocessorFile(TrackedFile): def __init__(self, attributes, settings, - ancestors=None, - ): + ancestors=None): super().__init__(attributes['filename'], attributes, ancestors) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 1518a9590a..a7be6479c1 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -479,20 +479,21 @@ def _multiproduct_statistics(products, def multi_model_statistics(input_products: set, statistics: list, output_products: set, - groupby: str=None, - span: str='overlap', - engine: str='esmvalcore'): + groupby: str = None, + span: str = 'overlap', + engine: str = 'esmvalcore'): """Entry point for multi model statistics. The products are grouped (if groupby argument is specified) and the cubes are extracted from their products. Resulting cubes are added to their corresponding `output_products`. - See also + See Also -------- multicube_statistics : core statistics function. """ - grouped_products = _groupby(input_products, keyfunc=lambda p: p.group(groupby)) + grouped_products = _groupby(input_products, + keyfunc=lambda p: p.group(groupby)) statistics_products = set() for identifier, products in grouped_products.items(): @@ -520,7 +521,7 @@ def ensemble_statistics(input_products: set, The products are grouped and the cubes are extracted from the products. Resulting cubes are assigned to `output_products`. - See also + See Also -------- multicube_statistics_iris : core statistics function. """ From 22286d45fff9cd1e6f94e85d725aea8f7d1facb8 Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 13:11:16 +0200 Subject: [PATCH 102/158] Fix Codacy issues --- esmvalcore/preprocessor/__init__.py | 3 +-- esmvalcore/preprocessor/_multimodel.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 94458dcd6c..b5506d0d67 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -410,8 +410,7 @@ def __init__(self, name='', order=DEFAULT_ORDER, debug=None, - write_ncl_interface=False, - ): + write_ncl_interface=False): """Initialize.""" _check_multi_model_settings(products) super().__init__(ancestors=ancestors, name=name, products=products) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index a7be6479c1..b8b7e50584 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -493,7 +493,7 @@ def multi_model_statistics(input_products: set, multicube_statistics : core statistics function. """ grouped_products = _groupby(input_products, - keyfunc=lambda p: p.group(groupby)) + keyfunc=lambda p: p.group(groupby)) statistics_products = set() for identifier, products in grouped_products.items(): From 3be28ed2edf1530d9adf19b0788757257c07fc6b Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 13:42:26 +0200 Subject: [PATCH 103/158] Fix problems with tests --- tests/unit/preprocessor/_multimodel/test_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 9f4ddf63a0..05c985fdc0 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -132,7 +132,7 @@ def test_compute_overlap_statistic_yr_cube(self): def test_multicube_statistics_fail(self): data = [self.cube1, self.cube1*2.0] with pytest.raises(ValueError): - stats = multicube_statistics( + multicube_statistics( data, span='overlap', statistics=['non-existant'] @@ -151,7 +151,7 @@ def test_multicube_statistics_iris(self): def test_multicube_statistics_iris_fail(self): data = [self.cube1, self.cube1*2.0] - with pytest.raises(KeyError): + with pytest.raises(ValueError): multicube_statistics_iris(data, statistics=['non-existent']) def test_compute_std(self): From becb7b71b87b6de755c675be1a6da18c6099e6f3 Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 13:44:17 +0200 Subject: [PATCH 104/158] Add `input_products` as valid itype for test This is used in multi_model statistics to differentiate between `input_products` and `output_products` --- tests/unit/preprocessor/test_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/preprocessor/test_runner.py b/tests/unit/preprocessor/test_runner.py index f0207b831a..4bc65c5b98 100644 --- a/tests/unit/preprocessor/test_runner.py +++ b/tests/unit/preprocessor/test_runner.py @@ -4,7 +4,8 @@ def test_first_argument_name(): """Check that the input type of all preprocessor functions is valid.""" - valid_itypes = ('file', 'files', 'cube', 'cubes', 'products') + valid_itypes = ('file', 'files', 'cube', 'cubes', 'products', + 'input_products') for step in DEFAULT_ORDER: itype = _get_itype(step) assert itype in valid_itypes, ( From d226a2175f66f7991275258357b5af43fa19aa4b Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 15:58:18 +0200 Subject: [PATCH 105/158] Add preprocessor argument checks for multimode/ensemble statistics --- esmvalcore/_recipe.py | 5 ++ esmvalcore/_recipe_checks.py | 92 ++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index ad18204b78..f187149606 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -480,6 +480,7 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars): def _update_fx_settings(settings, variable, config_user): """Update fx settings depending on the needed method.""" + # get fx variables either from user defined attribute or fixed def _get_fx_vars_from_attribute(step_settings, step_name): user_fx_vars = step_settings.get('fx_variables') @@ -798,6 +799,8 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, multi_model_step = 'multi_model_statistics' if ensemble_step in profile: + check.ensemble_statistics(settings[ensemble_step]) + ensemble_products, ensemble_settings = _update_multiproduct( products, order, preproc_dir, ensemble_step) @@ -811,6 +814,8 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ensemble_products = products if multi_model_step in profile: + check.multi_model_statistics(settings[multi_model_step]) + multimodel_products, multimodel_settings = _update_multiproduct( ensemble_products, order, preproc_dir, multi_model_step) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 4072c52f01..ecfd517dab 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -2,15 +2,15 @@ import itertools import logging import os +import re import subprocess from shutil import which -import re import yamale from ._data_finder import get_start_end_year from ._task import get_flattened_tasks -from .preprocessor import PreprocessingTask, TIME_PREPROCESSORS +from .preprocessor import TIME_PREPROCESSORS, PreprocessingTask logger = logging.getLogger(__name__) @@ -181,14 +181,86 @@ def extract_shape(settings): "{}".format(', '.join(f"'{k}'".lower() for k in valid[key]))) -def valid_multimodel_statistic(statistic): - """Check that `statistic` is a valid argument for multimodel stats.""" +def _validate_multi_model_statistics(statistics): + """Raise error if multi-model statistics cannot be validated.""" valid_names = ["mean", "median", "std", "min", "max"] valid_patterns = [r"^(p\d{1,2})(\.\d*)?$"] - if not (statistic in valid_names or - re.match(r'|'.join(valid_patterns), statistic)): + + for statistic in statistics: + if not (statistic in valid_names + or re.match(r'|'.join(valid_patterns), statistic)): + raise RecipeError( + "Invalid value encountered for `statistic` in preprocessor " + f"`multi_model_statistics`. Valid values are {valid_names} " + f"or patterns matching {valid_patterns}. Got '{statistic}.'") + + +def _validate_span_value(span): + """Raise error if span argument cannot be validated.""" + valid_names = ('overlap', 'full') + if span not in valid_names: raise RecipeError( - "Invalid value encountered for `statistic` in preprocessor " - f"`multi_model_statistics`. Valid values are {valid_names} " - f"or patterns matching {valid_patterns}. Got '{statistic}.'" - ) + "Invalid value encountered for `span` in preprocessor " + f"`multi_model_statistics`. Valid values are {valid_names}." + f"Got {span}.") + + +def _validate_groupby(groupby): + """Raise error if groupby arguments cannot be validated.""" + if not groupby: + return + if not isinstance(groupby, list): + raise RecipeError( + "Invalid value encountered for `groupby` in preprocessor. " + f"`groupby` must be defined as a list. Got {groupby}.") + + +def _validate_arguments(given, expected): + """Raise error if arguments cannot be validated.""" + for key in given: + if key not in expected: + raise RecipeError( + f"Unexpected keyword argument encountered: {key}. Valid " + "keywords are: {valid_keys}.") + + +def multi_model_statistics(settings): + """Check that the multi-model settings are valid.""" + valid_keys = ['span', 'groupby', 'statistics'] + _validate_arguments(settings.keys(), valid_keys) + + span = settings.get('span', None) # optional, default: overlap + if span: + _validate_span_value(span) + + groupby = settings.get('groupby', None) # optional, default: None + if groupby: + _validate_groupby(groupby) + + statistics = settings.get('statistics', None) # required + if statistics: + _validate_multi_model_statistics(statistics) + + +def _validate_ensemble_statistics(statistics): + """Raise error if ensemble statistics cannot be validated.""" + valid_names = ('count', 'gmean', 'hmean', 'max', 'mean', 'median', 'min', + 'peak', 'percentile', 'proportion', 'rms', 'std_dev', 'sum', + 'variance', 'wpercentile') + + for statistic in statistics: + if not statistic.lower() in valid_names: + raise RecipeError( + "Invalid value encountered for `statistic` in preprocessor " + f"`multi_model_statistics`. Valid values are {valid_names}." + f"Got '{statistic}.'") + + +def ensemble_statistics(settings): + """Check that the ensemble settings are valid.""" + valid_keys = ['statistics'] + _validate_arguments(settings.keys(), valid_keys) + + statistics = settings.get('statistics', None) + if statistics: + _validate_ensemble_statistics(statistics) From 88159432ddb211827e32f7d2fa1a51c60f46fa9e Mon Sep 17 00:00:00 2001 From: Stef Date: Mon, 14 Sep 2020 16:37:06 +0200 Subject: [PATCH 106/158] Code formatting --- esmvalcore/_provenance.py | 1 - esmvalcore/preprocessor/__init__.py | 10 +-- esmvalcore/preprocessor/_multimodel.py | 4 +- tests/integration/test_recipe.py | 72 +++++++++---------- .../_multimodel/test_multimodel.py | 42 ++++------- 5 files changed, 49 insertions(+), 80 deletions(-) diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index db1f4bf582..e4ce3f8d51 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -109,7 +109,6 @@ def get_task_provenance(task, recipe_entity): class TrackedFile(object): """File with provenance tracking.""" - def __init__(self, filename, attributes, ancestors=None): """Create an instance of a file with provenance tracking.""" self._filename = filename diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index b5506d0d67..f44e67b36a 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -273,13 +273,8 @@ def get_step_blocks(steps, order): class PreprocessorFile(TrackedFile): """Preprocessor output file.""" - - def __init__(self, - attributes, - settings, - ancestors=None): - super().__init__(attributes['filename'], - attributes, ancestors) + def __init__(self, attributes, settings, ancestors=None): + super().__init__(attributes['filename'], attributes, ancestors) self.settings = copy.deepcopy(settings) if 'save' not in self.settings: @@ -403,7 +398,6 @@ def _apply_multimodel(products, step, debug): class PreprocessingTask(BaseTask): """Task for running the preprocessor.""" - def __init__(self, products, ancestors=None, diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index b8b7e50584..d609a49b73 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -11,7 +11,6 @@ generalized functions that operate on iris cubes. These wrappers support grouped execution by passing a groupby keyword. """ - import copy import logging import re @@ -513,8 +512,7 @@ def multi_model_statistics(input_products: set, return statistics_products -def ensemble_statistics(input_products: set, - statistics: list, +def ensemble_statistics(input_products: set, statistics: list, output_products: set): """Entry point for ensemble statistics. diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 8cbe1cd23b..03f1e5817f 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -156,18 +156,22 @@ def find_files(_, filenames): @pytest.fixture def patched_tas_derivation(monkeypatch): - def get_required(short_name, _): if short_name != 'tas': assert False required = [ - {'short_name': 'pr'}, - {'short_name': 'areacella', 'mip': 'fx', 'optional': True}, + { + 'short_name': 'pr' + }, + { + 'short_name': 'areacella', + 'mip': 'fx', + 'optional': True + }, ] return required - monkeypatch.setattr( - esmvalcore._recipe, 'get_required', get_required) + monkeypatch.setattr(esmvalcore._recipe, 'get_required', get_required) DEFAULT_DOCUMENTATION = dedent(""" @@ -428,9 +432,8 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): preproc_dir = os.path.dirname(product.filename) assert preproc_dir.startswith(str(tmp_path)) - fix_dir = os.path.join( - preproc_dir, - 'CMIP5_CanESM2_fx_historical_r0i0p0_sftlf_fixed') + fix_dir = os.path.join(preproc_dir, + 'CMIP5_CanESM2_fx_historical_r0i0p0_sftlf_fixed') defaults = { 'load': { @@ -649,8 +652,7 @@ def test_cmip6_variable_autocomplete(tmp_path, patched_datafinder, assert variable[key] == reference[key] -def test_simple_cordex_recipe(tmp_path, patched_datafinder, - config_user): +def test_simple_cordex_recipe(tmp_path, patched_datafinder, config_user): """Test simple CORDEX recipe.""" content = dedent(""" diagnostics: @@ -1012,8 +1014,7 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): assert ancestor_product.filename in all_product_files -def test_derive_with_fx_ohc_fail(tmp_path, - patched_failing_datafinder, +def test_derive_with_fx_ohc_fail(tmp_path, patched_failing_datafinder, config_user): content = dedent(""" diagnostics: @@ -1039,10 +1040,8 @@ def test_derive_with_fx_ohc_fail(tmp_path, get_recipe(tmp_path, content, config_user) -def test_derive_with_optional_var(tmp_path, - patched_datafinder, - patched_tas_derivation, - config_user): +def test_derive_with_optional_var(tmp_path, patched_datafinder, + patched_tas_derivation, config_user): content = dedent(""" diagnostics: diagnostic_name: @@ -1080,8 +1079,7 @@ def test_derive_with_optional_var(tmp_path, # Check ancestors assert len(task.ancestors) == 2 - assert task.ancestors[0].name == ( - 'diagnostic_name/tas_derive_input_pr') + assert task.ancestors[0].name == ('diagnostic_name/tas_derive_input_pr') assert task.ancestors[1].name == ( 'diagnostic_name/tas_derive_input_areacella') for ancestor_product in task.ancestors[0].products: @@ -1092,10 +1090,8 @@ def test_derive_with_optional_var(tmp_path, assert ancestor_product.filename in all_product_files -def test_derive_with_optional_var_nodata(tmp_path, - patched_failing_datafinder, - patched_tas_derivation, - config_user): +def test_derive_with_optional_var_nodata(tmp_path, patched_failing_datafinder, + patched_tas_derivation, config_user): content = dedent(""" diagnostics: diagnostic_name: @@ -1133,8 +1129,7 @@ def test_derive_with_optional_var_nodata(tmp_path, # Check ancestors assert len(task.ancestors) == 1 - assert task.ancestors[0].name == ( - 'diagnostic_name/tas_derive_input_pr') + assert task.ancestors[0].name == ('diagnostic_name/tas_derive_input_pr') for ancestor_product in task.ancestors[0].products: assert ancestor_product.attributes['short_name'] == 'pr' assert ancestor_product.filename in all_product_files @@ -1214,10 +1209,10 @@ def simulate_diagnostic_run(diagnostic_task): def test_diagnostic_task_provenance( - tmp_path, - patched_datafinder, - monkeypatch, - config_user, + tmp_path, + patched_datafinder, + monkeypatch, + config_user, ): monkeypatch.setattr(esmvalcore._config, 'TAGS', TAGS) monkeypatch.setattr(esmvalcore._recipe, 'TAGS', TAGS) @@ -1557,8 +1552,8 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): datasets = set([var['dataset'] for var in variable]) products = next(iter(recipe.tasks)).products - product_out = _test_output_product_consistency( - products, preprocessor, statistics) + product_out = _test_output_product_consistency(products, preprocessor, + statistics) assert len(product_out) == len(datasets) * len(statistics) @@ -1600,14 +1595,14 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] products = next(iter(recipe.tasks)).products - product_out = _test_output_product_consistency( - products, preprocessor, statistics) + product_out = _test_output_product_consistency(products, preprocessor, + statistics) assert len(product_out) == len(statistics) -def test_groupby_combined_statistics( - tmp_path, patched_datafinder, config_user): +def test_groupby_combined_statistics(tmp_path, patched_datafinder, + config_user): diagnostic = 'diagnostic_name' variable = 'pr' @@ -1667,8 +1662,8 @@ def test_groupby_combined_statistics( ) assert len(ens_products) == len(datasets) * len(ens_statistics) - assert len(mm_products) == len( - mm_statistics) * len(ens_statistics) * len(groupby) + assert len( + mm_products) == len(mm_statistics) * len(ens_statistics) * len(groupby) def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): @@ -2409,9 +2404,8 @@ def test_wrong_project(tmp_path, patched_datafinder, config_user): - {dataset: CanESM2} scripts: null """) - msg = ( - "Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " - "with mip 'Omon'") + msg = ("Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " + "with mip 'Omon'") with pytest.raises(RecipeError) as wrong_proj: get_recipe(tmp_path, content, config_user) assert str(wrong_proj.value) == msg diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 05c985fdc0..682b93e02b 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -1,26 +1,20 @@ """Unit test for :func:`esmvalcore.preprocessor._multimodel`.""" - import unittest import iris import numpy as np -from cf_units import Unit import pytest +from cf_units import Unit import tests -from esmvalcore.preprocessor._multimodel import (_assemble_data, - _compute_statistic, - _get_time_slice, _plev_fix, - _put_in_cube, - _unify_time_coordinates, - multicube_statistics, - multicube_statistics_iris, - ) +from esmvalcore.preprocessor._multimodel import ( + _assemble_data, _compute_statistic, _get_time_slice, _plev_fix, + _put_in_cube, _unify_time_coordinates, multicube_statistics, + multicube_statistics_iris) class Test(tests.Test): """Test class for preprocessor/_multimodel.py.""" - def setUp(self): """Prepare tests.""" # Make various time arrays @@ -111,35 +105,25 @@ def test_compute_full_statistic_yr_cube(self): def test_compute_overlap_statistic_mon_cube(self): data = [self.cube1, self.cube1] - stats = multicube_statistics( - data, - span='overlap', - statistics=['mean'] - ) + stats = multicube_statistics(data, span='overlap', statistics=['mean']) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) def test_compute_overlap_statistic_yr_cube(self): data = [self.cube4, self.cube4] - stats = multicube_statistics( - data, - span='overlap', - statistics=['mean'] - ) + stats = multicube_statistics(data, span='overlap', statistics=['mean']) expected_ovlap_mean = np.ma.ones((2, 3, 2, 2)) self.assert_array_equal(stats['mean'].data, expected_ovlap_mean) def test_multicube_statistics_fail(self): - data = [self.cube1, self.cube1*2.0] + data = [self.cube1, self.cube1 * 2.0] with pytest.raises(ValueError): - multicube_statistics( - data, - span='overlap', - statistics=['non-existant'] - ) + multicube_statistics(data, + span='overlap', + statistics=['non-existant']) def test_multicube_statistics_iris(self): - data = [self.cube1, self.cube1*2.0] + data = [self.cube1, self.cube1 * 2.0] statistics = ['mean', 'min', 'max'] stats = multicube_statistics_iris(data, statistics=statistics) expected_mean = np.ma.ones((2, 3, 2, 2)) * 1.5 @@ -150,7 +134,7 @@ def test_multicube_statistics_iris(self): self.assert_array_equal(stats['max'].data, expected_max) def test_multicube_statistics_iris_fail(self): - data = [self.cube1, self.cube1*2.0] + data = [self.cube1, self.cube1 * 2.0] with pytest.raises(ValueError): multicube_statistics_iris(data, statistics=['non-existent']) From d04e65d147b63bfcdd59d97be307b0999cb2e6a6 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 15 Sep 2020 10:40:48 +0200 Subject: [PATCH 107/158] Refactor grouping function --- esmvalcore/_recipe.py | 10 +++------- esmvalcore/preprocessor/_multimodel.py | 7 ++----- esmvalcore/preprocessor/_other.py | 9 +++++++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index f187149606..68c4f3967b 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -27,7 +27,7 @@ from .preprocessor._derive import get_required from .preprocessor._download import synda_search from .preprocessor._io import DATASET_KEYS, concatenate_callback -from .preprocessor._other import _groupby +from .preprocessor._other import _group_products from .preprocessor._regrid import (get_cmor_levels, get_reference_levels, parse_cell_spec) @@ -653,21 +653,17 @@ def _update_multiproduct(input_products, order, preproc_dir, step): downstream_settings = _get_downstream_settings(step, order, products) - grouped_products = _groupby(products, keyfunc=lambda p: p.group(grouping)) - relevant_settings = { 'output_products': defaultdict(dict) } # pass to ancestors output_products = set() - for identifier, products in grouped_products.items(): + for identifier, products in _group_products(products, by=grouping): common_attributes = _get_common_attributes(products) for statistic in settings.get('statistics'): - tag = _get_tag(step, identifier, statistic) - - common_attributes[step] = tag + common_attributes[step] = _get_tag(step, identifier, statistic) filename = get_multiproduct_filename(common_attributes, preproc_dir) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index d609a49b73..4515ac737a 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -23,7 +23,7 @@ import scipy from iris.experimental.equalise_cubes import equalise_attributes -from ._other import _groupby +from ._other import _group_products logger = logging.getLogger(__name__) @@ -491,11 +491,8 @@ def multi_model_statistics(input_products: set, -------- multicube_statistics : core statistics function. """ - grouped_products = _groupby(input_products, - keyfunc=lambda p: p.group(groupby)) - statistics_products = set() - for identifier, products in grouped_products.items(): + for identifier, products in _group_products(input_products, by=groupby): sub_output_products = output_products[identifier] # Compute statistics on a single group diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 0fed73dfaf..671ab0ebe0 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -62,3 +62,12 @@ def _groupby(iterable, keyfunc: callable) -> dict: grouped[key].add(item) return grouped + + +def _group_products(products, by): + """Group products by the given list of attributes.""" + def grouper(product): + return product.group(by) + + grouped = _groupby(products, keyfunc=grouper) + return grouped.items() From 87f50281cbbc37e4f5062d73788b4b71881977af Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 15 Sep 2020 11:16:01 +0200 Subject: [PATCH 108/158] Refactor _match_products --- esmvalcore/_recipe.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 68c4f3967b..006d52cc96 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -703,31 +703,33 @@ def _update_extract_shape(settings, config_user): def _match_products(products, variables): """Match a list of input products to output product attributes.""" - grouped_products = {} + grouped_products = defaultdict(list) def get_matching(attributes): """Find the output filename which matches input attributes best.""" - score = 0 + best_score = 0 filenames = [] for variable in variables: filename = variable['filename'] - tmp = sum(v == variable.get(k) for k, v in attributes.items()) - if tmp > score: - score = tmp + score = sum(v == variable.get(k) for k, v in attributes.items()) + + if score > best_score: + best_score = score filenames = [filename] - elif tmp == score: + elif score == best_score: filenames.append(filename) + if not filenames: logger.warning( "Unable to find matching output file for input file %s", filename) + return filenames # Group input files by output file for product in products: - for filename in get_matching(product.attributes): - if filename not in grouped_products: - grouped_products[filename] = [] + matching_filenames = get_matching(product.attributes) + for filename in matching_filenames: grouped_products[filename].append(product) return grouped_products @@ -746,10 +748,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, for variable in variables: variable['filename'] = get_output_file(variable, preproc_dir) - if ancestor_products: - grouped_ancestors = _match_products(ancestor_products, variables) - else: - grouped_ancestors = {} + grouped_ancestors = _match_products(ancestor_products, variables) for variable in variables: settings = _get_default_settings( From b6fe30b60f93b90df151a8254c1ff55686dbfaf6 Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 15 Sep 2020 11:44:56 +0200 Subject: [PATCH 109/158] Update function names --- esmvalcore/_recipe.py | 2 +- esmvalcore/_recipe_checks.py | 34 ++++++++++++++--------------- esmvalcore/preprocessor/__init__.py | 4 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 006d52cc96..90a8a95167 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -809,7 +809,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ensemble_products = products if multi_model_step in profile: - check.multi_model_statistics(settings[multi_model_step]) + check.multimodel_statistics(settings[multi_model_step]) multimodel_products, multimodel_settings = _update_multiproduct( ensemble_products, order, preproc_dir, multi_model_step) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index ecfd517dab..3982b746c9 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -181,8 +181,8 @@ def extract_shape(settings): "{}".format(', '.join(f"'{k}'".lower() for k in valid[key]))) -def _validate_multi_model_statistics(statistics): - """Raise error if multi-model statistics cannot be validated.""" +def _verify_multimodel_statistics(statistics): + """Raise error if multi-model statistics cannot be verified.""" valid_names = ["mean", "median", "std", "min", "max"] valid_patterns = [r"^(p\d{1,2})(\.\d*)?$"] @@ -195,8 +195,8 @@ def _validate_multi_model_statistics(statistics): f"or patterns matching {valid_patterns}. Got '{statistic}.'") -def _validate_span_value(span): - """Raise error if span argument cannot be validated.""" +def _verify_span_value(span): + """Raise error if span argument cannot be verified.""" valid_names = ('overlap', 'full') if span not in valid_names: raise RecipeError( @@ -205,8 +205,8 @@ def _validate_span_value(span): f"Got {span}.") -def _validate_groupby(groupby): - """Raise error if groupby arguments cannot be validated.""" +def _verify_groupby(groupby): + """Raise error if groupby arguments cannot be verified.""" if not groupby: return if not isinstance(groupby, list): @@ -215,8 +215,8 @@ def _validate_groupby(groupby): f"`groupby` must be defined as a list. Got {groupby}.") -def _validate_arguments(given, expected): - """Raise error if arguments cannot be validated.""" +def _verify_arguments(given, expected): + """Raise error if arguments cannot be verified.""" for key in given: if key not in expected: raise RecipeError( @@ -224,26 +224,26 @@ def _validate_arguments(given, expected): "keywords are: {valid_keys}.") -def multi_model_statistics(settings): +def multimodel_statistics(settings): """Check that the multi-model settings are valid.""" valid_keys = ['span', 'groupby', 'statistics'] - _validate_arguments(settings.keys(), valid_keys) + _verify_arguments(settings.keys(), valid_keys) span = settings.get('span', None) # optional, default: overlap if span: - _validate_span_value(span) + _verify_span_value(span) groupby = settings.get('groupby', None) # optional, default: None if groupby: - _validate_groupby(groupby) + _verify_groupby(groupby) statistics = settings.get('statistics', None) # required if statistics: - _validate_multi_model_statistics(statistics) + _verify_multimodel_statistics(statistics) -def _validate_ensemble_statistics(statistics): - """Raise error if ensemble statistics cannot be validated.""" +def _verify_ensemble_statistics(statistics): + """Raise error if ensemble statistics cannot be verified.""" valid_names = ('count', 'gmean', 'hmean', 'max', 'mean', 'median', 'min', 'peak', 'percentile', 'proportion', 'rms', 'std_dev', 'sum', 'variance', 'wpercentile') @@ -259,8 +259,8 @@ def _validate_ensemble_statistics(statistics): def ensemble_statistics(settings): """Check that the ensemble settings are valid.""" valid_keys = ['statistics'] - _validate_arguments(settings.keys(), valid_keys) + _verify_arguments(settings.keys(), valid_keys) statistics = settings.get('statistics', None) if statistics: - _validate_ensemble_statistics(statistics) + _verify_ensemble_statistics(statistics) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index f44e67b36a..265068168e 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -415,7 +415,7 @@ def __init__(self, def _initialize_product_provenance(self): """Initialize product provenance.""" self._initialize_products(self.products) - self._initialize_multi_model_provenance() + self._initialize_multimodel_provenance() self._initialize_ensemble_provenance() def _initialize_multiproduct_provenance(self, step): @@ -432,7 +432,7 @@ def _initialize_multiproduct_provenance(self, step): self._initialize_products(statistic_products) - def _initialize_multi_model_provenance(self): + def _initialize_multimodel_provenance(self): """Initialize provenance for multi-model statistics.""" step = 'multi_model_statistics' self._initialize_multiproduct_provenance(step) From 0d447cfa668c7f49381f73ff5e6d034279d6d1fe Mon Sep 17 00:00:00 2001 From: Stef Date: Tue, 15 Sep 2020 11:54:49 +0200 Subject: [PATCH 110/158] Tweak error output and make error variable 3 letters --- esmvalcore/preprocessor/_multimodel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 4515ac737a..fe55ed96ae 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -400,11 +400,11 @@ def multicube_statistics_iris(cubes, statistics): for statistic in statistics: try: operator = operators[statistic.upper()] - except KeyError as e: + except KeyError as err: raise ValueError( - f'Statistic {statistic} not supported in', + f'Statistic `{statistic}` not supported in', '`ensemble_statistics`. Choose supported operator from', - '`iris.analysis package`.') from e + '`iris.analysis package`.') from err # this will always return a masked array statistic_cube = cube.collapsed('ens', operator) From d1125a494f9764f53fd4ef70a44ec611ee47066b Mon Sep 17 00:00:00 2001 From: Stef Date: Thu, 17 Sep 2020 13:08:11 +0200 Subject: [PATCH 111/158] Return empty dict in case products is None or empty --- esmvalcore/_recipe.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 90a8a95167..82411bbebe 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -705,6 +705,9 @@ def _match_products(products, variables): """Match a list of input products to output product attributes.""" grouped_products = defaultdict(list) + if not products: + return grouped_products + def get_matching(attributes): """Find the output filename which matches input attributes best.""" best_score = 0 From 640027013c6fb1d2f4a5ba3729330f9a7b62fe60 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Wed, 30 Sep 2020 15:19:30 +0200 Subject: [PATCH 112/158] add time bounds to output cube --- esmvalcore/preprocessor/_multimodel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fe55ed96ae..6ed537248c 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -136,6 +136,8 @@ def _put_in_cube(template_cube, cube_data, statistic, t_axis): """Quick cube building and saving.""" tunits = template_cube.coord('time').units times = iris.coords.DimCoord(t_axis, standard_name='time', units=tunits) + times.bounds = None + times.guess_bounds() coord_names = [c.long_name for c in template_cube.coords()] coord_names.extend([c.standard_name for c in template_cube.coords()]) From 2d37a0185d230073e721856a86dac4910fb04d2c Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Thu, 14 Jan 2021 11:31:08 +0100 Subject: [PATCH 113/158] Formatting --- esmvalcore/_recipe.py | 25 +++++++++++++------ .../_multimodel/test_multimodel.py | 12 ++++++--- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index a0d63e9235..bc0c6d911b 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -3,7 +3,7 @@ import logging import os import re -from collections import OrderedDict, defaultdict +from collections import defaultdict from copy import deepcopy from pprint import pformat @@ -12,10 +12,18 @@ from . import __version__ from . import _recipe_checks as check -from ._config import (TAGS, get_activity, get_institutes, get_project_config, - replace_tags) -from ._data_finder import (get_input_filelist, get_multiproduct_filename, - get_output_file) +from ._config import ( + TAGS, + get_activity, + get_institutes, + get_project_config, + replace_tags, +) +from ._data_finder import ( + get_input_filelist, + get_multiproduct_filename, + get_output_file, +) from ._provenance import TrackedFile, get_recipe_provenance from ._recipe_checks import RecipeError from ._task import ( @@ -37,8 +45,11 @@ from .preprocessor._download import synda_search from .preprocessor._io import DATASET_KEYS, concatenate_callback from .preprocessor._other import _group_products -from .preprocessor._regrid import (get_cmor_levels, get_reference_levels, - parse_cell_spec) +from .preprocessor._regrid import ( + get_cmor_levels, + get_reference_levels, + parse_cell_spec, +) logger = logging.getLogger(__name__) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 682b93e02b..5b2befc43a 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -8,9 +8,15 @@ import tests from esmvalcore.preprocessor._multimodel import ( - _assemble_data, _compute_statistic, _get_time_slice, _plev_fix, - _put_in_cube, _unify_time_coordinates, multicube_statistics, - multicube_statistics_iris) + _assemble_data, + _compute_statistic, + _get_time_slice, + _plev_fix, + _put_in_cube, + _unify_time_coordinates, + multicube_statistics, + multicube_statistics_iris, +) class Test(tests.Test): From 95ab23b54a49b22606a6e96ce20952adac6200d4 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Jan 2022 15:12:41 +0100 Subject: [PATCH 114/158] Add number of input cubes as cell method comment --- esmvalcore/preprocessor/_multimodel.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 63a3d8d71a..3555921491 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -299,6 +299,15 @@ def _compute_eager(cubes: list, *, operator: iris.analysis.Aggregator, result_cube.data = np.ma.array(result_cube.data) result_cube.remove_coord(CONCAT_DIM) + if result_cube.cell_methods: + cell_method = result_cube.cell_methods[0] + result_cube.cell_methods = None + updated_method = iris.coords.CellMethod( + method=cell_method.method, + coords=cell_method.coord_names, + intervals=cell_method.intervals, + comments=f'input_cubes: {len(cubes)}') + result_cube.add_cell_method(updated_method) return result_cube @@ -325,7 +334,6 @@ def _multicube_statistics(cubes, statistics, span): result_cube = _compute_eager(aligned_cubes, operator=operator, **kwargs) - statistics_cubes[statistic] = result_cube return statistics_cubes From 43d37c482782276851d57f51da66d66c35b499f4 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Jan 2022 15:12:52 +0100 Subject: [PATCH 115/158] Fix test --- tests/unit/test_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_recipe.py b/tests/unit/test_recipe.py index c396fda499..d4d39cf6b0 100644 --- a/tests/unit/test_recipe.py +++ b/tests/unit/test_recipe.py @@ -339,6 +339,6 @@ def test_multi_model_filename(): PreprocessorFile(cube, 'B', {'timerange': '1989/1990'}), PreprocessorFile(cube, 'C', {'timerange': '1991/1992'}), ] - attributes = _recipe._get_statistic_attributes(products) + attributes = _recipe._get_common_attributes(products) assert 'timerange' in attributes assert attributes['timerange'] == '1989/1992' From e44dd55d51b80a309693840538190d228ef85ecb Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Jan 2022 16:05:48 +0100 Subject: [PATCH 116/158] Include sub_exp in ensemble grouping --- esmvalcore/_recipe.py | 2 +- esmvalcore/preprocessor/__init__.py | 9 +++++---- esmvalcore/preprocessor/_multimodel.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index bbc96afc77..17a3951a3f 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -737,7 +737,7 @@ def _update_multiproduct(input_products, order, preproc_dir, step): settings = list(products)[0].settings[step] if step == 'ensemble_statistics': - grouping = ['project', 'dataset', 'exp'] + grouping = ['project', 'dataset', 'exp', 'sub_experiment'] else: grouping = settings.get('groupby', None) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index a6bcb31122..0a4514976e 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -483,10 +483,11 @@ def group(self, keys: list) -> str: identifier = [] for key in keys: - attribute = self.attributes[key] - if isinstance(attribute, (list, tuple)): - attribute = '-'.join(attribute) - identifier.append(attribute) + attribute = self.attributes.get(key) + if attribute: + if isinstance(attribute, (list, tuple)): + attribute = '-'.join(attribute) + identifier.append(attribute) return '_'.join(identifier) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 3555921491..6401ea47b9 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -471,7 +471,7 @@ def ensemble_statistics(products, statistics, -------- multicube_statistics_iris : core statistics function. """ - ensemble_grouping = ('project', 'dataset', 'exp') + ensemble_grouping = ('project', 'dataset', 'exp', 'sub_experiment') return multi_model_statistics( products=products, span='overlap', From 238ccac793e5764034fd2f3b688dd389a559733b Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Jan 2022 12:43:02 +0100 Subject: [PATCH 117/158] Move ensemble stats to end of chain --- esmvalcore/preprocessor/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 0a4514976e..d48bf95596 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -111,8 +111,6 @@ 'mask_glaciated', # Mask landseaice, sftgif only 'mask_landseaice', - # Ensemble statistics - 'ensemble_statistics', # Regridding 'regrid', # Point interpolation @@ -160,6 +158,8 @@ 'linear_trend_stderr', # Convert units 'convert_units', + # Ensemble statistics + 'ensemble_statistics', # Multi model statistics 'multi_model_statistics', # Bias calculation From 75bf641050550e31502915a22e9fe8afbaa30e2d Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Jan 2022 13:33:33 +0100 Subject: [PATCH 118/158] Allow to exclude datasets --- esmvalcore/_recipe.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 17a3951a3f..916e48461a 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -3,8 +3,8 @@ import logging import os import re -from collections import defaultdict import warnings +from collections import defaultdict from copy import deepcopy from pathlib import Path from pprint import pformat @@ -737,8 +737,10 @@ def _update_multiproduct(input_products, order, preproc_dir, step): settings = list(products)[0].settings[step] if step == 'ensemble_statistics': + check.ensemble_statistics(settings) grouping = ['project', 'dataset', 'exp', 'sub_experiment'] else: + check.multimodel_statistics(settings) grouping = settings.get('groupby', None) downstream_settings = _get_downstream_settings(step, order, products) @@ -773,9 +775,10 @@ def _update_multiproduct(input_products, order, preproc_dir, step): def update_ancestors(ancestors, step, downstream_settings): """Retroactively add settings to ancestor products.""" for product in ancestors: - settings = product.settings[step] - for key, value in downstream_settings.items(): - settings[key] = value + if step in product.settings: + settings = product.settings[step] + for key, value in downstream_settings.items(): + settings[key] = value def _update_extract_shape(settings, config_user): @@ -903,7 +906,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, grouped_ancestors = _match_products(ancestor_products, variables) else: grouped_ancestors = {} - + missing_vars = set() for variable in variables: settings = _get_default_settings( @@ -942,7 +945,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, products.add(product) - if missing_vars: separator = "\n- " raise InputFilesNotFound( @@ -954,7 +956,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ensemble_step = 'ensemble_statistics' multi_model_step = 'multi_model_statistics' if ensemble_step in profile: - check.ensemble_statistics(settings[ensemble_step]) ensemble_products, ensemble_settings = _update_multiproduct( products, order, preproc_dir, ensemble_step) @@ -969,7 +970,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ensemble_products = products if multi_model_step in profile: - check.multimodel_statistics(settings[multi_model_step]) multimodel_products, multimodel_settings = _update_multiproduct( ensemble_products, order, preproc_dir, multi_model_step) From 2a4272c4d1a25ad86a3e84bb881ddb5835925e1f Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Jan 2022 14:38:05 +0100 Subject: [PATCH 119/158] Fix style issues --- esmvalcore/_recipe.py | 2 +- esmvalcore/preprocessor/_multimodel.py | 26 ++++++++++---------------- esmvalcore/preprocessor/_other.py | 2 +- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 916e48461a..100fb99b55 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -732,7 +732,7 @@ def _update_multiproduct(input_products, order, preproc_dir, step): """ products = {p for p in input_products if step in p.settings} if not products: - return input_products, dict() + return input_products, {} settings = list(products)[0].settings[step] diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 6401ea47b9..9ae04d33c0 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -3,15 +3,10 @@ This module contains functions to compute statistics across multiple cubes or products. -Ensemble statistics uses iris built in functions and support lazy evaluation. -Multi-model statistics uses custom functions that operate directly on numpy -arrays. - Wrapper functions separate esmvalcore internals, operating on products, from generalized functions that operate on iris cubes. These wrappers support grouped execution by passing a groupby keyword. """ -import copy import logging import re import warnings @@ -21,18 +16,13 @@ import cf_units import iris import numpy as np -import scipy - -from ._other import _group_products - -logger = logging.getLogger(__name__) - - from iris.util import equalise_attributes from esmvalcore.iris_helpers import date2num from esmvalcore.preprocessor import remove_fx_variables +from ._other import _group_products + logger = logging.getLogger(__name__) STATISTIC_MAPPING = { @@ -416,6 +406,8 @@ def multi_model_statistics(products, For internal use only. A dict with statistics names as keys and preprocessorfiles as values. If products are passed as input, the statistics cubes will be assigned to these output products. + groupby: str + Group products by a given tag or attribute. keep_input_datasets: bool If True, the output will include the input datasets. If False, only the computed statistics will be returned. @@ -441,15 +433,16 @@ def multi_model_statistics(products, if all(type(p).__name__ == 'PreprocessorFile' for p in products): # Avoid circular input: https://stackoverflow.com/q/16964467 statistics_products = set() - for identifier, products in _group_products(products, by=groupby): + for identifier, input_prods in _group_products(products, by=groupby): sub_output_products = output_products[identifier] # Compute statistics on a single group group_statistics = _multiproduct_statistics( - products=products, + products=input_prods, statistics=statistics, output_products=sub_output_products, - span=span + span=span, + keep_input_datasets=keep_input_datasets ) statistics_products |= group_statistics @@ -460,6 +453,7 @@ def multi_model_statistics(products, "iris.cube.Cube or esmvalcore.preprocessor.PreprocessorFile, " "got {}".format(products)) + def ensemble_statistics(products, statistics, output_products, keep_input_datasets=True): """Entry point for ensemble statistics. @@ -479,4 +473,4 @@ def ensemble_statistics(products, statistics, output_products=output_products, groupby=ensemble_grouping, keep_input_datasets=keep_input_datasets - ) \ No newline at end of file + ) diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 671ab0ebe0..3402469b73 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -38,7 +38,7 @@ def clip(cube, minimum=None, maximum=None): return cube -def _groupby(iterable, keyfunc: callable) -> dict: +def _groupby(iterable, keyfunc): """Group iterable by key function. The items are grouped by the value that is returned by the `keyfunc` From 59aa0fac94c3e865daf811937cf2424bae92e1c6 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Jan 2022 14:40:40 +0100 Subject: [PATCH 120/158] Add check for keep_input_datasets --- esmvalcore/_recipe_checks.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 19465d35d1..3b282a18ea 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -13,7 +13,6 @@ from ._data_finder import get_start_end_year from .exceptions import InputFilesNotFound, RecipeError from .preprocessor import TIME_PREPROCESSORS, PreprocessingTask -from .preprocessor._multimodel import STATISTIC_MAPPING logger = logging.getLogger(__name__) @@ -254,18 +253,26 @@ def _verify_groupby(groupby): f"`groupby` must be defined as a list. Got {groupby}.") +def _verify_keep_input_datasets(keep_input_datasets): + if not isinstance(keep_input_datasets, bool): + raise RecipeError( + "Invalid value encountered for `keep_input_datasets`." + f"Must be defined as a boolean. Got {keep_input_datasets}." + ) + + def _verify_arguments(given, expected): """Raise error if arguments cannot be verified.""" for key in given: if key not in expected: raise RecipeError( f"Unexpected keyword argument encountered: {key}. Valid " - "keywords are: {valid_keys}.") + "keywords are: {expected}.") def multimodel_statistics(settings): """Check that the multi-model settings are valid.""" - valid_keys = ['span', 'groupby', 'statistics'] + valid_keys = ['span', 'groupby', 'statistics', 'keep_input_datasets'] _verify_arguments(settings.keys(), valid_keys) span = settings.get('span', None) # optional, default: overlap @@ -280,6 +287,9 @@ def multimodel_statistics(settings): if statistics: _verify_multimodel_statistics(statistics) + keep_input_datasets = settings.get('keep_input_datasets', True) + _verify_keep_input_datasets(keep_input_datasets) + def _verify_ensemble_statistics(statistics): """Raise error if ensemble statistics cannot be verified.""" @@ -297,12 +307,17 @@ def _verify_ensemble_statistics(statistics): def ensemble_statistics(settings): """Check that the ensemble settings are valid.""" - valid_keys = ['statistics'] + valid_keys = ['statistics', 'keep_input_datasets'] _verify_arguments(settings.keys(), valid_keys) statistics = settings.get('statistics', None) if statistics: _verify_ensemble_statistics(statistics) + + keep_input_datasets = settings.get('keep_input_datasets', True) + _verify_keep_input_datasets(keep_input_datasets) + + def _check_delimiter(timerange): if len(timerange) != 2: raise RecipeError("Invalid value encountered for `timerange`. " From 0b670b8b49b5b0b457e3e24884290bfdf799be08 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Jan 2022 16:50:04 +0100 Subject: [PATCH 121/158] Fix test --- tests/unit/preprocessor/_multimodel/test_multimodel.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 597f12b34b..0352eef37e 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -1,6 +1,4 @@ """Unit test for :func:`esmvalcore.preprocessor._multimodel`.""" -import unittest -"""Unit test for :func:`esmvalcore.preprocessor._multimodel`""" from datetime import datetime @@ -524,7 +522,7 @@ def __init__(self, cube=None, attributes=None): def wasderivedfrom(self, product): pass - + def group(self, keys: list) -> str: """Generate group keyword. @@ -558,12 +556,12 @@ def test_return_products(): products = set([input1, input2]) output = PreprocessorFile() - output_products = {'mean': output} + output_products = {'': {'mean': output}} kwargs = { 'statistics': ['mean'], 'span': 'full', - 'output_products': output_products + 'output_products': output_products[''] } result1 = mm._multiproduct_statistics(products, @@ -577,6 +575,7 @@ def test_return_products(): assert result1 == set([input1, input2, output]) assert result2 == set([output]) + kwargs['output_products'] = output_products result3 = mm.multi_model_statistics(products, **kwargs) result4 = mm.multi_model_statistics(products, keep_input_datasets=False, From a558e6d8fe45fd678f50c9403d55da553153959c Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Jan 2022 17:21:30 +0100 Subject: [PATCH 122/158] Fix more tests --- esmvalcore/_provenance.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index 1b6a6cf8ba..81258c9058 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -136,8 +136,7 @@ def __init__(self, def __str__(self): """Return summary string.""" - name = Path(self._filename).name - return f'{self.__class__.__name__}({repr(name)})' + return "{}: {}".format(self.__class__.__name__, self.filename) def __repr__(self): """Return representation string (e.g., used by ``pformat``).""" From 4312d3d1a0a23236acca6c352874f41d76199a9b Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Jan 2022 17:23:39 +0100 Subject: [PATCH 123/158] Fix flake --- esmvalcore/_data_finder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index ef87371186..ee9b790571 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -1,5 +1,4 @@ """Data finder module for the ESMValTool.""" -import copy import glob import logging import os From b93686ce8b0ebb84a5f0e6c571ee0a490cad31de Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 21 Jan 2022 10:08:59 +0100 Subject: [PATCH 124/158] Remove unused import --- esmvalcore/_provenance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index 81258c9058..0c08fd6ef8 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -2,7 +2,6 @@ import copy import logging import os -from pathlib import Path from netCDF4 import Dataset from PIL import Image From 765a1540a9c7a5d2adfd2e7cf2a5ce2fb42fdd4b Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 21 Jan 2022 16:32:04 +0100 Subject: [PATCH 125/158] Improve test coverage in recipe checks --- esmvalcore/_recipe_checks.py | 9 ++-- tests/integration/test_recipe_checks.py | 59 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 3b282a18ea..74ea85bc70 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -230,7 +230,7 @@ def _verify_multimodel_statistics(statistics): raise RecipeError( "Invalid value encountered for `statistic` in preprocessor " f"`multi_model_statistics`. Valid values are {valid_names} " - f"or patterns matching {valid_patterns}. Got '{statistic}.'") + f"or patterns matching {valid_patterns}. Got '{statistic}'.") def _verify_span_value(span): @@ -249,8 +249,9 @@ def _verify_groupby(groupby): return if not isinstance(groupby, list): raise RecipeError( - "Invalid value encountered for `groupby` in preprocessor. " - f"`groupby` must be defined as a list. Got {groupby}.") + "Invalid value encountered for `groupby` in preprocessor " + "`multi_model_statistics`.`groupby` must be defined as a " + f"list. Got {groupby}.") def _verify_keep_input_datasets(keep_input_datasets): @@ -267,7 +268,7 @@ def _verify_arguments(given, expected): if key not in expected: raise RecipeError( f"Unexpected keyword argument encountered: {key}. Valid " - "keywords are: {expected}.") + f"keywords are: {expected}.") def multimodel_statistics(settings): diff --git a/tests/integration/test_recipe_checks.py b/tests/integration/test_recipe_checks.py index 3fc749d2ca..f386901ba3 100644 --- a/tests/integration/test_recipe_checks.py +++ b/tests/integration/test_recipe_checks.py @@ -260,3 +260,62 @@ def test_reference_for_bias_preproc_two_refs(): assert "],\nfound 2:\n[" in str(rec_err.value) assert ("].\nPlease also ensure that the reference dataset is " "not excluded with the 'exclude' option") in str(rec_err.value) + + +INVALID_MM_SETTINGS = { + 'wrong_parametre': 'wrong', + 'statistics': ['wrong'], + 'span': 'wrong', + 'groupby': 'wrong', + 'keep_input_datasets': 'wrong' + } + + +def test_invalid_multi_model_settings(): + valid_keys = ['span', 'groupby', 'statistics', 'keep_input_datasets'] + with pytest.raises(RecipeError) as rec_err: + check._verify_arguments(INVALID_MM_SETTINGS, valid_keys) + assert str(rec_err.value) == ( + "Unexpected keyword argument encountered: wrong_parametre. " + "Valid keywords are: " + "['span', 'groupby', 'statistics', 'keep_input_datasets'] .") + + +def test_invalid_multi_model_statistics(): + with pytest.raises(RecipeError) as rec_err: + check._verify_multimodel_statistics(INVALID_MM_SETTINGS['statistics']) + assert str(rec_err.value) == ( + "Invalid value encountered for `statistic` in preprocessor " + "`multi_model_statistics`. Valid values are " + "['mean', 'median', 'std', 'min', 'max'] " + "or patterns matching ['^(p\\\\d{1,2})(\\\\.\\\\d*)?$']. " + "Got 'wrong'.") + + +def test_invalid_multi_model_span(): + with pytest.raises(RecipeError) as rec_err: + check._verify_span_value(INVALID_MM_SETTINGS['span']) + assert str(rec_err.value) == ( + "Invalid value encountered for `span` in preprocessor " + "`multi_model_statistics`. Valid values are ('overlap', 'full')." + "Got wrong." + ) + + +def test_invalid_multi_model_groupy(): + with pytest.raises(RecipeError) as rec_err: + check._verify_groupby(INVALID_MM_SETTINGS['groupby']) + assert str(rec_err.value) == ( + 'Invalid value encountered for `groupby` in preprocessor ' + '`multi_model_statistics`.`groupby` must be defined ' + 'as a list. Got wrong.' + ) + + +def test_invalid_multi_model_keep_input(): + with pytest.raises(RecipeError) as rec_err: + check._verify_keep_input_datasets( + INVALID_MM_SETTINGS['keep_input_datasets']) + assert str(rec_err.value) == ( + 'Invalid value encountered for `keep_input_datasets`.' + 'Must be defined as a boolean. Got wrong.') From 121b5bdad30b934dab9cdb17a0aedd5ee94b5494 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 21 Jan 2022 17:12:06 +0100 Subject: [PATCH 126/158] Improve coverage in multimodel --- .../_multimodel/test_multimodel.py | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 0352eef37e..3b26ca2d6e 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -537,10 +537,11 @@ def group(self, keys: list) -> str: identifier = [] for key in keys: - attribute = self.attributes[key] - if isinstance(attribute, (list, tuple)): - attribute = '-'.join(attribute) - identifier.append(attribute) + attribute = self.attributes.get(key) + if attribute: + if isinstance(attribute, (list, tuple)): + attribute = '-'.join(attribute) + identifier.append(attribute) return '_'.join(identifier) @@ -585,6 +586,51 @@ def test_return_products(): assert result4 == result2 +def test_ensemble_products(): + cube1 = generate_cube_from_dates('monthly', fill_val=1) + cube2 = generate_cube_from_dates('monthly', fill_val=9) + + attributes1 = { + 'project': 'project', 'dataset': 'dataset', + 'exp': 'exp', 'ensemble': '1'} + input1 = PreprocessorFile(cube1, attributes=attributes1) + + attributes2 = { + 'project': 'project', 'dataset': 'dataset', + 'exp': 'exp', 'ensemble': '2'} + input2 = PreprocessorFile(cube2, attributes=attributes2) + + attributes3 = { + 'project': 'project', 'dataset': 'dataset2', + 'exp': 'exp', 'ensemble': '1'} + input3 = PreprocessorFile(cube1, attributes=attributes3) + + attributes4 = { + 'project': 'project', 'dataset': 'dataset2', + 'exp': 'exp', 'ensemble': '2'} + + input4 = PreprocessorFile(cube1, attributes=attributes4) + products = set([input1, input2, input3, input4]) + + output1 = PreprocessorFile() + output2 = PreprocessorFile() + output_products = { + 'project_dataset_exp': {'mean': output1}, + 'project_dataset2_exp': {'mean': output2}} + + kwargs = { + 'statistics': ['mean'], + 'output_products': output_products, + } + + result1 = mm.ensemble_statistics(products, **kwargs) + assert len(result1) == 6 + + result2 = mm.ensemble_statistics( + products, keep_input_datasets=False, **kwargs) + assert len(result2) == 2 + + def test_ignore_tas_scalar_height_coord(): """Ignore conflicting aux_coords for height in tas.""" tas_2m = generate_cube_from_dates("monthly") From e50e6a24b862e49c3a690628f0c1ab0f496c0b4f Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 21 Jan 2022 17:36:42 +0100 Subject: [PATCH 127/158] Remove space --- tests/integration/test_recipe_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_recipe_checks.py b/tests/integration/test_recipe_checks.py index f386901ba3..dc37fb05c7 100644 --- a/tests/integration/test_recipe_checks.py +++ b/tests/integration/test_recipe_checks.py @@ -278,7 +278,7 @@ def test_invalid_multi_model_settings(): assert str(rec_err.value) == ( "Unexpected keyword argument encountered: wrong_parametre. " "Valid keywords are: " - "['span', 'groupby', 'statistics', 'keep_input_datasets'] .") + "['span', 'groupby', 'statistics', 'keep_input_datasets'].") def test_invalid_multi_model_statistics(): From 2a1dd49e9eefa788562df561c597a94e5ad8c935 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sun, 23 Jan 2022 19:12:25 +0100 Subject: [PATCH 128/158] Generalise check --- esmvalcore/_recipe_checks.py | 24 ++++-------------------- tests/integration/test_recipe_checks.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 74ea85bc70..6fcd84bcb9 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -219,7 +219,7 @@ def extract_shape(settings): "{}".format(', '.join(f"'{k}'".lower() for k in valid[key]))) -def _verify_multimodel_statistics(statistics): +def _verify_statistics(statistics, step): """Raise error if multi-model statistics cannot be verified.""" valid_names = ["mean", "median", "std", "min", "max"] valid_patterns = [r"^(p\d{1,2})(\.\d*)?$"] @@ -229,7 +229,7 @@ def _verify_multimodel_statistics(statistics): or re.match(r'|'.join(valid_patterns), statistic)): raise RecipeError( "Invalid value encountered for `statistic` in preprocessor " - f"`multi_model_statistics`. Valid values are {valid_names} " + f"{step}. Valid values are {valid_names} " f"or patterns matching {valid_patterns}. Got '{statistic}'.") @@ -245,8 +245,6 @@ def _verify_span_value(span): def _verify_groupby(groupby): """Raise error if groupby arguments cannot be verified.""" - if not groupby: - return if not isinstance(groupby, list): raise RecipeError( "Invalid value encountered for `groupby` in preprocessor " @@ -286,26 +284,12 @@ def multimodel_statistics(settings): statistics = settings.get('statistics', None) # required if statistics: - _verify_multimodel_statistics(statistics) + _verify_statistics(statistics, 'multi_model_statistics') keep_input_datasets = settings.get('keep_input_datasets', True) _verify_keep_input_datasets(keep_input_datasets) -def _verify_ensemble_statistics(statistics): - """Raise error if ensemble statistics cannot be verified.""" - valid_names = ('count', 'gmean', 'hmean', 'max', 'mean', 'median', 'min', - 'peak', 'percentile', 'proportion', 'rms', 'std_dev', 'sum', - 'variance', 'wpercentile') - - for statistic in statistics: - if not statistic.lower() in valid_names: - raise RecipeError( - "Invalid value encountered for `statistic` in preprocessor " - f"`multi_model_statistics`. Valid values are {valid_names}." - f"Got '{statistic}.'") - - def ensemble_statistics(settings): """Check that the ensemble settings are valid.""" valid_keys = ['statistics', 'keep_input_datasets'] @@ -313,7 +297,7 @@ def ensemble_statistics(settings): statistics = settings.get('statistics', None) if statistics: - _verify_ensemble_statistics(statistics) + _verify_statistics(statistics, 'ensemble_statistics') keep_input_datasets = settings.get('keep_input_datasets', True) _verify_keep_input_datasets(keep_input_datasets) diff --git a/tests/integration/test_recipe_checks.py b/tests/integration/test_recipe_checks.py index dc37fb05c7..97b30b8c78 100644 --- a/tests/integration/test_recipe_checks.py +++ b/tests/integration/test_recipe_checks.py @@ -283,10 +283,11 @@ def test_invalid_multi_model_settings(): def test_invalid_multi_model_statistics(): with pytest.raises(RecipeError) as rec_err: - check._verify_multimodel_statistics(INVALID_MM_SETTINGS['statistics']) + check._verify_statistics( + INVALID_MM_SETTINGS['statistics'], 'multi_model_statistics') assert str(rec_err.value) == ( "Invalid value encountered for `statistic` in preprocessor " - "`multi_model_statistics`. Valid values are " + "multi_model_statistics. Valid values are " "['mean', 'median', 'std', 'min', 'max'] " "or patterns matching ['^(p\\\\d{1,2})(\\\\.\\\\d*)?$']. " "Got 'wrong'.") @@ -319,3 +320,14 @@ def test_invalid_multi_model_keep_input(): assert str(rec_err.value) == ( 'Invalid value encountered for `keep_input_datasets`.' 'Must be defined as a boolean. Got wrong.') + + +def test_invalid_ensemble_statistics(): + with pytest.raises(RecipeError) as rec_err: + check._verify_statistics(['wrong'], 'ensemble_statistics') + assert str(rec_err.value) == ( + "Invalid value encountered for `statistic` in preprocessor " + "ensemble_statistics. Valid values are " + "['mean', 'median', 'std', 'min', 'max'] " + "or patterns matching ['^(p\\\\d{1,2})(\\\\.\\\\d*)?$']. " + "Got 'wrong'.") From 718e76b8368006a9c8b283e05278afc6bb43dbaa Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sun, 23 Jan 2022 19:19:32 +0100 Subject: [PATCH 129/158] Add more tests in recipe --- tests/__init__.py | 1 + tests/unit/test_recipe.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index 55d5f563a8..41d3e3f830 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -79,5 +79,6 @@ def __init__(self, cubes, filename, attributes, **kwargs): self.cubes = cubes self.filename = filename self.attributes = attributes + self.settings = {} self.mock_ancestors = set() self.wasderivedfrom = mock.Mock(side_effect=self.mock_ancestors.add) diff --git a/tests/unit/test_recipe.py b/tests/unit/test_recipe.py index d4d39cf6b0..a98fed28c7 100644 --- a/tests/unit/test_recipe.py +++ b/tests/unit/test_recipe.py @@ -1,3 +1,5 @@ +from collections import defaultdict + import iris import numpy as np import pyesgf.search.results @@ -342,3 +344,22 @@ def test_multi_model_filename(): attributes = _recipe._get_common_attributes(products) assert 'timerange' in attributes assert attributes['timerange'] == '1989/1992' + + +def test_update_multiproduct_no_product(): + cube = iris.cube.Cube(np.array([1])) + products = [ + PreprocessorFile(cube, 'A', attributes=None, settings={'step': {}})] + order = ('load', 'save') + preproc_dir = '/preproc_dir' + step = 'multi_model_statistics' + output, settings = _recipe._update_multiproduct( + products, order, preproc_dir, step) + assert output == products + assert settings == {} + + +def test_match_products_no_product(): + variables = [{'var_name': 'var'}] + grouped_products = _recipe._match_products(None, variables) + assert grouped_products == defaultdict(list) From a8195257c68e7322432551ce33281927b448b930 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sun, 23 Jan 2022 20:08:39 +0100 Subject: [PATCH 130/158] Improve test coverage of PreprocessorFile --- tests/unit/preprocessor/_other/test_other.py | 24 +++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/unit/preprocessor/_other/test_other.py b/tests/unit/preprocessor/_other/test_other.py index 08a1ee26c0..5795c56db2 100644 --- a/tests/unit/preprocessor/_other/test_other.py +++ b/tests/unit/preprocessor/_other/test_other.py @@ -9,7 +9,8 @@ from iris.cube import Cube from numpy.testing import assert_array_equal -from esmvalcore.preprocessor._other import clip +from esmvalcore.preprocessor import PreprocessorFile +from esmvalcore.preprocessor._other import _group_products, clip class TestOther(unittest.TestCase): @@ -43,5 +44,26 @@ def test_clip(self): clip(cube, 10, 8) +def test_group_products_string_list(): + products = [ + PreprocessorFile( + attributes={ + 'project': 'A', + 'dataset': 'B', + 'filename': 'A_B.nc'}, + settings={}), + PreprocessorFile( + attributes={ + 'project': 'A', + 'dataset': 'C', + 'filename': 'A_C.nc'}, + settings={}) + ] + grouped_by_string = _group_products(products, 'project') + grouped_by_list = _group_products(products, ['project']) + + assert grouped_by_list == grouped_by_string + + if __name__ == '__main__': unittest.main() From 509b1160104be2b7b1e29c7a30d19bb1c0600114 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sun, 23 Jan 2022 20:09:40 +0100 Subject: [PATCH 131/158] Improve test coverage of PreprocessorTask --- tests/integration/test_recipe.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index f14dcc1edb..21264d0f04 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1894,13 +1894,17 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): recipe = get_recipe(tmp_path, content, config_user) variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] datasets = set([var['dataset'] for var in variable]) + task = next(iter(recipe.tasks)) - products = next(iter(recipe.tasks)).products + products = task.products product_out = _test_output_product_consistency(products, preprocessor, statistics) assert len(product_out) == len(datasets) * len(statistics) + task._initialize_product_provenance() + assert next(iter(products)).provenance is not None + def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): statistics = ['mean', 'max'] @@ -1937,13 +1941,17 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): recipe = get_recipe(tmp_path, content, config_user) variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] + task = next(iter(recipe.tasks)) - products = next(iter(recipe.tasks)).products + products = task.products product_out = _test_output_product_consistency(products, preprocessor, statistics) assert len(product_out) == len(statistics) + task._initialize_product_provenance() + assert next(iter(products)).provenance is not None + def test_groupby_combined_statistics(tmp_path, patched_datafinder, config_user): From 915a3a485a2072b4d1d211fa2d259b4ba95a7387 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 24 Jan 2022 10:01:08 +0100 Subject: [PATCH 132/158] Merge remote-tracking branch 'origin/main' into dev_multimodel_ensemble --- .circleci/config.yml | 6 +- .../workflows/build-and-deploy-on-pypi.yml | 4 +- .github/workflows/install-from-conda.yml | 4 +- .github/workflows/install-from-pypi.yml | 5 +- .github/workflows/install-from-source.yml | 4 +- .github/workflows/run-tests-monitor.yml | 4 +- .github/workflows/run-tests.yml | 4 +- doc/quickstart/install.rst | 2 +- doc/recipe/preprocessor.rst | 121 ++++++++++------ doc/requirements.txt | 1 + environment.yml | 1 + esmvalcore/_main.py | 25 +++- esmvalcore/_recipe.py | 133 ++++++++++++++---- .../experimental/config/_config_validators.py | 2 +- esmvalcore/preprocessor/__init__.py | 5 +- esmvalcore/preprocessor/_derive/__init__.py | 2 +- esmvalcore/preprocessor/_derive/_shared.py | 11 +- esmvalcore/preprocessor/_regrid.py | 116 +++++++++++++-- package/meta.yaml | 1 + setup.py | 2 + .../_regrid/test_extract_levels.py | 4 +- .../_regrid/test_extract_location.py | 124 ++++++++++++++++ tests/integration/test_recipe.py | 76 ++++++++++ tests/unit/main/test_esmvaltool.py | 2 +- .../_regrid/test_extract_levels.py | 25 +++- tests/unit/test_recipe.py | 122 ++++++++++++++-- 26 files changed, 670 insertions(+), 136 deletions(-) create mode 100644 tests/integration/preprocessor/_regrid/test_extract_location.py diff --git a/.circleci/config.yml b/.circleci/config.yml index c34a385cfd..8f349fbfca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,7 +73,7 @@ jobs: set -x mkdir /logs # Pin Python version for faster environment solve - echo " - python=3.9" >> environment.yml + echo " - python=3.10" >> environment.yml # Add additional requirements for running all tests echo " - r-base @@ -146,7 +146,7 @@ jobs: . /opt/conda/etc/profile.d/conda.sh mkdir /logs # Pin Python version for faster environment solve - echo " - python=3.9" >> environment.yml + echo " - python=3.10" >> environment.yml # Add additional requirements for running all tests echo " - r-base @@ -183,7 +183,7 @@ jobs: mkdir /logs # conda update -y conda > /logs/conda.txt 2>&1 # Create and activate conda environment - conda create -y --name esmvaltool -c conda-forge 'python=3.9' + conda create -y --name esmvaltool -c conda-forge 'python=3.10' set +x; conda activate esmvaltool; set -x # Install conda install -y esmvalcore -c conda-forge diff --git a/.github/workflows/build-and-deploy-on-pypi.yml b/.github/workflows/build-and-deploy-on-pypi.yml index 4d2d0b34df..8e641c52f8 100644 --- a/.github/workflows/build-and-deploy-on-pypi.yml +++ b/.github/workflows/build-and-deploy-on-pypi.yml @@ -14,10 +14,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@master - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: "3.10" - name: Install pep517 run: >- python -m diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml index b31c0f41b4..967328b8bb 100644 --- a/.github/workflows/install-from-conda.yml +++ b/.github/workflows/install-from-conda.yml @@ -34,7 +34,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] # fail-fast set to False allows all other tests # in the worflow to run regardless of any fail fail-fast: false @@ -71,7 +71,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/install-from-pypi.yml b/.github/workflows/install-from-pypi.yml index dbb151c5f2..3f01c97982 100644 --- a/.github/workflows/install-from-pypi.yml +++ b/.github/workflows/install-from-pypi.yml @@ -21,7 +21,6 @@ on: push: branches: - main - # run the test only if the PR is to main # turn it on if required #pull_request: @@ -35,7 +34,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] # fail-fast set to False allows all other tests # in the worflow to run regardless of any fail fail-fast: false @@ -72,7 +71,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/install-from-source.yml b/.github/workflows/install-from-source.yml index ae3115b5fe..2e9c9971e6 100644 --- a/.github/workflows/install-from-source.yml +++ b/.github/workflows/install-from-source.yml @@ -32,7 +32,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -67,7 +67,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml index 3664b8d7ff..be742c2103 100644 --- a/.github/workflows/run-tests-monitor.yml +++ b/.github/workflows/run-tests-monitor.yml @@ -16,7 +16,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -57,7 +57,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 4e18d7fa2b..12e2b7b082 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -34,7 +34,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -69,7 +69,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false name: OSX Python ${{ matrix.python-version }} steps: diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index 7c2dd703d8..d973e3caa1 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -22,7 +22,7 @@ and install ESMValCore into it with a single command: .. code-block:: bash - conda create --name esmvalcore -c conda-forge esmvalcore 'python=3.9' + conda create --name esmvalcore -c conda-forge esmvalcore 'python=3.10' Don't forget to activate the newly created environment after the installation: diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 73ef024d08..95a0ba1ee7 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -335,7 +335,7 @@ extract the levels and vertically regrid onto the vertical levels of levels: ERA-Interim # This also works, but allows specifying the pressure coordinate name # levels: {dataset: ERA-Interim, coordinate: air_pressure} - scheme: linear_horizontal_extrapolate_vertical + scheme: linear_extrapolate By default, vertical interpolation is performed in the dimension coordinate of the z axis. If you want to explicitly declare the z axis coordinate to use @@ -349,7 +349,7 @@ the name of the desired coordinate: preproc_select_levels_from_dataset: extract_levels: levels: ERA-Interim - scheme: linear_horizontal_extrapolate_vertical + scheme: linear_extrapolate coordinate: air_pressure If ``coordinate`` is specified, pressure levels (if present) can be converted @@ -375,29 +375,39 @@ The meaning of 'very close' can be changed by providing the parameters: By default, `atol` will be set to 10^-7 times the mean value of of the available levels. +Schemes for vertical interpolation and extrapolation +---------------------------------------------------- + +The vertical interpolation currently supports the following schemes: + +* ``linear``: Linear interpolation without extrapolation, i.e., extrapolation + points will be masked even if the source data is not a masked array. +* ``linear_extrapolate``: Linear interpolation with **nearest-neighbour** + extrapolation, i.e., extrapolation points will take their value from the + nearest source point. +* ``nearest``: Nearest-neighbour interpolation without extrapolation, i.e., + extrapolation points will be masked even if the source data is not a masked + array. +* ``nearest_extrapolate``: Nearest-neighbour interpolation with nearest-neighbour + extrapolation, i.e., extrapolation points will take their value from the + nearest source point. + +.. note:: + Previous versions of ESMValCore (<2.5.0) supported the schemes + ``linear_horizontal_extrapolate_vertical`` and + ``nearest_horizontal_extrapolate_vertical``. These schemes have been renamed + to ``linear_extrapolate`` and ``nearest_extrapolate``, respectively, in + version 2.5.0 and are identical to the new schemes. Support for the old + names will be removed in version 2.7.0. * See also :func:`esmvalcore.preprocessor.extract_levels`. * See also :func:`esmvalcore.preprocessor.get_cmor_levels`. .. note:: - For both vertical and horizontal regridding one can control the - extrapolation mode when defining the interpolation scheme. Controlling the - extrapolation mode allows us to avoid situations where extrapolating values - makes little physical sense (e.g. extrapolating beyond the last data point). - The extrapolation mode is controlled by the `extrapolation_mode` - keyword. For the available interpolation schemes available in Iris, the - extrapolation_mode keyword must be one of: - - * ``extrapolate``: the extrapolation points will be calculated by - extending the gradient of the closest two points; - * ``error``: a ``ValueError`` exception will be raised, notifying an - attempt to extrapolate; - * ``nan``: the extrapolation points will be be set to NaN; - * ``mask``: the extrapolation points will always be masked, even if the - source data is not a ``MaskedArray``; or - * ``nanmask``: if the source data is a MaskedArray the extrapolation - points will be masked, otherwise they will be set to NaN. + Controlling the extrapolation mode allows us to avoid situations where + extrapolating values makes little physical sense (e.g. extrapolating beyond + the last data point). .. _weighting: @@ -819,35 +829,32 @@ Regridding (interpolation, extrapolation) schemes The schemes used for the interpolation and extrapolation operations needed by the horizontal regridding functionality directly map to their corresponding -implementations in Iris: - -* ``linear``: ``Linear(extrapolation_mode='mask')``, see :obj:`iris.analysis.Linear`. -* ``linear_extrapolate``: ``Linear(extrapolation_mode='extrapolate')``, see :obj:`iris.analysis.Linear`. -* ``nearest``: ``Nearest(extrapolation_mode='mask')``, see :obj:`iris.analysis.Nearest`. -* ``area_weighted``: ``AreaWeighted()``, see :obj:`iris.analysis.AreaWeighted`. -* ``unstructured_nearest``: ``UnstructuredNearest()``, see :obj:`iris.analysis.UnstructuredNearest`. +implementations in :mod:`iris`: + +* ``linear``: Linear interpolation without extrapolation, i.e., extrapolation + points will be masked even if the source data is not a masked array (uses + ``Linear(extrapolation_mode='mask')``, see :obj:`iris.analysis.Linear`). +* ``linear_extrapolate``: Linear interpolation with extrapolation, i.e., + extrapolation points will be calculated by extending the gradient of the + closest two points (uses ``Linear(extrapolation_mode='extrapolate')``, see + :obj:`iris.analysis.Linear`). +* ``nearest``: Nearest-neighbour interpolation without extrapolation, i.e., + extrapolation points will be masked even if the source data is not a masked + array (uses ``Nearest(extrapolation_mode='mask')``, see + :obj:`iris.analysis.Nearest`). +* ``area_weighted``: Area-weighted regridding (uses ``AreaWeighted()``, see + :obj:`iris.analysis.AreaWeighted`). +* ``unstructured_nearest``: Nearest-neighbour interpolation for unstructured + grids (uses ``UnstructuredNearest()``, see + :obj:`iris.analysis.UnstructuredNearest`). See also :func:`esmvalcore.preprocessor.regrid` .. note:: - For both vertical and horizontal regridding one can control the - extrapolation mode when defining the interpolation scheme. Controlling the - extrapolation mode allows us to avoid situations where extrapolating values - makes little physical sense (e.g. extrapolating beyond the last data - point). The extrapolation mode is controlled by the `extrapolation_mode` - keyword. For the available interpolation schemes available in Iris, the - extrapolation_mode keyword must be one of: - - * ``extrapolate`` – the extrapolation points will be calculated by - extending the gradient of the closest two points; - * ``error`` – a ``ValueError`` exception will be raised, notifying an - attempt to extrapolate; - * ``nan`` – the extrapolation points will be be set to NaN; - * ``mask`` – the extrapolation points will always be masked, even if - the source data is not a ``MaskedArray``; or - * ``nanmask`` – if the source data is a MaskedArray the extrapolation - points will be masked, otherwise they will be set to NaN. + Controlling the extrapolation mode allows us to avoid situations where + extrapolating values makes little physical sense (e.g. extrapolating beyond + the last data point). .. note:: @@ -1396,6 +1403,7 @@ The area manipulation module contains the following preprocessor functions: coordinate. * extract_shape_: Extract a region defined by a shapefile. * extract_point_: Extract a single point (with interpolation) +* extract_location_: Extract a single point by its location (with interpolation) * zonal_statistics_: Compute zonal statistics. * meridional_statistics_: Compute meridional statistics. * area_statistics_: Compute area statistics. @@ -1521,6 +1529,33 @@ Parameters: * ``scheme``: interpolation scheme: either ``'linear'`` or ``'nearest'``. There is no default. +See also :func:`esmvalcore.preprocessor.extract_point`. + + +``extract_location`` +-------------------- + +Extract a single point using a location name, with interpolation +(either linear or nearest). This preprocessor extracts a single +location point from a cube, according to the given interpolation +scheme ``scheme``. The function retrieves the coordinates of the +location and then calls the :func:`esmvalcore.preprocessor.extract_point` +preprocessor. It can be used to locate cities and villages, +but also mountains or other geographical locations. + +.. note:: + Note that this function's geolocator application needs a + working internet connection. + +Parameters + * ``cube``: the input dataset cube to extract a point from. + * ``location``: the reference location. Examples: 'mount everest', + 'romania', 'new york, usa'. Raises ValueError if none supplied. + * ``scheme`` : interpolation scheme. ``'linear'`` or ``'nearest'``. + There is no default, raises ValueError if none supplied. + +See also :func:`esmvalcore.preprocessor.extract_location`. + ``zonal_statistics`` -------------------- diff --git a/doc/requirements.txt b/doc/requirements.txt index 8a2f8ad90a..4128607c6a 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,7 @@ dask[array] esgf-pyclient docutils <0.17 fiona +geopy humanfriendly importlib_resources isodate diff --git a/environment.yml b/environment.yml index f5d51627fa..34ce3b87d5 100644 --- a/environment.yml +++ b/environment.yml @@ -11,6 +11,7 @@ dependencies: # and pypi ver built with older gdal - fiona - esmpy!=8.1.0 # see github.com/ESMValGroup/ESMValCore/issues/1208 + - geopy - iris>=3.1.0 - pandas - pip!=21.3 diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 457be0f46b..d6b9d94cd5 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -368,12 +368,35 @@ def run(self, strict (fail if there are any warnings). """ import os + import warnings from ._config import configure_logging, read_config_user_file from ._recipe import TASKSEP from .cmor.check import CheckLevels from .esgf._logon import logon + # Check validity of optional command line arguments with experimental + # API + with warnings.catch_warnings(): + # ignore experimental API warning + warnings.simplefilter("ignore") + from .experimental.config._config_object import Config as ExpConfig + explicit_optional_kwargs = { + 'config_file': config_file, + 'resume_from': resume_from, + 'max_datasets': max_datasets, + 'max_years': max_years, + 'skip_nonexistent': skip_nonexistent, + 'offline': offline, + 'diagnostics': diagnostics, + 'check_level': check_level, + } + all_optional_kwargs = dict(kwargs) + for (key, val) in explicit_optional_kwargs.items(): + if val is not None: + all_optional_kwargs[key] = val + ExpConfig(all_optional_kwargs) + recipe = self._get_recipe(recipe) cfg = read_config_user_file(config_file, recipe.stem, kwargs) @@ -390,7 +413,7 @@ def run(self, self._log_header(cfg['config_file'], log_files) cfg['resume_from'] = parse_resume(resume_from, recipe) - cfg['skip-nonexistent'] = skip_nonexistent + cfg['skip_nonexistent'] = skip_nonexistent if isinstance(diagnostics, str): diagnostics = diagnostics.split(' ') cfg['diagnostics'] = { diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 100fb99b55..68c45970d1 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -880,7 +880,7 @@ def get_matching(attributes): def _allow_skipping(ancestors, variable, config_user): """Allow skipping of datasets.""" allow_skipping = all([ - config_user.get('skip-nonexistent'), + config_user.get('skip_nonexistent'), not ancestors, variable['dataset'] != variable.get('reference_dataset'), ]) @@ -1590,7 +1590,8 @@ def _resolve_diagnostic_ancestors(self, tasks): for diagnostic_name, diagnostic in self.diagnostics.items(): for script_name, script_cfg in diagnostic['scripts'].items(): task_id = diagnostic_name + TASKSEP + script_name - if isinstance(tasks[task_id], DiagnosticTask): + if task_id in tasks and isinstance(tasks[task_id], + DiagnosticTask): logger.debug("Linking tasks for diagnostic %s script %s", diagnostic_name, script_name) ancestors = [] @@ -1605,6 +1606,75 @@ def _resolve_diagnostic_ancestors(self, tasks): ancestors.extend(tasks[a] for a in ancestor_ids) tasks[task_id].ancestors = ancestors + def _get_tasks_to_run(self): + """Get tasks filtered and add ancestors if needed.""" + tasknames_to_run = self._cfg.get('diagnostics', []) + if tasknames_to_run: + tasknames_to_run = set(tasknames_to_run) + while self._update_with_ancestors(tasknames_to_run): + pass + return tasknames_to_run + + def _update_with_ancestors(self, tasknames_to_run): + """Add ancestors for all selected tasks.""" + num_filters = len(tasknames_to_run) + + # Iterate over all tasks and add all ancestors to tasknames_to_run of + # those tasks that match one of the patterns given by tasknames_to_run + # to + for diagnostic_name, diagnostic in self.diagnostics.items(): + for script_name, script_cfg in diagnostic['scripts'].items(): + task_name = diagnostic_name + TASKSEP + script_name + for pattern in tasknames_to_run: + if fnmatch.fnmatch(task_name, pattern): + ancestors = script_cfg.get('ancestors', []) + if isinstance(ancestors, str): + ancestors = ancestors.split() + for ancestor in ancestors: + tasknames_to_run.add(ancestor) + break + + # If new ancestors have been added (num_filters != + # len(tasknames_to_run)) -> return True. This causes another call of + # this function in the while() loop of _get_tasks_to_run to ensure that + # nested ancestors are found. + + # If no new ancestors have been found (num_filters == + # len(tasknames_to_run)) -> return False. This terminates the search + # for ancestors. + + return num_filters != len(tasknames_to_run) + + def _create_diagnostic_tasks(self, diagnostic_name, diagnostic, + tasknames_to_run): + """Create diagnostic tasks.""" + tasks = [] + + if self._cfg.get('run_diagnostic', True): + for script_name, script_cfg in diagnostic['scripts'].items(): + task_name = diagnostic_name + TASKSEP + script_name + + # Skip diagnostic tasks if desired by the user + if tasknames_to_run: + for pattern in tasknames_to_run: + if fnmatch.fnmatch(task_name, pattern): + break + else: + logger.info("Skipping task %s due to filter", + task_name) + continue + + logger.info("Creating diagnostic task %s", task_name) + task = DiagnosticTask( + script=script_cfg['script'], + output_dir=script_cfg['output_dir'], + settings=script_cfg['settings'], + name=task_name, + ) + tasks.append(task) + + return tasks + def _fill_wildcards(self, variable_group, preprocessor_output): """Fill wildcards in the `timerange` . @@ -1650,12 +1720,26 @@ def _fill_wildcards(self, variable_group, preprocessor_output): variable_group].update( {'additional_datasets': updated_datasets}) - def _create_preprocessor_tasks(self, diagnostic_name, diagnostic): + def _create_preprocessor_tasks(self, diagnostic_name, diagnostic, + tasknames_to_run, any_diag_script_is_run): """Create preprocessor tasks.""" tasks = [] failed_tasks = [] for variable_group in diagnostic['preprocessor_output']: task_name = diagnostic_name + TASKSEP + variable_group + + # Skip preprocessor if not a single diagnostic script is run and + # the preprocessing task is not explicitly requested by the user + if tasknames_to_run: + if not any_diag_script_is_run: + for pattern in tasknames_to_run: + if fnmatch.fnmatch(task_name, pattern): + break + else: + logger.info("Skipping task %s due to filter", + task_name) + continue + # Resume previous runs if requested, else create a new task for resume_dir in self._cfg['resume_from']: prev_preproc_dir = Path( @@ -1700,14 +1784,28 @@ def _create_tasks(self): logger.info("Creating tasks from recipe") tasks = TaskSet() + tasknames_to_run = self._get_tasks_to_run() + priority = 0 failed_tasks = [] + for diagnostic_name, diagnostic in self.diagnostics.items(): logger.info("Creating tasks for diagnostic %s", diagnostic_name) + # Create diagnostic tasks + new_tasks = self._create_diagnostic_tasks(diagnostic_name, + diagnostic, + tasknames_to_run) + any_diag_script_is_run = bool(new_tasks) + for task in new_tasks: + task.priority = priority + tasks.add(task) + priority += 1 + # Create preprocessor tasks new_tasks, failed = self._create_preprocessor_tasks( - diagnostic_name, diagnostic) + diagnostic_name, diagnostic, tasknames_to_run, + any_diag_script_is_run) failed_tasks.extend(failed) for task in new_tasks: for task0 in task.flatten(): @@ -1715,21 +1813,6 @@ def _create_tasks(self): tasks.add(task) priority += 1 - # Create diagnostic tasks - if self._cfg.get('run_diagnostic', True): - for script_name, script_cfg in diagnostic['scripts'].items(): - task_name = diagnostic_name + TASKSEP + script_name - logger.info("Creating diagnostic task %s", task_name) - task = DiagnosticTask( - script=script_cfg['script'], - output_dir=script_cfg['output_dir'], - settings=script_cfg['settings'], - name=task_name, - ) - task.priority = priority - tasks.add(task) - priority += 1 - if failed_tasks: recipe_error = RecipeError('Could not create all tasks') recipe_error.failed_tasks.extend(failed_tasks) @@ -1746,18 +1829,6 @@ def _create_tasks(self): def initialize_tasks(self): """Define tasks in recipe.""" tasks = self._create_tasks() - - # Select only requested tasks - tasknames_to_run = self._cfg.get('diagnostics') - - if tasknames_to_run: - tasks = tasks.flatten() - names = {t.name for t in tasks} - selection = set() - for pattern in tasknames_to_run: - selection |= set(fnmatch.filter(names, pattern)) - tasks = TaskSet(t for t in tasks if t.name in selection) - tasks = tasks.flatten() logger.info("These tasks will be executed: %s", ', '.join(t.name for t in tasks)) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py index 10d2d31cba..c87411ba99 100644 --- a/esmvalcore/experimental/config/_config_validators.py +++ b/esmvalcore/experimental/config/_config_validators.py @@ -270,7 +270,7 @@ def deprecate(func, variable, version: str = None): # From CLI "resume_from": validate_pathlist, - "skip-nonexistent": validate_bool, + "skip_nonexistent": validate_bool, "diagnostics": validate_diagnostics, "check_level": validate_check_level, "offline": validate_bool, diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index d48bf95596..b347337d8f 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -42,9 +42,9 @@ mask_multimodel, mask_outside_range, ) -from ._multimodel import multi_model_statistics, ensemble_statistics +from ._multimodel import ensemble_statistics, multi_model_statistics from ._other import clip -from ._regrid import extract_levels, extract_point, regrid +from ._regrid import extract_levels, extract_location, extract_point, regrid from ._time import ( annual_statistics, anomalies, @@ -115,6 +115,7 @@ 'regrid', # Point interpolation 'extract_point', + 'extract_location', # Masking missing values 'mask_multimodel', 'mask_fillvalues', diff --git a/esmvalcore/preprocessor/_derive/__init__.py b/esmvalcore/preprocessor/_derive/__init__.py index 17209f0006..6ef2ec9d2a 100644 --- a/esmvalcore/preprocessor/_derive/__init__.py +++ b/esmvalcore/preprocessor/_derive/__init__.py @@ -95,7 +95,7 @@ def derive(cubes, short_name, long_name, units, standard_name=None): cube = DerivedVariable().calculate(cubes) except Exception as exc: msg = (f"Derivation of variable '{short_name}' failed. If you used " - f"the option '--skip-nonexistent' for running your recipe, " + f"the option '--skip_nonexistent' for running your recipe, " f"this might be caused by missing input data for derivation") raise ValueError(msg) from exc diff --git a/esmvalcore/preprocessor/_derive/_shared.py b/esmvalcore/preprocessor/_derive/_shared.py index 4c58cb076f..1688bca082 100644 --- a/esmvalcore/preprocessor/_derive/_shared.py +++ b/esmvalcore/preprocessor/_derive/_shared.py @@ -97,13 +97,14 @@ def column_average(cube, hus_cube, zg_cube, ps_cube): p_layer_widths.data / (mw_air * g_4d_array)) # Number of gas molecules per layer - cube = cube * n_dry + cube.data = cube.core_data() * n_dry.core_data() # Column-average - x_cube = ( - cube.collapsed('air_pressure', iris.analysis.SUM) / - n_dry.collapsed('air_pressure', iris.analysis.SUM)) - return x_cube + cube = cube.collapsed('air_pressure', iris.analysis.SUM) + cube.data = ( + cube.core_data() / + n_dry.collapsed('air_pressure', iris.analysis.SUM).core_data()) + return cube def pressure_level_widths(cube, ps_cube, top_limit=0.0): diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 7e8bb4756a..e36a51a36a 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -3,6 +3,7 @@ import logging import os import re +import warnings from copy import deepcopy from decimal import Decimal from typing import Dict @@ -11,9 +12,12 @@ import numpy as np import stratify from dask import array as da +from geopy.geocoders import Nominatim from iris.analysis import AreaWeighted, Linear, Nearest, UnstructuredNearest from iris.util import broadcast_to_shape +from esmvalcore.exceptions import ESMValCoreDeprecationWarning + from ..cmor._fixes.shared import add_altitude_from_plev, add_plev_from_altitude from ..cmor.fix import fix_file, fix_metadata from ..cmor.table import CMOR_TABLES @@ -63,9 +67,12 @@ } # Supported vertical interpolation schemes. -VERTICAL_SCHEMES = ('linear', 'nearest', - 'linear_horizontal_extrapolate_vertical', - 'nearest_horizontal_extrapolate_vertical') +VERTICAL_SCHEMES = ( + 'linear', + 'nearest', + 'linear_extrapolate', + 'nearest_extrapolate', +) def parse_cell_spec(spec): @@ -306,6 +313,66 @@ def _attempt_irregular_regridding(cube, scheme): return False +def extract_location(cube, location, scheme): + """Extract a point using a location name, with interpolation. + + Extracts a single location point from a cube, according + to the interpolation scheme ``scheme``. + + The function just retrieves the coordinates of the location and then calls + the ``extract_point`` preprocessor. + + It can be used to locate cities and villages, but also mountains or other + geographical locations. + + Note + ---- + The geolocator needs a working internet connection. + + Parameters + ---------- + cube : cube + The source cube to extract a point from. + + location : str + The reference location. Examples: 'mount everest', + 'romania','new york, usa' + + scheme : str + The interpolation scheme. 'linear' or 'nearest'. No default. + + Returns + ------- + Returns a cube with the extracted point, and with adjusted + latitude and longitude coordinates. + + Raises + ------ + ValueError: + If location is not supplied as a preprocessor parameter. + ValueError: + If scheme is not supplied as a preprocessor parameter. + ValueError: + If given location cannot be found by the geolocator. + """ + if location is None: + raise ValueError("Location needs to be specified." + " Examples: 'mount everest', 'romania'," + " 'new york, usa'") + if scheme is None: + raise ValueError("Interpolation scheme needs to be specified." + " Use either 'linear' or 'nearest'.") + geolocator = Nominatim(user_agent='esmvalcore') + geolocation = geolocator.geocode(location) + if geolocation is None: + raise ValueError(f'Requested location {location} can not be found.') + logger.info("Extracting data for %s (%s °N, %s °E)", geolocation, + geolocation.latitude, geolocation.longitude) + + return extract_point(cube, geolocation.latitude, + geolocation.longitude, scheme) + + def extract_point(cube, latitude, longitude, scheme): """Extract a point, with interpolation. @@ -335,6 +402,10 @@ def extract_point(cube, latitude, longitude, scheme): Returns a cube with the extracted point(s), and with adjusted latitude and longitude coordinates (see above). + Raises + ------ + ValueError: + If the interpolation scheme is None or unrecognized. Examples -------- @@ -705,29 +776,46 @@ def parse_vertical_scheme(scheme): The vertical interpolation scheme to use. Choose from 'linear', 'nearest', - 'nearest_horizontal_extrapolate_vertical', - 'linear_horizontal_extrapolate_vertical'. + 'linear_extrapolate', + 'nearest_extrapolate'. Returns ------- (str, str) A tuple containing the interpolation and extrapolation scheme. """ + # Issue warning when deprecated schemes are used + deprecated_schemes = { + 'linear_horizontal_extrapolate_vertical': 'linear_extrapolate', + 'nearest_horizontal_extrapolate_vertical': 'nearest_extrapolate', + } + if scheme in deprecated_schemes: + new_scheme = deprecated_schemes[scheme] + deprecation_msg = ( + f"The vertical regridding scheme ``{scheme}`` has been deprecated " + f"in ESMValCore version 2.5.0 and is scheduled for removal in " + f"version 2.7.0. It has been renamed to the identical scheme " + f"``{new_scheme}`` without any change in functionality.") + warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) + scheme = new_scheme + + # Check if valid scheme is given if scheme not in VERTICAL_SCHEMES: - emsg = 'Unknown vertical interpolation scheme, got {!r}. ' - emsg += 'Possible schemes: {!r}' - raise ValueError(emsg.format(scheme, VERTICAL_SCHEMES)) + raise ValueError( + f"Unknown vertical interpolation scheme, got '{scheme}', possible " + f"schemes are {VERTICAL_SCHEMES}") # This allows us to put level 0. to load the ocean surface. extrap_scheme = 'nan' - if scheme == 'nearest_horizontal_extrapolate_vertical': - scheme = 'nearest' - extrap_scheme = 'nearest' - if scheme == 'linear_horizontal_extrapolate_vertical': + if scheme == 'linear_extrapolate': scheme = 'linear' extrap_scheme = 'nearest' + if scheme == 'nearest_extrapolate': + scheme = 'nearest' + extrap_scheme = 'nearest' + return scheme, extrap_scheme @@ -753,8 +841,8 @@ def extract_levels(cube, The vertical interpolation scheme to use. Choose from 'linear', 'nearest', - 'nearest_horizontal_extrapolate_vertical', - 'linear_horizontal_extrapolate_vertical'. + 'linear_extrapolate', + 'nearest_extrapolate'. coordinate : optional str The coordinate to interpolate. If specified, pressure levels (if present) can be converted to height levels and vice versa using diff --git a/package/meta.yaml b/package/meta.yaml index 267949a64e..997553e07b 100644 --- a/package/meta.yaml +++ b/package/meta.yaml @@ -48,6 +48,7 @@ requirements: - esmpy!=8.1.0 # see github.com/ESMValGroup/ESMValCore/issues/1208 - fiona - fire + - geopy - humanfriendly - isodate - jinja2 diff --git a/setup.py b/setup.py index 36706b114c..6b3941bdce 100755 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ 'esmpy!=8.1.0', # see github.com/ESMValGroup/ESMValCore/issues/1208 'fiona', 'fire', + 'geopy', 'humanfriendly', "importlib_resources;python_version<'3.9'", 'isodate', @@ -210,6 +211,7 @@ def read_description(filename): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Atmospheric Science', 'Topic :: Scientific/Engineering :: GIS', diff --git a/tests/integration/preprocessor/_regrid/test_extract_levels.py b/tests/integration/preprocessor/_regrid/test_extract_levels.py index 267c067566..8687f19228 100644 --- a/tests/integration/preprocessor/_regrid/test_extract_levels.py +++ b/tests/integration/preprocessor/_regrid/test_extract_levels.py @@ -113,7 +113,7 @@ def test_add_alt_coord(self): assert self.cube.coords('air_pressure') assert not self.cube.coords('altitude') result = extract_levels(self.cube, [1, 2], - 'linear_horizontal_extrapolate_vertical', + 'linear_extrapolate', coordinate='altitude') assert not result.coords('air_pressure') assert result.coords('altitude') @@ -129,7 +129,7 @@ def test_add_plev_coord(self): assert not self.cube.coords('air_pressure') assert self.cube.coords('altitude') result = extract_levels(self.cube, [1, 2], - 'linear_horizontal_extrapolate_vertical', + 'linear_extrapolate', coordinate='air_pressure') assert result.coords('air_pressure') assert not result.coords('altitude') diff --git a/tests/integration/preprocessor/_regrid/test_extract_location.py b/tests/integration/preprocessor/_regrid/test_extract_location.py new file mode 100644 index 0000000000..7577a1806b --- /dev/null +++ b/tests/integration/preprocessor/_regrid/test_extract_location.py @@ -0,0 +1,124 @@ +"""Integration tests for :func:`esmvalcore.preprocessor.extract_location.""" + +import iris +import iris.fileformats +import numpy as np +from iris.coords import CellMethod, DimCoord + +import tests +from esmvalcore.preprocessor import extract_location + + +class Test(tests.Test): + def setUp(self): + """Prepare tests.""" + shape = (3, 45, 36) + data = np.arange(np.prod(shape)).reshape(shape) + self.cube = self._make_cube(data) + + @staticmethod + def _make_cube(data): + """Create a 3d synthetic test cube.""" + z, y, x = data.shape + + # Create the cube. + cm = CellMethod(method='mean', + coords='time', + intervals='20 minutes', + comments=None) + kwargs = dict(standard_name='air_temperature', + long_name='Air Temperature', + var_name='ta', + units='K', + attributes=dict(cube='attribute'), + cell_methods=(cm, )) + cube = iris.cube.Cube(data, **kwargs) + + # Create a synthetic test latitude coordinate. + data = np.linspace(-90, 90, y) + cs = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) + kwargs = dict(standard_name='latitude', + long_name='Latitude', + var_name='lat', + units='degrees_north', + attributes=dict(latitude='attribute'), + coord_system=cs) + ycoord = DimCoord(data, **kwargs) + ycoord.guess_bounds() + cube.add_dim_coord(ycoord, 1) + + # Create a synthetic test longitude coordinate. + data = np.linspace(0, 360, x) + kwargs = dict(standard_name='longitude', + long_name='Longitude', + var_name='lon', + units='degrees_east', + attributes=dict(longitude='attribute'), + coord_system=cs) + xcoord = DimCoord(data, **kwargs) + xcoord.guess_bounds() + cube.add_dim_coord(xcoord, 2) + return cube + + def test_extract_only_town_name(self): + """Test only town name.""" + point = extract_location(self.cube, + scheme='nearest', + location='Peñacaballera') + self.assertEqual(point.shape, (3, )) + np.testing.assert_equal(point.data, [1186, 2806, 4426]) + + def test_extract_town_name_and_region(self): + """Test town plus region.""" + point = extract_location(self.cube, + scheme='nearest', + location='Salamanca,Castilla y León') + self.assertEqual(point.shape, (3, )) + np.testing.assert_equal(point.data, [1186, 2806, 4426]) + + def test_extract_town_and_country(self): + """Test town plus country.""" + point = extract_location(self.cube, + scheme='nearest', + location='Salamanca,USA') + self.assertEqual(point.shape, (3, )) + np.testing.assert_equal(point.data, [1179, 2799, 4419]) + + def test_extract_all_params(self): + """Test town plus region plus country.""" + point = extract_location(self.cube, + scheme='nearest', + location='Salamanca,Castilla y León,Spain') + self.assertEqual(point.shape, (3, )) + print(point.data) + np.testing.assert_equal(point.data, [1186, 2806, 4426]) + + def test_extract_mountain(self): + """Test town plus region plus country.""" + point = extract_location(self.cube, + scheme='nearest', + location='Calvitero,Candelario') + self.assertEqual(point.shape, (3, )) + print(point.data) + np.testing.assert_equal(point.data, [1186, 2806, 4426]) + + def test_non_existing_location(self): + """Test town plus region plus country.""" + with self.assertRaises(ValueError): + extract_location(self.cube, + scheme='nearest', + location='Minas Tirith,Gondor') + + def test_no_location_parameter(self): + """Test if no location supplied.""" + with self.assertRaises(ValueError): + extract_location(self.cube, + scheme='nearest', + location=None) + + def test_no_scheme_parameter(self): + """Test if no scheme supplied.""" + with self.assertRaises(ValueError): + extract_location(self.cube, + scheme=None, + location='Calvitero,Candelario') diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 21264d0f04..cd9961efe9 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -106,6 +106,7 @@ def config_user(tmp_path): cfg = esmvalcore._config.read_config_user_file(filename, 'recipe_test', {}) cfg['offline'] = True cfg['check_level'] = CheckLevels.DEFAULT + cfg['diagnostics'] = set() return cfg @@ -3589,3 +3590,78 @@ def test_get_derive_input_variables(patched_datafinder, config_user): }], } assert derive_input == expected_derive_input + + +TEST_DIAG_SELECTION = [ + (None, {'d1/tas', 'd1/s1', 'd2/s1', 'd3/s1', 'd3/s2', 'd4/s1'}), + ({''}, set()), + ({'wrong_diag/*'}, set()), + ({'d1/*'}, {'d1/tas', 'd1/s1'}), + ({'d2/*'}, {'d1/tas', 'd1/s1', 'd2/s1'}), + ({'d3/*'}, {'d1/tas', 'd1/s1', 'd2/s1', 'd3/s1', 'd3/s2'}), + ({'d4/*'}, {'d1/tas', 'd1/s1', 'd2/s1', 'd3/s2', 'd4/s1'}), + ({'wrong_diag/*', 'd1/*'}, {'d1/tas', 'd1/s1'}), + ({'d1/tas'}, {'d1/tas'}), + ({'d1/tas', 'd2/*'}, {'d1/tas', 'd1/s1', 'd2/s1'}), + ({'d1/tas', 'd3/s1'}, {'d1/tas', 'd3/s1', 'd1/s1'}), + ({'d4/*', 'd3/s1'}, {'d1/tas', 'd1/s1', 'd2/s1', 'd3/s1', 'd3/s2', + 'd4/s1'}), +] + + +@pytest.mark.parametrize('diags_to_run,tasks_run', TEST_DIAG_SELECTION) +def test_diag_selection(tmp_path, patched_datafinder, config_user, + diags_to_run, tasks_run): + """Test selection of individual diagnostics via --diagnostics option.""" + TAGS.set_tag_values(TAGS_FOR_TESTING) + script = tmp_path / 'diagnostic.py' + script.write_text('') + + if diags_to_run is not None: + config_user['diagnostics'] = diags_to_run + + content = dedent(""" + diagnostics: + + d1: + variables: + tas: + project: CMIP6 + mip: Amon + exp: historical + start_year: 2000 + end_year: 2000 + ensemble: r1i1p1f1 + grid: gn + additional_datasets: + - dataset: CanESM5 + scripts: + s1: + script: {script} + + d2: + scripts: + s1: + script: {script} + ancestors: [d1/*] + + d3: + scripts: + s1: + script: {script} + ancestors: [d1/s1] + s2: + script: {script} + ancestors: [d2/*] + + d4: + scripts: + s1: + script: {script} + ancestors: [d3/s2] + """).format(script=script) + + recipe = get_recipe(tmp_path, content, config_user) + task_names = {task.name for task in recipe.tasks.flatten()} + + assert tasks_run == task_names diff --git a/tests/unit/main/test_esmvaltool.py b/tests/unit/main/test_esmvaltool.py index 08937d6383..634a8c6899 100644 --- a/tests/unit/main/test_esmvaltool.py +++ b/tests/unit/main/test_esmvaltool.py @@ -42,7 +42,7 @@ def test_run(mocker, tmp_path, cmd_offline, cfg_offline): 'diagnostics': set(), 'offline': offline, 'resume_from': [], - 'skip-nonexistent': False, + 'skip_nonexistent': False, }) diff --git a/tests/unit/preprocessor/_regrid/test_extract_levels.py b/tests/unit/preprocessor/_regrid/test_extract_levels.py index bb0f8b570e..43bf715164 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_levels.py +++ b/tests/unit/preprocessor/_regrid/test_extract_levels.py @@ -6,15 +6,17 @@ import iris import numpy as np +import pytest from numpy import ma import tests +from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.preprocessor._regrid import ( _MDI, VERTICAL_SCHEMES, + _preserve_fx_vars, extract_levels, parse_vertical_scheme, - _preserve_fx_vars, ) from tests.unit.preprocessor._regrid import _make_cube, _make_vcoord @@ -33,8 +35,7 @@ def setUp(self): 'esmvalcore.preprocessor._regrid._create_cube', return_value=self.created_cube) self.schemes = [ - 'linear', 'nearest', 'linear_horizontal_extrapolate_vertical', - 'nearest_horizontal_extrapolate_vertical' + 'linear', 'nearest', 'linear_extrapolate', 'nearest_extrapolate', ] descriptor, self.filename = tempfile.mkstemp('.nc') os.close(descriptor) @@ -53,13 +54,27 @@ def test_parse_vertical_schemes(self): reference = { 'linear': ('linear', 'nan'), 'nearest': ('nearest', 'nan'), - 'linear_horizontal_extrapolate_vertical': ('linear', 'nearest'), - 'nearest_horizontal_extrapolate_vertical': ('nearest', 'nearest'), + 'linear_extrapolate': ('linear', 'nearest'), + 'nearest_extrapolate': ('nearest', 'nearest'), } for scheme in self.schemes: interpolation, extrapolation = parse_vertical_scheme(scheme) assert interpolation, extrapolation == reference[scheme] + # Deprecated schemes (remove in v2.7) + deprecated_references = { + 'linear_horizontal_extrapolate_vertical': ('linear', 'nearest'), + 'nearest_horizontal_extrapolate_vertical': ('nearest', 'nearest'), + } + for scheme in deprecated_references: + warn_msg = ( + "`` has been deprecated in ESMValCore version 2.5.0 and is " + "scheduled for removal in version 2.7.0. It has been renamed " + "to the identical scheme ``") + with pytest.warns(ESMValCoreDeprecationWarning, match=warn_msg): + interp, extrap = parse_vertical_scheme(scheme) + assert interp, extrap == deprecated_references[scheme] + def test_nop__levels_match(self): vcoord = _make_vcoord(self.z, dtype=self.dtype) self.assertEqual(self.cube.coord(axis='z', dim_coords=True), vcoord) diff --git a/tests/unit/test_recipe.py b/tests/unit/test_recipe.py index a98fed28c7..b48a731958 100644 --- a/tests/unit/test_recipe.py +++ b/tests/unit/test_recipe.py @@ -1,4 +1,5 @@ from collections import defaultdict +from unittest import mock import iris import numpy as np @@ -12,6 +13,15 @@ from tests import PreprocessorFile +class MockRecipe(_recipe.Recipe): + """Mocked Recipe class with simple constructor.""" + + def __init__(self, cfg, diagnostics): + """Simple constructor used for testing.""" + self._cfg = cfg + self.diagnostics = diagnostics + + class TestRecipe: def test_expand_ensemble(self): @@ -85,45 +95,45 @@ def test_expand_ensemble_nolist(self): TEST_ALLOW_SKIPPING = [ ([], VAR_A, {}, False), ([], VAR_A, { - 'skip-nonexistent': False + 'skip_nonexistent': False }, False), ([], VAR_A, { - 'skip-nonexistent': True + 'skip_nonexistent': True }, True), ([], VAR_A_REF_A, {}, False), ([], VAR_A_REF_A, { - 'skip-nonexistent': False + 'skip_nonexistent': False }, False), ([], VAR_A_REF_A, { - 'skip-nonexistent': True + 'skip_nonexistent': True }, False), ([], VAR_A_REF_B, {}, False), ([], VAR_A_REF_B, { - 'skip-nonexistent': False + 'skip_nonexistent': False }, False), ([], VAR_A_REF_B, { - 'skip-nonexistent': True + 'skip_nonexistent': True }, True), (['A'], VAR_A, {}, False), (['A'], VAR_A, { - 'skip-nonexistent': False + 'skip_nonexistent': False }, False), (['A'], VAR_A, { - 'skip-nonexistent': True + 'skip_nonexistent': True }, False), (['A'], VAR_A_REF_A, {}, False), (['A'], VAR_A_REF_A, { - 'skip-nonexistent': False + 'skip_nonexistent': False }, False), (['A'], VAR_A_REF_A, { - 'skip-nonexistent': True + 'skip_nonexistent': True }, False), (['A'], VAR_A_REF_B, {}, False), (['A'], VAR_A_REF_B, { - 'skip-nonexistent': False + 'skip_nonexistent': False }, False), (['A'], VAR_A_REF_B, { - 'skip-nonexistent': True + 'skip_nonexistent': True }, False), ] @@ -160,7 +170,7 @@ def test_resume_preprocessor_tasks(mocker, tmp_path): # Create tasks tasks, failed = _recipe.Recipe._create_preprocessor_tasks( - recipe, diagnostic_name, diagnostic) + recipe, diagnostic_name, diagnostic, [], True) assert tasks == [resume_task] assert not failed @@ -363,3 +373,89 @@ def test_match_products_no_product(): variables = [{'var_name': 'var'}] grouped_products = _recipe._match_products(None, variables) assert grouped_products == defaultdict(list) + + +SCRIPTS_CFG = { + 'output_dir': mock.sentinel.output_dir, + 'script': mock.sentinel.script, + 'settings': mock.sentinel.settings, +} +DIAGNOSTICS = { + 'd1': {'scripts': {'s1': {'ancestors': [], **SCRIPTS_CFG}}}, + 'd2': {'scripts': {'s1': {'ancestors': ['d1/pr', 'd1/s1'], + **SCRIPTS_CFG}}}, + 'd3': {'scripts': {'s1': {'ancestors': ['d2/s1'], **SCRIPTS_CFG}}}, + 'd4': {'scripts': { + 's1': {'ancestors': 'd1/pr d1/tas', **SCRIPTS_CFG}, + 's2': {'ancestors': ['d4/pr', 'd4/tas'], **SCRIPTS_CFG}, + 's3': {'ancestors': ['d3/s1'], **SCRIPTS_CFG}, + }}, +} +TEST_GET_TASKS_TO_RUN = [ + (None, []), + ({''}, {''}), + ({'wrong_task/*'}, {'wrong_task/*'}), + ({'d1/*'}, {'d1/*'}), + ({'d2/*'}, {'d2/*', 'd1/pr', 'd1/s1'}), + ({'d3/*'}, {'d3/*', 'd2/s1', 'd1/pr', 'd1/s1'}), + ({'d4/*'}, {'d4/*', 'd1/pr', 'd1/tas', 'd4/pr', 'd4/tas', 'd3/s1', + 'd2/s1', 'd1/s1'}), + ({'wrong_task/*', 'd1/*'}, {'wrong_task/*', 'd1/*'}), + ({'d1/ta'}, {'d1/ta'}), + ({'d4/s2'}, {'d4/s2', 'd4/pr', 'd4/tas'}), + ({'d2/s1', 'd3/ta', 'd1/s1'}, {'d2/s1', 'd1/pr', 'd1/s1', 'd3/ta'}), + ({'d4/s1', 'd4/s2'}, {'d4/s1', 'd1/pr', 'd1/tas', 'd4/s2', 'd4/pr', + 'd4/tas'}), + ({'d4/s3', 'd3/ta'}, {'d4/s3', 'd3/s1', 'd2/s1', 'd1/pr', 'd1/s1', + 'd3/ta'}), +] + + +@pytest.mark.parametrize('diags_to_run,tasknames_to_run', + TEST_GET_TASKS_TO_RUN) +def test_get_tasks_to_run(diags_to_run, tasknames_to_run): + """Test ``Recipe._get_tasks_to_run``.""" + cfg = {} + if diags_to_run is not None: + cfg = {'diagnostics': diags_to_run} + + recipe = MockRecipe(cfg, DIAGNOSTICS) + tasks_to_run = recipe._get_tasks_to_run() + + assert tasks_to_run == tasknames_to_run + + +TEST_CREATE_DIAGNOSTIC_TASKS = [ + (set(), ['s1', 's2', 's3']), + ({'d4/*'}, ['s1', 's2', 's3']), + ({'d4/s1'}, ['s1']), + ({'d4/s1', 'd3/*'}, ['s1']), + ({'d4/s1', 'd4/s2'}, ['s1', 's2']), + ({''}, []), + ({'d3/*'}, []), +] + + +@pytest.mark.parametrize('tasks_to_run,tasks_run', + TEST_CREATE_DIAGNOSTIC_TASKS) +@mock.patch('esmvalcore._recipe.DiagnosticTask', autospec=True) +def test_create_diagnostic_tasks(mock_diag_task, tasks_to_run, tasks_run): + """Test ``Recipe._create_diagnostic_tasks``.""" + cfg = {'run_diagnostic': True} + diag_name = 'd4' + diag_cfg = DIAGNOSTICS['d4'] + n_tasks = len(tasks_run) + + recipe = MockRecipe(cfg, DIAGNOSTICS) + tasks = recipe._create_diagnostic_tasks(diag_name, diag_cfg, tasks_to_run) + + assert len(tasks) == n_tasks + assert mock_diag_task.call_count == n_tasks + for task_name in tasks_run: + expected_call = mock.call( + script=mock.sentinel.script, + output_dir=mock.sentinel.output_dir, + settings=mock.sentinel.settings, + name=f'{diag_name}{_recipe.TASKSEP}{task_name}', + ) + assert expected_call in mock_diag_task.mock_calls From 6eb8534f328ab372b005a60495dfa503add262ad Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 24 Jan 2022 10:27:30 +0100 Subject: [PATCH 133/158] Fix doc --- doc/recipe/preprocessor.rst | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 18e7b94968..7300614091 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -1532,33 +1532,6 @@ Parameters: See also :func:`esmvalcore.preprocessor.extract_point`. -``extract_location`` --------------------- - -Extract a single point using a location name, with interpolation -(either linear or nearest). This preprocessor extracts a single -location point from a cube, according to the given interpolation -scheme ``scheme``. The function retrieves the coordinates of the -location and then calls the :func:`esmvalcore.preprocessor.extract_point` -preprocessor. It can be used to locate cities and villages, -but also mountains or other geographical locations. - -.. note:: - Note that this function's geolocator application needs a - working internet connection. - -Parameters - * ``cube``: the input dataset cube to extract a point from. - * ``location``: the reference location. Examples: 'mount everest', - 'romania', 'new york, usa'. Raises ValueError if none supplied. - * ``scheme`` : interpolation scheme. ``'linear'`` or ``'nearest'``. - There is no default, raises ValueError if none supplied. - -See also :func:`esmvalcore.preprocessor.extract_location`. - -See also :func:`esmvalcore.preprocessor.extract_point`. - - ``extract_location`` -------------------- From 24e467a5031822c84ca8d81003f76945eb55a49e Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 26 Jan 2022 10:11:33 +0100 Subject: [PATCH 134/158] Change argument name --- esmvalcore/preprocessor/_multimodel.py | 2 +- esmvalcore/preprocessor/_other.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9ae04d33c0..b99ec021e1 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -433,7 +433,7 @@ def multi_model_statistics(products, if all(type(p).__name__ == 'PreprocessorFile' for p in products): # Avoid circular input: https://stackoverflow.com/q/16964467 statistics_products = set() - for identifier, input_prods in _group_products(products, by=groupby): + for identifier, input_prods in _group_products(products, by_key=groupby): sub_output_products = output_products[identifier] # Compute statistics on a single group diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 3402469b73..36bbccafb6 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -64,10 +64,10 @@ def _groupby(iterable, keyfunc): return grouped -def _group_products(products, by): +def _group_products(products, by_key): """Group products by the given list of attributes.""" def grouper(product): - return product.group(by) + return product.group(by_key) grouped = _groupby(products, keyfunc=grouper) return grouped.items() From 0567d25029c4257d6784fe0df80e44c7325c3ce0 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 26 Jan 2022 10:54:49 +0100 Subject: [PATCH 135/158] Change name of variable --- esmvalcore/_recipe.py | 2 +- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 68c45970d1..7a17b66e47 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -750,7 +750,7 @@ def _update_multiproduct(input_products, order, preproc_dir, step): } # pass to ancestors output_products = set() - for identifier, products in _group_products(products, by=grouping): + for identifier, products in _group_products(products, by_key=grouping): common_attributes = _get_common_attributes(products) for statistic in settings.get('statistics'): diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index b99ec021e1..50e23effa8 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -433,8 +433,8 @@ def multi_model_statistics(products, if all(type(p).__name__ == 'PreprocessorFile' for p in products): # Avoid circular input: https://stackoverflow.com/q/16964467 statistics_products = set() - for identifier, input_prods in _group_products(products, by_key=groupby): - sub_output_products = output_products[identifier] + for group, input_prods in _group_products(products, by_key=groupby): + sub_output_products = output_products[group] # Compute statistics on a single group group_statistics = _multiproduct_statistics( From a1c83f7bd071927a39c1a324cfc5e62494a331a7 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:22:09 +0100 Subject: [PATCH 136/158] Update esmvalcore/preprocessor/_multimodel.py Co-authored-by: Valeriu Predoi --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 50e23effa8..fdd460aee4 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -91,7 +91,7 @@ def _unify_time_coordinates(cubes): This function extracts the date information from the cube and reconstructs the time coordinate, resetting the actual dates to the 15th of the month or - 1st of july for yearly data (consistent with `regrid_time`), so that there + 1st of July for yearly data (consistent with `regrid_time`), so that there are no mismatches in the time arrays. If cubes have different time units, it will use reset the calendar to a From d09412bfdb72f475dc2538721833406fea5bd09f Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:22:45 +0100 Subject: [PATCH 137/158] Update esmvalcore/preprocessor/_multimodel.py Co-authored-by: Valeriu Predoi --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index fdd460aee4..94bc5f9e21 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -94,7 +94,7 @@ def _unify_time_coordinates(cubes): 1st of July for yearly data (consistent with `regrid_time`), so that there are no mismatches in the time arrays. - If cubes have different time units, it will use reset the calendar to a + If cubes have different time units, it will reset the calendar to a default gregorian calendar with unit "days since 1850-01-01". Might not work for (sub)daily data, because different calendars may have From 2d62b2acc8bc727fc02e6b9a245c5d73d994dfee Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 28 Jan 2022 09:29:22 +0100 Subject: [PATCH 138/158] Improve docstring --- esmvalcore/preprocessor/_multimodel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 50e23effa8..065a5990e3 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -406,8 +406,9 @@ def multi_model_statistics(products, For internal use only. A dict with statistics names as keys and preprocessorfiles as values. If products are passed as input, the statistics cubes will be assigned to these output products. - groupby: str - Group products by a given tag or attribute. + groupby: tuple + Group products by a given tag or attribute, e.g. + ('project', 'dataset', 'tag1'). keep_input_datasets: bool If True, the output will include the input datasets. If False, only the computed statistics will be returned. From 6e52fc73ed2a88d6d64a0baf70bc8ff0bc34a31a Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:33:43 +0100 Subject: [PATCH 139/158] Apply suggestions from code review Co-authored-by: Valeriu Predoi --- esmvalcore/_recipe.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 7a17b66e47..0b4c4bd9d3 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -754,18 +754,13 @@ def _update_multiproduct(input_products, order, preproc_dir, step): common_attributes = _get_common_attributes(products) for statistic in settings.get('statistics'): - common_attributes[step] = _get_tag(step, identifier, statistic) - filename = get_multiproduct_filename(common_attributes, preproc_dir) common_attributes['filename'] = filename - statistic_product = PreprocessorFile(common_attributes, downstream_settings) - output_products.add(statistic_product) - relevant_settings['output_products'][identifier][ statistic] = statistic_product @@ -956,7 +951,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ensemble_step = 'ensemble_statistics' multi_model_step = 'multi_model_statistics' if ensemble_step in profile: - ensemble_products, ensemble_settings = _update_multiproduct( products, order, preproc_dir, ensemble_step) @@ -970,7 +964,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ensemble_products = products if multi_model_step in profile: - multimodel_products, multimodel_settings = _update_multiproduct( ensemble_products, order, preproc_dir, multi_model_step) From 3aaec4ef162d35d9f54346b46364d24dec46b157 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:49:06 +0100 Subject: [PATCH 140/158] Apply suggestions from code review Co-authored-by: Valeriu Predoi --- doc/recipe/preprocessor.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 7300614091..d02c5c042d 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -870,7 +870,7 @@ See also :func:`esmvalcore.preprocessor.regrid` Ensemble statistics =================== For certain use cases it may be desirable to compute ensemble statistics. For -example to prevent models with many ensemble member getting excessive weight in +example to prevent models with many ensemble members getting excessive weight in the multi-model statistics functions. Theoretically, ensemble statistics are a special case (grouped) multi-model @@ -940,7 +940,7 @@ days in a year may vary between calendars, (sub-)daily data are not supported. exclude: [NCEP] Multi-model statistics now also supports a ``groupby`` argument. You can group by -any dataset key (``project``, ``experiment``, etc.) or a combination of keys. You can +any dataset key (``project``, ``experiment``, etc.) or a combination of keys in a list. You can also add an arbitrary 'tag' to a dataset definition and then groupby that tag. When using this preprocessor in conjunction with `ensemble statistics`_ preprocessor, you can groupby ``ensemble_statistics`` as well. For example: From 35390194e2c51e4a37e0e322dfc239a0835ef179 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 28 Jan 2022 10:24:20 +0100 Subject: [PATCH 141/158] Change function name --- esmvalcore/_recipe.py | 4 ++-- esmvalcore/_recipe_checks.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 7a17b66e47..fa27d48cbc 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -737,10 +737,10 @@ def _update_multiproduct(input_products, order, preproc_dir, step): settings = list(products)[0].settings[step] if step == 'ensemble_statistics': - check.ensemble_statistics(settings) + check.ensemble_statistics_preproc(settings) grouping = ['project', 'dataset', 'exp', 'sub_experiment'] else: - check.multimodel_statistics(settings) + check.multimodel_statistics_preproc(settings) grouping = settings.get('groupby', None) downstream_settings = _get_downstream_settings(step, order, products) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 6fcd84bcb9..1971cc1a40 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -269,7 +269,7 @@ def _verify_arguments(given, expected): f"keywords are: {expected}.") -def multimodel_statistics(settings): +def multimodel_statistics_preproc(settings): """Check that the multi-model settings are valid.""" valid_keys = ['span', 'groupby', 'statistics', 'keep_input_datasets'] _verify_arguments(settings.keys(), valid_keys) @@ -290,7 +290,7 @@ def multimodel_statistics(settings): _verify_keep_input_datasets(keep_input_datasets) -def ensemble_statistics(settings): +def ensemble_statistics_preproc(settings): """Check that the ensemble settings are valid.""" valid_keys = ['statistics', 'keep_input_datasets'] _verify_arguments(settings.keys(), valid_keys) From 7d39a128b9f6239326ef420ed3ff4f18e6ab3b6c Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 28 Jan 2022 15:15:49 +0100 Subject: [PATCH 142/158] Change name of the function --- esmvalcore/preprocessor/_multimodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index eb4cbb2b1f..e290fda335 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -464,7 +464,7 @@ def ensemble_statistics(products, statistics, See Also -------- - multicube_statistics_iris : core statistics function. + multicube_model_statistics : core statistics function. """ ensemble_grouping = ('project', 'dataset', 'exp', 'sub_experiment') return multi_model_statistics( From fa117454c37bd558c7e7564635c89e0ca5367b66 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 28 Jan 2022 18:25:24 +0100 Subject: [PATCH 143/158] Add test for exclude --- tests/integration/test_recipe.py | 67 +++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index cd9961efe9..09c505fad5 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1845,12 +1845,13 @@ def _test_output_product_consistency(products, preprocessor, statistics): product_out = defaultdict(list) for i, product in enumerate(products): - settings = product.settings[preprocessor] - output_products = settings['output_products'] + settings = product.settings.get(preprocessor) + if settings: + output_products = settings['output_products'] - for identifier, statistic_out in output_products.items(): - for statistic, preproc_file in statistic_out.items(): - product_out[identifier, statistic].append(preproc_file) + for identifier, statistic_out in output_products.items(): + for statistic, preproc_file in statistic_out.items(): + product_out[identifier, statistic].append(preproc_file) # Make sure that output products are consistent for (identifier, statistic), value in product_out.items(): @@ -1954,6 +1955,62 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): assert next(iter(products)).provenance is not None +def test_multi_model_statistics_exclude(tmp_path, + patched_datafinder, + config_user): + statistics = ['mean', 'max'] + diagnostic = 'diagnostic_name' + variable = 'pr' + preprocessor = 'multi_model_statistics' + + content = dedent(f""" + preprocessors: + default: &default + custom_order: true + area_statistics: + operator: mean + {preprocessor}: + span: overlap + statistics: {statistics} + groupby: ['project'] + exclude: ['TEST'] + + diagnostics: + {diagnostic}: + variables: + {variable}: + project: CMIP5 + mip: Amon + start_year: 2000 + end_year: 2002 + preprocessor: default + additional_datasets: + - {{dataset: CanESM2, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1"}} + - {{dataset: CCSM4, exp: [historical, rcp45], + ensemble: "r(1:2)i1p1"}} + - {{dataset: TEST, project: OBS, type: reanaly, version: 1, + tier: 1}} + scripts: null + """) + + recipe = get_recipe(tmp_path, content, config_user) + variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] + task = next(iter(recipe.tasks)) + + products = task.products + product_out = _test_output_product_consistency(products, preprocessor, + statistics) + + assert len(product_out) == len(statistics) + assert 'OBS' not in product_out + for id, prods in product_out: + assert id != 'OBS' + assert id == 'CMIP5' + task._initialize_product_provenance() + assert next(iter(products)).provenance is not None + + def test_groupby_combined_statistics(tmp_path, patched_datafinder, config_user): diagnostic = 'diagnostic_name' From 6471d002ea0000b42b6946dac261e1a693656a21 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 1 Feb 2022 15:50:35 +0100 Subject: [PATCH 144/158] Add docstrings to ensemble stats --- esmvalcore/preprocessor/_multimodel.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index e290fda335..f4ae5553da 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -462,9 +462,25 @@ def ensemble_statistics(products, statistics, The products are grouped and the cubes are extracted from the products. Resulting cubes are assigned to `output_products`. + Parameters + ---------- + products: list + Cubes (or products) over which the statistics will be computed. + statistics: list + Statistical metrics to be computed, e.g. [``mean``, ``max``]. Choose + from the operators listed in the iris.analysis package. Percentiles can + be specified like ``pXX.YY``. + output_products: dict + For internal use only. A dict with statistics names as keys and + preprocessorfiles as values. If products are passed as input, the + statistics cubes will be assigned to these output products. + keep_input_datasets: bool + If True, the output will include the input datasets. + If False, only the computed statistics will be returned. + See Also -------- - multicube_model_statistics : core statistics function. + multi_model_statistics : core statistics function. """ ensemble_grouping = ('project', 'dataset', 'exp', 'sub_experiment') return multi_model_statistics( From f17faca4e35e62934b55debd57e3f5b4af6d4102 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 1 Feb 2022 16:28:50 +0100 Subject: [PATCH 145/158] Add documentation about groupby tag --- doc/recipe/preprocessor.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index d02c5c042d..04d395907c 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -964,6 +964,29 @@ can groupby ``ensemble_statistics`` as well. For example: This will first compute ensemble mean and median, and then compute the multi-model min and max separately for the ensemble means and medians. +When grouping by a `tag` not defined in all datasets, the datasets missing the tag will +be grouped together. In the example below, datasets `UKESM` and `ERA5` would belong to the same +group. + +.. code-block:: yaml + + datasets: + - {dataset: CanESM2, exp: historical, ensemble: "r(1:2)i1p1", tag: 'group1'} + - {dataset: CanESM5, exp: historical, ensemble: "r(1:2)i1p1", tag: 'group2'} + - {dataset: CCSM4, exp: historical, ensemble: "r(1:2)i1p1", tag: 'group2'} + - {dataset: UKESM, exp: historical, ensemble: "r(1:2)i1p1"} + - {dataset: ERA5} + + preprocessors: + example_preprocessor: + ensemble_statistics: + statistics: [median, mean] + multi_model_statistics: + span: overlap + statistics: [min, max] + groupby: [tag] + +Note that those datasets can be excluded if listed in the `exclude` option. see also :func:`esmvalcore.preprocessor.multiproduct_statistics`. From d1de89181807a03fd9bada0db7b99f6572398b42 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Thu, 3 Feb 2022 12:40:34 +0100 Subject: [PATCH 146/158] Update doc/recipe/preprocessor.rst Co-authored-by: Valeriu Predoi --- doc/recipe/preprocessor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 04d395907c..5f5430c43a 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -966,7 +966,7 @@ min and max separately for the ensemble means and medians. When grouping by a `tag` not defined in all datasets, the datasets missing the tag will be grouped together. In the example below, datasets `UKESM` and `ERA5` would belong to the same -group. +group ``group``, a different group than ``group2`` that the other datasets are grouped in. .. code-block:: yaml From 42d7050383d927987891983ed404c7f234d410d5 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 3 Feb 2022 13:25:15 +0100 Subject: [PATCH 147/158] Improve docstrings --- esmvalcore/preprocessor/_multimodel.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index f4ae5553da..4f47e72b1f 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -459,8 +459,10 @@ def ensemble_statistics(products, statistics, output_products, keep_input_datasets=True): """Entry point for ensemble statistics. - The products are grouped and the cubes are extracted from - the products. Resulting cubes are assigned to `output_products`. + An ensemble grouping is performed on the input products. + The statistics are then computed calling + the :func:`esmvalcore.preprocessor.multi_model_statistics` module, + taking the grouped products as an input. Parameters ---------- @@ -478,9 +480,15 @@ def ensemble_statistics(products, statistics, If True, the output will include the input datasets. If False, only the computed statistics will be returned. + Returns + ------- + set + A set of output_products with the resulting ensemble statistics. + See Also -------- - multi_model_statistics : core statistics function. + :func:`esmvalcore.preprocessor.multi_model_statistics` for + the full description of the core statistics function. """ ensemble_grouping = ('project', 'dataset', 'exp', 'sub_experiment') return multi_model_statistics( From 3f110334ef0360f18091f9efd0991e630e2b9457 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 4 Feb 2022 09:18:39 +0100 Subject: [PATCH 148/158] Apply suggestions from code review Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- doc/recipe/preprocessor.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 5f5430c43a..b5fd1a3219 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -890,12 +890,11 @@ The preprocessor takes a list of statistics as input: statistics: [mean, median] This preprocessor function exposes the iris analysis package, and works with all -(capitalized) statistics from the `iris' analysis package -`_ +(capitalized) statistics from the :mod:`iris.analysis` package that can be executed without additional arguments (e.g. percentiles are not supported because it requires additional keywords: percentile.). -see also :func:`esmvalcore.preprocessor.multiproduct_statistics_iris`. +See also :func:`esmvalcore.preprocessor.ensemble_statistics`. .. _multi-model statistics: @@ -941,9 +940,9 @@ days in a year may vary between calendars, (sub-)daily data are not supported. Multi-model statistics now also supports a ``groupby`` argument. You can group by any dataset key (``project``, ``experiment``, etc.) or a combination of keys in a list. You can -also add an arbitrary 'tag' to a dataset definition and then groupby that tag. When +also add an arbitrary tag to a dataset definition and then group by that tag. When using this preprocessor in conjunction with `ensemble statistics`_ preprocessor, you -can groupby ``ensemble_statistics`` as well. For example: +can group by ``ensemble_statistics`` as well. For example: .. code-block:: yaml @@ -964,7 +963,7 @@ can groupby ``ensemble_statistics`` as well. For example: This will first compute ensemble mean and median, and then compute the multi-model min and max separately for the ensemble means and medians. -When grouping by a `tag` not defined in all datasets, the datasets missing the tag will +When grouping by a tag not defined in all datasets, the datasets missing the tag will be grouped together. In the example below, datasets `UKESM` and `ERA5` would belong to the same group ``group``, a different group than ``group2`` that the other datasets are grouped in. @@ -986,9 +985,9 @@ group ``group``, a different group than ``group2`` that the other datasets are g statistics: [min, max] groupby: [tag] -Note that those datasets can be excluded if listed in the `exclude` option. +Note that those datasets can be excluded if listed in the ``exclude`` option. -see also :func:`esmvalcore.preprocessor.multiproduct_statistics`. +See also :func:`esmvalcore.preprocessor.multi_model_statistics`. .. note:: From effa0ad9d5ad01f5dca301d2826585b8178b1ec9 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 4 Feb 2022 15:21:10 +0100 Subject: [PATCH 149/158] Remove keep_input..., add span in ensemble stats --- esmvalcore/_recipe_checks.py | 9 +++++---- esmvalcore/preprocessor/_multimodel.py | 6 +++--- tests/integration/test_recipe.py | 1 + tests/unit/preprocessor/_multimodel/test_multimodel.py | 9 +++------ 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 1971cc1a40..2597b10874 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -292,16 +292,17 @@ def multimodel_statistics_preproc(settings): def ensemble_statistics_preproc(settings): """Check that the ensemble settings are valid.""" - valid_keys = ['statistics', 'keep_input_datasets'] + valid_keys = ['statistics', 'span'] _verify_arguments(settings.keys(), valid_keys) + span = settings.get('span', 'overlap') # optional, default: overlap + if span: + _verify_span_value(span) + statistics = settings.get('statistics', None) if statistics: _verify_statistics(statistics, 'ensemble_statistics') - keep_input_datasets = settings.get('keep_input_datasets', True) - _verify_keep_input_datasets(keep_input_datasets) - def _check_delimiter(timerange): if len(timerange) != 2: diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 4f47e72b1f..1c8e11a15e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -456,7 +456,7 @@ def multi_model_statistics(products, def ensemble_statistics(products, statistics, - output_products, keep_input_datasets=True): + output_products, span='overlap'): """Entry point for ensemble statistics. An ensemble grouping is performed on the input products. @@ -493,9 +493,9 @@ def ensemble_statistics(products, statistics, ensemble_grouping = ('project', 'dataset', 'exp', 'sub_experiment') return multi_model_statistics( products=products, - span='overlap', + span=span, statistics=statistics, output_products=output_products, groupby=ensemble_grouping, - keep_input_datasets=keep_input_datasets + keep_input_datasets=False ) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index fbbaf289f8..c82369698d 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -2031,6 +2031,7 @@ def test_groupby_combined_statistics(tmp_path, patched_datafinder, area_statistics: operator: mean {ens_preprocessor}: + span: 'overlap' statistics: {ens_statistics} {mm_preprocessor}: span: overlap diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 3b26ca2d6e..7de15d6a0c 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -623,12 +623,9 @@ def test_ensemble_products(): 'output_products': output_products, } - result1 = mm.ensemble_statistics(products, **kwargs) - assert len(result1) == 6 - - result2 = mm.ensemble_statistics( - products, keep_input_datasets=False, **kwargs) - assert len(result2) == 2 + result = mm.ensemble_statistics( + products, **kwargs) + assert len(result) == 2 def test_ignore_tas_scalar_height_coord(): From 4ed27801e1d9c47b078bd2f3a442714283a323ea Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 4 Feb 2022 15:22:22 +0100 Subject: [PATCH 150/158] Improve doc --- doc/recipe/preprocessor.rst | 53 ++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index b5fd1a3219..4f3325f506 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -874,7 +874,9 @@ example to prevent models with many ensemble members getting excessive weight in the multi-model statistics functions. Theoretically, ensemble statistics are a special case (grouped) multi-model -statistics. However, they should typically be computed earlier in the workflow. +statistics. This grouping is performed taking into account the dataset tags +`project`, `dataset`, `experiment`, and (if present) `sub_experiment`. +However, they should typically be computed earlier in the workflow. Moreover, because multiple ensemble members of the same model are typically more consistent/homogeneous than datasets from different models, the implementation is more straigtforward and can benefit from lazy evaluation and more efficient @@ -894,6 +896,7 @@ This preprocessor function exposes the iris analysis package, and works with all that can be executed without additional arguments (e.g. percentiles are not supported because it requires additional keywords: percentile.). + See also :func:`esmvalcore.preprocessor.ensemble_statistics`. .. _multi-model statistics: @@ -924,6 +927,9 @@ across overlapping times only (``span: overlap``) or across the full time span of the combined models (``span: full``). The preprocessor sets a common time coordinate on all datasets. As the number of days in a year may vary between calendars, (sub-)daily data with different calendars are not supported. +The preprocessor saves both the input single model files as well as the multi-model +results. In case of not wanting to keep the single model files, set +parameter `keep_input_datasets` to `false` (default value is `true`). Input datasets may have different time coordinates. The multi-model statistics preprocessor sets a common time coordinate on all datasets. As the number of @@ -932,11 +938,18 @@ days in a year may vary between calendars, (sub-)daily data are not supported. .. code-block:: yaml preprocessors: - multi_model_preprocessor: + multi_model_save_input: multi_model_statistics: span: overlap statistics: [mean, median] exclude: [NCEP] + multi_model_save_input: + multi_model_without_saving_input: + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [NCEP] + keep_input_datasets: false Multi-model statistics now also supports a ``groupby`` argument. You can group by any dataset key (``project``, ``experiment``, etc.) or a combination of keys in a list. You can @@ -947,8 +960,8 @@ can group by ``ensemble_statistics`` as well. For example: .. code-block:: yaml datasets: - - {dataset: CanESM2, exp: historical, ensemble: "r(1:2)i1p1", tag: 'group1'} - - {dataset: CCSM4, exp: historical, ensemble: "r(1:2)i1p1", tag: 'group2'} + - {dataset: CanESM2, exp: historical, ensemble: "r(1:2)i1p1"} + - {dataset: CCSM4, exp: historical, ensemble: "r(1:2)i1p1"} preprocessors: example_preprocessor: @@ -957,15 +970,37 @@ can group by ``ensemble_statistics`` as well. For example: multi_model_statistics: span: overlap statistics: [min, max] - groupby: [ensemble_statistics, tag] + groupby: [ensemble_statistics] exclude: [NCEP] This will first compute ensemble mean and median, and then compute the multi-model -min and max separately for the ensemble means and medians. +min and max separately for the ensemble means and medians. Note that this combination +will not save the individual ensemble members, only the ensemble and multimodel statistics results. +In case of wanting to save both individual ensemble members as well as the statistic results, +the preprocessor chains could be defined as: + +.. code-block:: yaml + preprocessors: + everything_else: &everything_else + area_statistics: ... + regrid_time: ... + multimodel: + <<: *everything_else + ensemble_statistics: + +variables: + tas_datasets: + short_name: tas + preprocessor: everything_else + ... + tas_multimodel: + short_name: tas + preprocessor: multimodel + ... When grouping by a tag not defined in all datasets, the datasets missing the tag will be grouped together. In the example below, datasets `UKESM` and `ERA5` would belong to the same -group ``group``, a different group than ``group2`` that the other datasets are grouped in. +group, while the other datasets would belong to either `group1` or `group2` .. code-block:: yaml @@ -978,8 +1013,6 @@ group ``group``, a different group than ``group2`` that the other datasets are g preprocessors: example_preprocessor: - ensemble_statistics: - statistics: [median, mean] multi_model_statistics: span: overlap statistics: [min, max] @@ -1550,7 +1583,7 @@ Parameters: be an array of floating point values. * ``scheme``: interpolation scheme: either ``'linear'`` or ``'nearest'``. There is no default. - + See also :func:`esmvalcore.preprocessor.extract_point`. From 901a47d7dcc3755b67b18a284c4323f00e93a9fe Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 4 Feb 2022 15:25:44 +0100 Subject: [PATCH 151/158] Improve doc --- doc/recipe/preprocessor.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 4f3325f506..cf919d1c71 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -896,6 +896,9 @@ This preprocessor function exposes the iris analysis package, and works with all that can be executed without additional arguments (e.g. percentiles are not supported because it requires additional keywords: percentile.). +Note that `ensemble_statistics` will not return the single model and ensemble files, +only the requested ensemble statistics results. + See also :func:`esmvalcore.preprocessor.ensemble_statistics`. From 89d0ee3cd15a4c6af9f7f1f04f722399a49947f6 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 4 Feb 2022 17:14:34 +0100 Subject: [PATCH 152/158] Apply suggestions from code review Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- doc/recipe/preprocessor.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index cf919d1c71..617f51a4c9 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -896,7 +896,7 @@ This preprocessor function exposes the iris analysis package, and works with all that can be executed without additional arguments (e.g. percentiles are not supported because it requires additional keywords: percentile.). -Note that `ensemble_statistics` will not return the single model and ensemble files, +Note that ``ensemble_statistics`` will not return the single model and ensemble files, only the requested ensemble statistics results. @@ -931,8 +931,8 @@ of the combined models (``span: full``). The preprocessor sets a common time coordinate on all datasets. As the number of days in a year may vary between calendars, (sub-)daily data with different calendars are not supported. The preprocessor saves both the input single model files as well as the multi-model -results. In case of not wanting to keep the single model files, set -parameter `keep_input_datasets` to `false` (default value is `true`). +results. In case you do not want to keep the single model files, set the +parameter ``keep_input_datasets`` to ``false`` (default value is ``true``). Input datasets may have different time coordinates. The multi-model statistics preprocessor sets a common time coordinate on all datasets. As the number of @@ -954,7 +954,7 @@ days in a year may vary between calendars, (sub-)daily data are not supported. exclude: [NCEP] keep_input_datasets: false -Multi-model statistics now also supports a ``groupby`` argument. You can group by +Multi-model statistics also supports a ``groupby`` argument. You can group by any dataset key (``project``, ``experiment``, etc.) or a combination of keys in a list. You can also add an arbitrary tag to a dataset definition and then group by that tag. When using this preprocessor in conjunction with `ensemble statistics`_ preprocessor, you @@ -1003,7 +1003,7 @@ variables: When grouping by a tag not defined in all datasets, the datasets missing the tag will be grouped together. In the example below, datasets `UKESM` and `ERA5` would belong to the same -group, while the other datasets would belong to either `group1` or `group2` +group, while the other datasets would belong to either ``group1`` or ``group2`` .. code-block:: yaml From 149c6713b02ee1caa568d1eb128dbdf00a6360e7 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 4 Feb 2022 17:45:39 +0100 Subject: [PATCH 153/158] Update doc/recipe/preprocessor.rst Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- doc/recipe/preprocessor.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 617f51a4c9..43124b76b4 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -946,7 +946,6 @@ days in a year may vary between calendars, (sub-)daily data are not supported. span: overlap statistics: [mean, median] exclude: [NCEP] - multi_model_save_input: multi_model_without_saving_input: multi_model_statistics: span: overlap From a3892404c7bdc9b300be2cc4fafb56d902dbb758 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 4 Feb 2022 17:49:53 +0100 Subject: [PATCH 154/158] Reorder doc --- doc/recipe/preprocessor.rst | 51 +++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 617f51a4c9..11e6c68652 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -899,6 +899,28 @@ supported because it requires additional keywords: percentile.). Note that ``ensemble_statistics`` will not return the single model and ensemble files, only the requested ensemble statistics results. +In case of wanting to save both individual ensemble members as well as the statistic results, +the preprocessor chains could be defined as: + +.. code-block:: yaml + preprocessors: + everything_else: &everything_else + area_statistics: ... + regrid_time: ... + multimodel: + <<: *everything_else + ensemble_statistics: + + variables: + tas_datasets: + short_name: tas + preprocessor: everything_else + ... + tas_multimodel: + short_name: tas + preprocessor: multimodel + ... + See also :func:`esmvalcore.preprocessor.ensemble_statistics`. @@ -934,10 +956,6 @@ The preprocessor saves both the input single model files as well as the multi-mo results. In case you do not want to keep the single model files, set the parameter ``keep_input_datasets`` to ``false`` (default value is ``true``). -Input datasets may have different time coordinates. The multi-model statistics -preprocessor sets a common time coordinate on all datasets. As the number of -days in a year may vary between calendars, (sub-)daily data are not supported. - .. code-block:: yaml preprocessors: @@ -954,6 +972,10 @@ days in a year may vary between calendars, (sub-)daily data are not supported. exclude: [NCEP] keep_input_datasets: false +Input datasets may have different time coordinates. The multi-model statistics +preprocessor sets a common time coordinate on all datasets. As the number of +days in a year may vary between calendars, (sub-)daily data are not supported. + Multi-model statistics also supports a ``groupby`` argument. You can group by any dataset key (``project``, ``experiment``, etc.) or a combination of keys in a list. You can also add an arbitrary tag to a dataset definition and then group by that tag. When @@ -979,27 +1001,6 @@ can group by ``ensemble_statistics`` as well. For example: This will first compute ensemble mean and median, and then compute the multi-model min and max separately for the ensemble means and medians. Note that this combination will not save the individual ensemble members, only the ensemble and multimodel statistics results. -In case of wanting to save both individual ensemble members as well as the statistic results, -the preprocessor chains could be defined as: - -.. code-block:: yaml - preprocessors: - everything_else: &everything_else - area_statistics: ... - regrid_time: ... - multimodel: - <<: *everything_else - ensemble_statistics: - -variables: - tas_datasets: - short_name: tas - preprocessor: everything_else - ... - tas_multimodel: - short_name: tas - preprocessor: multimodel - ... When grouping by a tag not defined in all datasets, the datasets missing the tag will be grouped together. In the example below, datasets `UKESM` and `ERA5` would belong to the same From 6419137e6af7cadcc4e502e7b01320c77397e8fd Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 4 Feb 2022 18:11:44 +0100 Subject: [PATCH 155/158] Fix identations --- doc/recipe/preprocessor.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index c0bf8e0ae2..bc858ded77 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -903,23 +903,23 @@ In case of wanting to save both individual ensemble members as well as the stati the preprocessor chains could be defined as: .. code-block:: yaml - preprocessors: - everything_else: &everything_else - area_statistics: ... - regrid_time: ... - multimodel: - <<: *everything_else - ensemble_statistics: - - variables: - tas_datasets: - short_name: tas - preprocessor: everything_else - ... - tas_multimodel: - short_name: tas - preprocessor: multimodel - ... + preprocessors: + everything_else: &everything_else + area_statistics: ... + regrid_time: ... + multimodel: + <<: *everything_else + ensemble_statistics: + + variables: + tas_datasets: + short_name: tas + preprocessor: everything_else + ... + tas_multimodel: + short_name: tas + preprocessor: multimodel + ... See also :func:`esmvalcore.preprocessor.ensemble_statistics`. From b78a84d73d525967848be24d9941c3dfae2080d7 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 4 Feb 2022 18:24:56 +0100 Subject: [PATCH 156/158] Missed one space --- doc/recipe/preprocessor.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index bc858ded77..5986495ca5 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -903,23 +903,23 @@ In case of wanting to save both individual ensemble members as well as the stati the preprocessor chains could be defined as: .. code-block:: yaml - preprocessors: - everything_else: &everything_else - area_statistics: ... - regrid_time: ... - multimodel: - <<: *everything_else - ensemble_statistics: - - variables: - tas_datasets: - short_name: tas - preprocessor: everything_else - ... - tas_multimodel: - short_name: tas - preprocessor: multimodel - ... + preprocessors: + everything_else: &everything_else + area_statistics: ... + regrid_time: ... + multimodel: + <<: *everything_else + ensemble_statistics: + + variables: + tas_datasets: + short_name: tas + preprocessor: everything_else + ... + tas_multimodel: + short_name: tas + preprocessor: multimodel + ... See also :func:`esmvalcore.preprocessor.ensemble_statistics`. From d43c4738bb5676ff28d4422231bebbbe2e3415c1 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 4 Feb 2022 18:43:52 +0100 Subject: [PATCH 157/158] ... --- doc/recipe/preprocessor.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 5986495ca5..8fe97ae41a 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -903,6 +903,7 @@ In case of wanting to save both individual ensemble members as well as the stati the preprocessor chains could be defined as: .. code-block:: yaml + preprocessors: everything_else: &everything_else area_statistics: ... From 266537426294559a92ff18724f46def7247dc958 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Feb 2022 09:18:42 +0100 Subject: [PATCH 158/158] Adapted docstring of multi_model_statistic functions --- esmvalcore/preprocessor/_multimodel.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 1c8e11a15e..800a035ab0 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -394,14 +394,14 @@ def multi_model_statistics(products, ---------- products: list Cubes (or products) over which the statistics will be computed. - statistics: list - Statistical metrics to be computed, e.g. [``mean``, ``max``]. Choose - from the operators listed in the iris.analysis package. Percentiles can - be specified like ``pXX.YY``. span: str Overlap or full; if overlap, statitstics are computed on common time- span; if full, statistics are computed on full time spans, ignoring missing data. + statistics: list + Statistical metrics to be computed, e.g. [``mean``, ``max``]. Choose + from the operators listed in the iris.analysis package. Percentiles can + be specified like ``pXX.YY``. output_products: dict For internal use only. A dict with statistics names as keys and preprocessorfiles as values. If products are passed as input, the @@ -476,9 +476,10 @@ def ensemble_statistics(products, statistics, For internal use only. A dict with statistics names as keys and preprocessorfiles as values. If products are passed as input, the statistics cubes will be assigned to these output products. - keep_input_datasets: bool - If True, the output will include the input datasets. - If False, only the computed statistics will be returned. + span: str (default: 'overlap') + Overlap or full; if overlap, statitstics are computed on common time- + span; if full, statistics are computed on full time spans, ignoring + missing data. Returns -------