From 9db397343b3bdaf0b0165bf7812ff5d545b96bbe Mon Sep 17 00:00:00 2001 From: RaghuSpaceRajan <15613406+RaghuSpaceRajan@users.noreply.github.com> Date: Wed, 22 Jan 2020 20:35:40 +0100 Subject: [PATCH 01/15] Added grid generation utlity for ConfigSpaces --- ConfigSpace/util.pyx | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index dcd549f3..14f08429 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -430,3 +430,62 @@ def fix_types(configuration: dict, else: raise TypeError("Unknown hyperparameter type %s" % type(param)) return configuration + +def generate_grid(configuration_space: ConfigurationSpace, + num_steps_dict: Union[None, Dict[str, Union[str, float, int]]] = None) -> List: + """ + configuration_space: :class:`~ConfigSpace.configuration_space.ConfigurationSpace` + The Configuration space over which to create a grid of HyperParameter Configuration values. It knows the types for all parameter values. + + num_steps_dict: dict + A dict containing the number of points to divide the grid side formed by Hyperparameters which are either of type UniformFloatHyperparameter or type UniformIntegerHyperparameter. The keys in the dict should be the names of the corresponding Hyperparameters and the values should be the number of points to divide the grid side formed by the corresponding Hyperparameter in to. + """ + + value_sets = [] # list of tuples: each tuple is the grid values to be taken on by a Hyperparameter + grid = [] + + for param in configuration_space.get_hyperparameters(): + param_name = param.name + if isinstance(param, (CategoricalHyperparameter)): + value_sets.append(param.choices) + + elif isinstance(param, (OrdinalHyperparameter)): + value_sets.append(param.sequence) + + elif isinstance(param, Constant): + value_sets.append(tuple([param.value, ])) + + elif isinstance(param, UniformFloatHyperparameter): + num_steps = num_steps_dict[param.name] + if param.log: + lower, upper = np.log([param.lower, param.upper]) + else: + lower, upper = param.lower, param.upper + grid_points = np.linspace(lower, upper, num_steps) + if param.log: + grid_points = np.exp(grid_points) + value_sets.append(tuple(grid_points)) + + elif isinstance(param, UniformIntegerHyperparameter): + num_steps = num_steps_dict[param.name] + if param.log: + lower, upper = np.log([param.lower, param.upper]) + else: + lower, upper = param.lower, param.upper + grid_points = np.linspace(lower, upper, num_steps) + if param.log: + grid_points = np.exp(grid_points).astype(int) + value_sets.append(tuple(grid_points)) + + else: + raise TypeError("Unknown hyperparameter type %s" % type(param)) + + import itertools + for element in itertools.product(*value_sets): + config_dict = {} + for i, param in enumerate(configuration_space.get_hyperparameters()): + config_dict[param.name] = element[i] + grid_point = Configuration(cs, config_dict) + grid.append(grid_point) + + return grid From 7e8ecf09a40d4433012ea65683d46cfef5ee9f31 Mon Sep 17 00:00:00 2001 From: RaghuRajan <15613406+RaghuSpaceRajan@users.noreply.github.com> Date: Thu, 23 Jan 2020 13:32:30 +0100 Subject: [PATCH 02/15] Added missing List import from typing. Updated documentation. --- ConfigSpace/util.pyx | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 14f08429..8c1a84ff 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -30,7 +30,7 @@ from collections import deque import copy -from typing import Union, Dict, Generator +from typing import Union, Dict, Generator, List import numpy as np # type: ignore from ConfigSpace import Configuration, ConfigurationSpace @@ -434,27 +434,37 @@ def fix_types(configuration: dict, def generate_grid(configuration_space: ConfigurationSpace, num_steps_dict: Union[None, Dict[str, Union[str, float, int]]] = None) -> List: """ + Generate a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. + + Parameters + ---------- configuration_space: :class:`~ConfigSpace.configuration_space.ConfigurationSpace` The Configuration space over which to create a grid of HyperParameter Configuration values. It knows the types for all parameter values. num_steps_dict: dict A dict containing the number of points to divide the grid side formed by Hyperparameters which are either of type UniformFloatHyperparameter or type UniformIntegerHyperparameter. The keys in the dict should be the names of the corresponding Hyperparameters and the values should be the number of points to divide the grid side formed by the corresponding Hyperparameter in to. + + Returns + ------- + list + List containing Configurations. It is a cartesian product of tuples of HyperParameter values. Each tuple lists the possible values taken by the corresponding HyperParameter. Within the cartesian product, in each element, the ordering of HyperParameters is the same for the OrderedDict within the ConfigurationSpace. + """ - + value_sets = [] # list of tuples: each tuple is the grid values to be taken on by a Hyperparameter grid = [] - + for param in configuration_space.get_hyperparameters(): param_name = param.name if isinstance(param, (CategoricalHyperparameter)): value_sets.append(param.choices) - + elif isinstance(param, (OrdinalHyperparameter)): value_sets.append(param.sequence) elif isinstance(param, Constant): value_sets.append(tuple([param.value, ])) - + elif isinstance(param, UniformFloatHyperparameter): num_steps = num_steps_dict[param.name] if param.log: @@ -479,7 +489,7 @@ def generate_grid(configuration_space: ConfigurationSpace, else: raise TypeError("Unknown hyperparameter type %s" % type(param)) - + import itertools for element in itertools.product(*value_sets): config_dict = {} From 325a91f0c4357453e365020d16dfef7af3c1bbd3 Mon Sep 17 00:00:00 2001 From: RaghuRajan <15613406+RaghuSpaceRajan@users.noreply.github.com> Date: Thu, 23 Jan 2020 14:09:23 +0100 Subject: [PATCH 03/15] Removed bug - typo. --- ConfigSpace/util.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 8c1a84ff..1151a80b 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -495,7 +495,7 @@ def generate_grid(configuration_space: ConfigurationSpace, config_dict = {} for i, param in enumerate(configuration_space.get_hyperparameters()): config_dict[param.name] = element[i] - grid_point = Configuration(cs, config_dict) + grid_point = Configuration(configuration_space, config_dict) grid.append(grid_point) return grid From 1db639577d668dda2c619f80a212d13e460f9f77 Mon Sep 17 00:00:00 2001 From: RaghuRajan <15613406+RaghuSpaceRajan@users.noreply.github.com> Date: Thu, 23 Jan 2020 14:14:34 +0100 Subject: [PATCH 04/15] Updated doc. --- ConfigSpace/util.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 1151a80b..73bf82e1 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -434,7 +434,7 @@ def fix_types(configuration: dict, def generate_grid(configuration_space: ConfigurationSpace, num_steps_dict: Union[None, Dict[str, Union[str, float, int]]] = None) -> List: """ - Generate a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. + Generate a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. Currently ignores conditional HyperParameters. Parameters ---------- From 78c839eb6f5e89e41a24cb50eda4c94ac5b90053 Mon Sep 17 00:00:00 2001 From: RaghuRajan <15613406+RaghuSpaceRajan@users.noreply.github.com> Date: Thu, 13 Feb 2020 18:19:30 +0100 Subject: [PATCH 05/15] Updated code to be object-oriented and consider conditional spaces. --- ConfigSpace/util.pyx | 148 ++++++++++++++++++++++++++++++++----------- 1 file changed, 111 insertions(+), 37 deletions(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 73bf82e1..3be1f6c3 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -431,71 +431,145 @@ def fix_types(configuration: dict, raise TypeError("Unknown hyperparameter type %s" % type(param)) return configuration -def generate_grid(configuration_space: ConfigurationSpace, - num_steps_dict: Union[None, Dict[str, Union[str, float, int]]] = None) -> List: - """ - Generate a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. Currently ignores conditional HyperParameters. +class ConfigSpaceGrid: - Parameters - ---------- - configuration_space: :class:`~ConfigSpace.configuration_space.ConfigurationSpace` - The Configuration space over which to create a grid of HyperParameter Configuration values. It knows the types for all parameter values. + def __init__(self, configuration_space: ConfigurationSpace,): + self.configuration_space = configuration_space - num_steps_dict: dict - A dict containing the number of points to divide the grid side formed by Hyperparameters which are either of type UniformFloatHyperparameter or type UniformIntegerHyperparameter. The keys in the dict should be the names of the corresponding Hyperparameters and the values should be the number of points to divide the grid side formed by the corresponding Hyperparameter in to. + def generate_grid(self, + num_steps_dict: Union[None, Dict[str, int]] = None, + ) -> List[Configuration]: + """ + Generates a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. - Returns - ------- - list - List containing Configurations. It is a cartesian product of tuples of HyperParameter values. Each tuple lists the possible values taken by the corresponding HyperParameter. Within the cartesian product, in each element, the ordering of HyperParameters is the same for the OrderedDict within the ConfigurationSpace. + Parameters + ---------- + configuration_space: :class:`~ConfigSpace.configuration_space.ConfigurationSpace` + The Configuration space over which to create a grid of HyperParameter Configuration values. It knows the types for all parameter values. - """ + num_steps_dict: dict + A dict containing the number of points to divide the grid side formed by Hyperparameters which are either of type UniformFloatHyperparameter or type UniformIntegerHyperparameter. The keys in the dict should be the names of the corresponding Hyperparameters and the values should be the number of points to divide the grid side formed by the corresponding Hyperparameter in to. - value_sets = [] # list of tuples: each tuple is the grid values to be taken on by a Hyperparameter - grid = [] + Returns + ------- + list + List containing Configurations. It is a cartesian product of tuples of HyperParameter values. Each tuple lists the possible values taken by the corresponding HyperParameter. Within the cartesian product, in each element, the ordering of HyperParameters is the same for the OrderedDict within the ConfigurationSpace. + """ - for param in configuration_space.get_hyperparameters(): - param_name = param.name + value_sets = [] # list of tuples: each tuple within is the grid values to be taken on by a Hyperparameter + hp_names = [] + + for hp_name in self.configuration_space._children['__HPOlib_configuration_space_root__']: + value_sets.append(self.get_value_set(hp_name)) + hp_names.append(hp_name) + + unchecked_grid_pts = self.get_cartesian_product(value_sets, hp_names) + checked_grid_pts = [] + condtional_grid_lens = [] + + while len(unchecked_grid_pts) > 0: + try: + grid_point = Configuration(self.configuration_space, unchecked_grid_pts[0]) + checked_grid_pts.append(grid_point) + except ValueError as e: + value_sets = [] + hp_names = [] + new_active_hp_names = [] + + for hp_name in unchecked_grid_pts[0]: # For loop over currently active HP names + value_sets.append(tuple([unchecked_grid_pts[0][hp_name], ])) + hp_names.append(hp_name) + for new_hp_name in self.configuration_space._children[hp_name]: # Checks the HPs already active for their children also being active + if new_hp_name not in new_active_hp_names and new_hp_name not in unchecked_grid_pts[0]: + all_cond_ = True + for cond in self.configuration_space._parent_conditions_of[new_hp_name]: + if not cond.evaluate(unchecked_grid_pts[0]): + all_cond_ = False + if all_cond_: + new_active_hp_names.append(new_hp_name) + + + for hp_name in new_active_hp_names: + value_sets.append(self.get_value_set(hp_name)) + hp_names.append(hp_name) + if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? + new_conditonal_grid = self.get_cartesian_product(value_sets, hp_names) + condtional_grid_lens.append(len(new_conditonal_grid)) + unchecked_grid_pts += new_conditonal_grid + del unchecked_grid_pts[0] + + return checked_grid_pts + + def get_value_set(self, hp_name): + param = self.configuration_space.get_hyperparameter(hp_name) if isinstance(param, (CategoricalHyperparameter)): - value_sets.append(param.choices) + return param.choices elif isinstance(param, (OrdinalHyperparameter)): - value_sets.append(param.sequence) + return param.sequence elif isinstance(param, Constant): - value_sets.append(tuple([param.value, ])) + return tuple([param.value, ]) elif isinstance(param, UniformFloatHyperparameter): - num_steps = num_steps_dict[param.name] if param.log: lower, upper = np.log([param.lower, param.upper]) else: lower, upper = param.lower, param.upper - grid_points = np.linspace(lower, upper, num_steps) + + if num_steps_dict is not None: + num_steps = num_steps_dict[param.name] + grid_points = np.linspace(lower, upper, num_steps) + else: + grid_points = np.arange(lower, upper, param.q) # check for log and for rounding issues + if param.log: grid_points = np.exp(grid_points) - value_sets.append(tuple(grid_points)) + + # Avoiding rounding off issues + if grid_points[0] < param.lower: + grid_points[0] = param.lower + if grid_points[-1] > param.upper: + grid_points[-1] = param.upper + + return tuple(grid_points) elif isinstance(param, UniformIntegerHyperparameter): - num_steps = num_steps_dict[param.name] if param.log: lower, upper = np.log([param.lower, param.upper]) else: lower, upper = param.lower, param.upper - grid_points = np.linspace(lower, upper, num_steps) + + if num_steps_dict is not None: + num_steps = num_steps_dict[param.name] + grid_points = np.linspace(lower, upper, num_steps) + else: + grid_points = np.arange(lower, upper, param.q) # check for log and for rounding issues + if param.log: - grid_points = np.exp(grid_points).astype(int) - value_sets.append(tuple(grid_points)) + grid_points = np.exp(grid_points) + grid_points = grid_points.astype(int) + + # Avoiding rounding off issues + if grid_points[0] < param.lower: + grid_points[0] = param.lower + if grid_points[-1] > param.upper: + grid_points[-1] = param.upper + + return tuple(grid_points) else: raise TypeError("Unknown hyperparameter type %s" % type(param)) - import itertools - for element in itertools.product(*value_sets): - config_dict = {} - for i, param in enumerate(configuration_space.get_hyperparameters()): - config_dict[param.name] = element[i] - grid_point = Configuration(configuration_space, config_dict) - grid.append(grid_point) - return grid + + def get_cartesian_product(self, value_sets, hp_names): + grid = [] + import itertools + for i, element in enumerate(itertools.product(*value_sets)): + config_dict = {} + for j, hp_name in enumerate(hp_names): + config_dict[hp_name] = element[j] + grid.append(config_dict) + + return grid From 137fc9eb29c4984bc7fbfd883942974073f4d701 Mon Sep 17 00:00:00 2001 From: RaghuSpaceRajan <15613406+RaghuSpaceRajan@users.noreply.github.com> Date: Thu, 13 Feb 2020 18:48:50 +0100 Subject: [PATCH 06/15] Removed bug with num_steps_dict --- ConfigSpace/util.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 3be1f6c3..34d18aa2 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -460,7 +460,7 @@ class ConfigSpaceGrid: hp_names = [] for hp_name in self.configuration_space._children['__HPOlib_configuration_space_root__']: - value_sets.append(self.get_value_set(hp_name)) + value_sets.append(self.get_value_set(num_steps_dict, hp_name)) hp_names.append(hp_name) unchecked_grid_pts = self.get_cartesian_product(value_sets, hp_names) @@ -490,7 +490,7 @@ class ConfigSpaceGrid: for hp_name in new_active_hp_names: - value_sets.append(self.get_value_set(hp_name)) + value_sets.append(self.get_value_set(num_steps_dict, hp_name)) hp_names.append(hp_name) if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? new_conditonal_grid = self.get_cartesian_product(value_sets, hp_names) @@ -500,7 +500,7 @@ class ConfigSpaceGrid: return checked_grid_pts - def get_value_set(self, hp_name): + def get_value_set(self, num_steps_dict, hp_name): param = self.configuration_space.get_hyperparameter(hp_name) if isinstance(param, (CategoricalHyperparameter)): return param.choices From 1d85ba72acbe64519c7a363c8a24d12d82eac14f Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Mon, 15 Mar 2021 14:53:45 +0100 Subject: [PATCH 07/15] Added some test cases --- test/test_util.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_util.py b/test/test_util.py index b731f3f2..48be5b2d 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -307,3 +307,31 @@ def test_check_neighbouring_config_diamond_str(self): expected_array = np.array([1, np.nan, np.nan, np.nan]) np.testing.assert_almost_equal(new_array, expected_array) + + + def test_generate_grid(self): + cs = ConfigurationSpace(seed=1234) + + cat1 = CategoricalHyperparameter(name='cat1', choices=['T', 'F']) + const1 = Constant(name='const1', value=4) + float1 = UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) + int1 = UniformIntegerHyperparameter(name='int1', lower=10, upper=100, log=True) + ord1 = OrdinalHyperparameter(name='ord1', sequence=['1', '2', '3']) + + cs.add_hyperparameters([float1, int1, cat1, ord1, const1]) + + num_steps_dict = {'float1': 11, 'int1': 6} + generated_grid = generate_grid(cs, num_steps_dict) + + # Check randomly pre-selected values in the generated_grid + self.assertEqual(len(generated_grid), 396) # 2 * 1 * 11 * 6 * 3 total diff. possibilities for the Configuration (i.e., for each tuple of HPs) + self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') + self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') # + self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) # + self.assertAlmostEqual(generated_grid[55].get_dictionary()['float1'], -0.4, places=2) # The 2 most frequently changing HPs (int1 and ord1) have 3*6 = 18 different values for each value of float1, so the 4th value of float1 of -0.4 is reached after 3*18 = 54 values. + self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) # 5th diff. value for int1 after 4*3 = 12 values. Reasoning as above. + self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') # + self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') # + self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') # + + #tests for conditional spaces: 1 basic one for 1 condition; 1 for tree of conditions; 1 for tree of conditions with OrConjunction and AndConjunction; From 0e0cd1dbafc8485c3d3e44add342f0c9a223ed9f Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Tue, 16 Mar 2021 18:11:59 +0100 Subject: [PATCH 08/15] Added incomplete test cases for quantization and conditional ConfigSpaces. --- ConfigSpace/util.pyx | 62 ++++++++++++++++++++++++++++++++++++-------- test/test_util.py | 57 +++++++++++++++++++++++++++++----------- 2 files changed, 93 insertions(+), 26 deletions(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 34d18aa2..5c2542f7 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -431,7 +431,7 @@ def fix_types(configuration: dict, raise TypeError("Unknown hyperparameter type %s" % type(param)) return configuration -class ConfigSpaceGrid: +class ConfigSpaceGrid: #TODO cdef? def __init__(self, configuration_space: ConfigurationSpace,): self.configuration_space = configuration_space @@ -459,13 +459,15 @@ class ConfigSpaceGrid: value_sets = [] # list of tuples: each tuple within is the grid values to be taken on by a Hyperparameter hp_names = [] + # Get HP names and allowed grid values they can take for the HPs at the top level of ConfigSpace tree for hp_name in self.configuration_space._children['__HPOlib_configuration_space_root__']: value_sets.append(self.get_value_set(num_steps_dict, hp_name)) hp_names.append(hp_name) - unchecked_grid_pts = self.get_cartesian_product(value_sets, hp_names) + # Create a Cartesian product of above allowed values for the HPs. Hold them in an "unchecked" list because some of the conditionally dependent HPs may become active for some of the elements of the Cartesian product and in these cases creating a Configuration would throw an Error (see below). + unchecked_grid_pts = self.get_cartesian_product(value_sets, hp_names) # Creates a list of Configuration dicts checked_grid_pts = [] - condtional_grid_lens = [] + conditional_grid_lens = [] while len(unchecked_grid_pts) > 0: try: @@ -476,10 +478,10 @@ class ConfigSpaceGrid: hp_names = [] new_active_hp_names = [] - for hp_name in unchecked_grid_pts[0]: # For loop over currently active HP names + for hp_name in unchecked_grid_pts[0]: # "for" loop over currently active HP names value_sets.append(tuple([unchecked_grid_pts[0][hp_name], ])) hp_names.append(hp_name) - for new_hp_name in self.configuration_space._children[hp_name]: # Checks the HPs already active for their children also being active + for new_hp_name in self.configuration_space._children[hp_name]: # Checks if the conditionally dependent children of already active HPs are now active if new_hp_name not in new_active_hp_names and new_hp_name not in unchecked_grid_pts[0]: all_cond_ = True for cond in self.configuration_space._parent_conditions_of[new_hp_name]: @@ -492,15 +494,34 @@ class ConfigSpaceGrid: for hp_name in new_active_hp_names: value_sets.append(self.get_value_set(num_steps_dict, hp_name)) hp_names.append(hp_name) - if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? + if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? #TODO new_conditonal_grid = self.get_cartesian_product(value_sets, hp_names) - condtional_grid_lens.append(len(new_conditonal_grid)) + conditional_grid_lens.append(len(new_conditonal_grid)) unchecked_grid_pts += new_conditonal_grid del unchecked_grid_pts[0] return checked_grid_pts def get_value_set(self, num_steps_dict, hp_name): + ''' + Gets values along the grid for a particular hyperparameter. + + Uses the num_steps_dict to determine number of grid values for UniformFloatHyperparameter and UniformIntegerHyperparameter. If these values are not present in num_steps_dict, the quantization, q, objects of these classes will be used to divide the grid. NOTE: If q is None, this results in np.arange using None for its step size parameter which results in a quantization of 1. + + Parameters + ---------- + num_steps_dict: dict + Same description as above + + hp_name: str + Hyperparameter name + + Returns + ------- + tuple + Holds grid values for the given hyperparameter + + ''' param = self.configuration_space.get_hyperparameter(hp_name) if isinstance(param, (CategoricalHyperparameter)): return param.choices @@ -517,11 +538,11 @@ class ConfigSpaceGrid: else: lower, upper = param.lower, param.upper - if num_steps_dict is not None: + if param.name in num_steps_dict: num_steps = num_steps_dict[param.name] grid_points = np.linspace(lower, upper, num_steps) else: - grid_points = np.arange(lower, upper, param.q) # check for log and for rounding issues + grid_points = np.arange(lower, upper + param.q, param.q) # check for log and for rounding issues if param.log: grid_points = np.exp(grid_points) @@ -540,11 +561,11 @@ class ConfigSpaceGrid: else: lower, upper = param.lower, param.upper - if num_steps_dict is not None: + if param.name in num_steps_dict: num_steps = num_steps_dict[param.name] grid_points = np.linspace(lower, upper, num_steps) else: - grid_points = np.arange(lower, upper, param.q) # check for log and for rounding issues + grid_points = np.arange(lower, upper + param.q, param.q) # check for log and for rounding issues if param.log: grid_points = np.exp(grid_points) @@ -564,6 +585,25 @@ class ConfigSpaceGrid: def get_cartesian_product(self, value_sets, hp_names): + ''' + Returns a grid for a subspace of the configuration with given hyperparameters and their grid values. + + Takes a list of tuples of grid values of the hyperparameters and list of hyperparameter names. The outer list iterates over the hyperparameters corresponding to the order in the list of hyperparameter names. The inner tuples contain grid values of the hyperparameters for each hyperparameter. + + Parameters + ---------- + value_sets: list of tuples + Same description as return value of get_value_set() + + hp_names: list of strs + List of hyperparameter names + + Returns + ------- + list of dicts + List of configuration dicts + + ''' grid = [] import itertools for i, element in enumerate(itertools.product(*value_sets)): diff --git a/test/test_util.py b/test/test_util.py index 48be5b2d..70f99737 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -36,7 +36,7 @@ EqualsCondition, AndConjunction, OrConjunction from ConfigSpace.read_and_write.pcs import read from ConfigSpace.util import impute_inactive_values, get_random_neighbor, \ - get_one_exchange_neighbourhood, deactivate_inactive_hyperparameters + get_one_exchange_neighbourhood, deactivate_inactive_hyperparameters, ConfigSpaceGrid import ConfigSpace.c_util @@ -321,17 +321,44 @@ def test_generate_grid(self): cs.add_hyperparameters([float1, int1, cat1, ord1, const1]) num_steps_dict = {'float1': 11, 'int1': 6} - generated_grid = generate_grid(cs, num_steps_dict) - - # Check randomly pre-selected values in the generated_grid - self.assertEqual(len(generated_grid), 396) # 2 * 1 * 11 * 6 * 3 total diff. possibilities for the Configuration (i.e., for each tuple of HPs) - self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') - self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') # - self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) # - self.assertAlmostEqual(generated_grid[55].get_dictionary()['float1'], -0.4, places=2) # The 2 most frequently changing HPs (int1 and ord1) have 3*6 = 18 different values for each value of float1, so the 4th value of float1 of -0.4 is reached after 3*18 = 54 values. - self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) # 5th diff. value for int1 after 4*3 = 12 values. Reasoning as above. - self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') # - self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') # - self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') # - - #tests for conditional spaces: 1 basic one for 1 condition; 1 for tree of conditions; 1 for tree of conditions with OrConjunction and AndConjunction; + #TODO uncomment below and add asserts for later test cases + # generated_grid = generate_grid(cs, num_steps_dict) + # + # # Check randomly pre-selected values in the generated_grid + # self.assertEqual(len(generated_grid), 396) # 2 * 1 * 11 * 6 * 3 total diff. possibilities for the Configuration (i.e., for each tuple of HPs) + # self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') + # self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') # + # self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) # + # self.assertAlmostEqual(generated_grid[55].get_dictionary()['float1'], -0.4, places=2) # The 2 most frequently changing HPs (int1 and ord1) have 3*6 = 18 different values for each value of float1, so the 4th value of float1 of -0.4 is reached after 3*18 = 54 values. + # self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) # 5th diff. value for int1 after 4*3 = 12 values. Reasoning as above. + # self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') # + # self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') # + # self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') # + + #tests for quantization and conditional spaces: 1 basic one for 1 condition; 1 for tree of conditions; 1 for tree of conditions with OrConjunction and AndConjunction; + cs2 = CS.ConfigurationSpace(seed=123) + float1 = CSH.UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) + int1 = CSH.UniformIntegerHyperparameter(name='int1', lower=0, upper=1000, log=False, q=500) + cs2.add_hyperparameters([float1, int1]) + + int2_cond = CSH.UniformIntegerHyperparameter(name='int2_cond', lower=10, upper=100, log=True) + cs2.add_hyperparameters([int2_cond]) + cond_1 = CS.AndConjunction(CS.LessThanCondition(int2_cond, float1, -0.5), + CS.GreaterThanCondition(int2_cond, int1, 600)) + cs2.add_conditions([cond_1]) + cat1_cond = CSH.CategoricalHyperparameter(name='cat1_cond', choices=['apple', 'orange']) + cs2.add_hyperparameters([cat1_cond]) + cond_2 = CS.AndConjunction(CS.GreaterThanCondition(cat1_cond, int1, 300), + CS.LessThanCondition(cat1_cond, int1, 700), + CS.GreaterThanCondition(cat1_cond, float1, -0.5), + CS.LessThanCondition(cat1_cond, float1, 0.5) + ) + cs2.add_conditions([cond_2]) + float2_cond = CSH.UniformFloatHyperparameter(name='float2_cond', lower=10., upper=100., log=True) + cs2.add_hyperparameters([float2_cond]) + cond_3 = CS.GreaterThanCondition(float2_cond, int2_cond, 50) + cs2.add_conditions([cond_3]) + print(cs2) + num_steps_dict1 = {'float1': 4, 'int2_cond': 3, 'float2_cond': 3 } + configspace_grid = ConfigSpaceGrid(cs2) + generated_grid = configspace_grid.generate_grid(num_steps_dict1) From 758e41b3277e49e81be60dffdb0c2bd12bb901cf Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Tue, 16 Mar 2021 18:29:32 +0100 Subject: [PATCH 09/15] Updated conditional grid generation test cases with asserts --- test/test_util.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index b5e7aae3..a862b9a3 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -308,7 +308,7 @@ def test_check_neighbouring_config_diamond_str(self): np.testing.assert_almost_equal(new_array, expected_array) - + def test_fix_types(self): # Test categorical and ordinal for hyperparameter_type in [CategoricalHyperparameter, OrdinalHyperparameter]: @@ -343,7 +343,7 @@ def test_fix_types(self): c = cs.get_default_configuration().get_dictionary() c_str = {k: str(v) for k, v in c.items()} self.assertEqual(fix_types(c_str, cs), c) - + def test_generate_grid(self): '''Test grid generation''' cs = ConfigurationSpace(seed=1234) @@ -357,7 +357,7 @@ def test_generate_grid(self): cs.add_hyperparameters([float1, int1, cat1, ord1, const1]) num_steps_dict = {'float1': 11, 'int1': 6} - #TODO uncomment below and add asserts for later test cases + #TODO uncomment below # generated_grid = generate_grid(cs, num_steps_dict) # # # Check randomly pre-selected values in the generated_grid @@ -371,7 +371,7 @@ def test_generate_grid(self): # self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') # # self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') # - #tests for quantization and conditional spaces: 1 basic one for 1 condition; 1 for tree of conditions; 1 for tree of conditions with OrConjunction and AndConjunction; + # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps for only some of the int and float HPs. The rest are taken from the 'q' member variables of these HPs. The conditional space tested has 2 levels of conditions. cs2 = CS.ConfigurationSpace(seed=123) float1 = CSH.UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) int1 = CSH.UniformIntegerHyperparameter(name='int1', lower=0, upper=1000, log=False, q=500) @@ -398,3 +398,9 @@ def test_generate_grid(self): num_steps_dict1 = {'float1': 4, 'int2_cond': 3, 'float2_cond': 3 } configspace_grid = ConfigSpaceGrid(cs2) generated_grid = configspace_grid.generate_grid(num_steps_dict1) + self.assertEqual(len(generated_grid), 18) + + # RR: I manually generated the grid and verified the values were correct. Here, we test that a few randomly chosen values in the generated grid correspond to the ones I checked. + self.assertEqual(generated_grid[3].get_dictionary()['int1'], 1000) + self.assertEqual(generated_grid[12].get_dictionary()['cat1_cond'], 'orange') + self.assertAlmostEqual(generated_grid[-2].get_dictionary()['float2_cond'], 31.622776601683803, places=3) From 4ad55a4dc0bb9baf0c32282a334a29ac8ddb134a Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Tue, 16 Mar 2021 18:35:08 +0100 Subject: [PATCH 10/15] Removed bugs for grid gen. tests --- test/test_util.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index a862b9a3..ba098cd6 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -372,27 +372,27 @@ def test_generate_grid(self): # self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') # # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps for only some of the int and float HPs. The rest are taken from the 'q' member variables of these HPs. The conditional space tested has 2 levels of conditions. - cs2 = CS.ConfigurationSpace(seed=123) - float1 = CSH.UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) - int1 = CSH.UniformIntegerHyperparameter(name='int1', lower=0, upper=1000, log=False, q=500) + cs2 = ConfigurationSpace(seed=123) + float1 = UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) + int1 = UniformIntegerHyperparameter(name='int1', lower=0, upper=1000, log=False, q=500) cs2.add_hyperparameters([float1, int1]) - int2_cond = CSH.UniformIntegerHyperparameter(name='int2_cond', lower=10, upper=100, log=True) + int2_cond = UniformIntegerHyperparameter(name='int2_cond', lower=10, upper=100, log=True) cs2.add_hyperparameters([int2_cond]) - cond_1 = CS.AndConjunction(CS.LessThanCondition(int2_cond, float1, -0.5), - CS.GreaterThanCondition(int2_cond, int1, 600)) + cond_1 = AndConjunction(LessThanCondition(int2_cond, float1, -0.5), + GreaterThanCondition(int2_cond, int1, 600)) cs2.add_conditions([cond_1]) - cat1_cond = CSH.CategoricalHyperparameter(name='cat1_cond', choices=['apple', 'orange']) + cat1_cond = CategoricalHyperparameter(name='cat1_cond', choices=['apple', 'orange']) cs2.add_hyperparameters([cat1_cond]) - cond_2 = CS.AndConjunction(CS.GreaterThanCondition(cat1_cond, int1, 300), - CS.LessThanCondition(cat1_cond, int1, 700), - CS.GreaterThanCondition(cat1_cond, float1, -0.5), - CS.LessThanCondition(cat1_cond, float1, 0.5) + cond_2 = AndConjunction(GreaterThanCondition(cat1_cond, int1, 300), + LessThanCondition(cat1_cond, int1, 700), + GreaterThanCondition(cat1_cond, float1, -0.5), + LessThanCondition(cat1_cond, float1, 0.5) ) cs2.add_conditions([cond_2]) - float2_cond = CSH.UniformFloatHyperparameter(name='float2_cond', lower=10., upper=100., log=True) + float2_cond = UniformFloatHyperparameter(name='float2_cond', lower=10., upper=100., log=True) cs2.add_hyperparameters([float2_cond]) - cond_3 = CS.GreaterThanCondition(float2_cond, int2_cond, 50) + cond_3 = GreaterThanCondition(float2_cond, int2_cond, 50) cs2.add_conditions([cond_3]) print(cs2) num_steps_dict1 = {'float1': 4, 'int2_cond': 3, 'float2_cond': 3 } From 26a3ba4c8786a35e85668e771a49d0268edbae9f Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Tue, 16 Mar 2021 18:41:07 +0100 Subject: [PATCH 11/15] Removed bug --- test/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index ba098cd6..4bb34b9b 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -33,7 +33,7 @@ from ConfigSpace import Configuration, ConfigurationSpace, UniformIntegerHyperparameter, \ UniformFloatHyperparameter, CategoricalHyperparameter, Constant, OrdinalHyperparameter, \ - EqualsCondition, AndConjunction, OrConjunction + EqualsCondition, AndConjunction, OrConjunction, LessThanCondition, GreaterThanCondition from ConfigSpace.read_and_write.pcs import read from ConfigSpace.util import impute_inactive_values, get_random_neighbor, \ get_one_exchange_neighbourhood, deactivate_inactive_hyperparameters, fix_types, ConfigSpaceGrid From 19144c34b4b0cce2e9df1bb66943bbd17392c818 Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Tue, 23 Mar 2021 23:25:15 +0100 Subject: [PATCH 12/15] Removed Object-Orientedness, corrected tests, flake8 stuff. --- .gitignore | 6 ++ ConfigSpace/util.pyx | 143 +++++++++++++++++++++---------------------- test/test_util.py | 67 +++++++++++--------- 3 files changed, 115 insertions(+), 101 deletions(-) diff --git a/.gitignore b/.gitignore index 0e2d7cea..d812bf16 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,9 @@ coverage.xml *,cover .hypothesis/ prof/ + +# VSCode +.vscode + +# Running pre-commit seems to generate these +.mypy_cache diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 780d0d16..bfff3533 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -437,78 +437,30 @@ def fix_types(configuration: dict, raise TypeError("Unknown hyperparameter type %s" % type(param)) return configuration -class ConfigSpaceGrid: #TODO cdef? - - def __init__(self, configuration_space: ConfigurationSpace,): - self.configuration_space = configuration_space - - def generate_grid(self, - num_steps_dict: Union[None, Dict[str, int]] = None, - ) -> List[Configuration]: - """ - Generates a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. - - Parameters - ---------- - configuration_space: :class:`~ConfigSpace.configuration_space.ConfigurationSpace` - The Configuration space over which to create a grid of HyperParameter Configuration values. It knows the types for all parameter values. - - num_steps_dict: dict - A dict containing the number of points to divide the grid side formed by Hyperparameters which are either of type UniformFloatHyperparameter or type UniformIntegerHyperparameter. The keys in the dict should be the names of the corresponding Hyperparameters and the values should be the number of points to divide the grid side formed by the corresponding Hyperparameter in to. - - Returns - ------- - list - List containing Configurations. It is a cartesian product of tuples of HyperParameter values. Each tuple lists the possible values taken by the corresponding HyperParameter. Within the cartesian product, in each element, the ordering of HyperParameters is the same for the OrderedDict within the ConfigurationSpace. - """ +cimport cython +@cython.boundscheck(True) # Activate bounds checking +@cython.wraparound(True) # Activate negative indexing +def generate_grid(configuration_space: ConfigurationSpace, + num_steps_dict: Union[None, Dict[str, int]] = None, + ) -> List[Configuration]: + """ + Generates a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. - value_sets = [] # list of tuples: each tuple within is the grid values to be taken on by a Hyperparameter - hp_names = [] + Parameters + ---------- + configuration_space: :class:`~ConfigSpace.configuration_space.ConfigurationSpace` + The Configuration space over which to create a grid of HyperParameter Configuration values. It knows the types for all parameter values. - # Get HP names and allowed grid values they can take for the HPs at the top level of ConfigSpace tree - for hp_name in self.configuration_space._children['__HPOlib_configuration_space_root__']: - value_sets.append(self.get_value_set(num_steps_dict, hp_name)) - hp_names.append(hp_name) + num_steps_dict: dict + A dict containing the number of points to divide the grid side formed by Hyperparameters which are either of type UniformFloatHyperparameter or type UniformIntegerHyperparameter. The keys in the dict should be the names of the corresponding Hyperparameters and the values should be the number of points to divide the grid side formed by the corresponding Hyperparameter in to. - # Create a Cartesian product of above allowed values for the HPs. Hold them in an "unchecked" list because some of the conditionally dependent HPs may become active for some of the elements of the Cartesian product and in these cases creating a Configuration would throw an Error (see below). - unchecked_grid_pts = self.get_cartesian_product(value_sets, hp_names) # Creates a list of Configuration dicts - checked_grid_pts = [] - conditional_grid_lens = [] + Returns + ------- + list + List containing Configurations. It is a cartesian product of tuples of HyperParameter values. Each tuple lists the possible values taken by the corresponding HyperParameter. Within the cartesian product, in each element, the ordering of HyperParameters is the same for the OrderedDict within the ConfigurationSpace. + """ - while len(unchecked_grid_pts) > 0: - try: - grid_point = Configuration(self.configuration_space, unchecked_grid_pts[0]) - checked_grid_pts.append(grid_point) - except ValueError as e: - value_sets = [] - hp_names = [] - new_active_hp_names = [] - - for hp_name in unchecked_grid_pts[0]: # "for" loop over currently active HP names - value_sets.append(tuple([unchecked_grid_pts[0][hp_name], ])) - hp_names.append(hp_name) - for new_hp_name in self.configuration_space._children[hp_name]: # Checks if the conditionally dependent children of already active HPs are now active - if new_hp_name not in new_active_hp_names and new_hp_name not in unchecked_grid_pts[0]: - all_cond_ = True - for cond in self.configuration_space._parent_conditions_of[new_hp_name]: - if not cond.evaluate(unchecked_grid_pts[0]): - all_cond_ = False - if all_cond_: - new_active_hp_names.append(new_hp_name) - - - for hp_name in new_active_hp_names: - value_sets.append(self.get_value_set(num_steps_dict, hp_name)) - hp_names.append(hp_name) - if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? #TODO - new_conditonal_grid = self.get_cartesian_product(value_sets, hp_names) - conditional_grid_lens.append(len(new_conditonal_grid)) - unchecked_grid_pts += new_conditonal_grid - del unchecked_grid_pts[0] - - return checked_grid_pts - - def get_value_set(self, num_steps_dict, hp_name): + def get_value_set(num_steps_dict, hp_name): ''' Gets values along the grid for a particular hyperparameter. @@ -528,7 +480,7 @@ class ConfigSpaceGrid: #TODO cdef? Holds grid values for the given hyperparameter ''' - param = self.configuration_space.get_hyperparameter(hp_name) + param = configuration_space.get_hyperparameter(hp_name) if isinstance(param, (CategoricalHyperparameter)): return param.choices @@ -554,6 +506,7 @@ class ConfigSpaceGrid: #TODO cdef? grid_points = np.exp(grid_points) # Avoiding rounding off issues + print(grid_points) if grid_points[0] < param.lower: grid_points[0] = param.lower if grid_points[-1] > param.upper: @@ -588,9 +541,7 @@ class ConfigSpaceGrid: #TODO cdef? else: raise TypeError("Unknown hyperparameter type %s" % type(param)) - - - def get_cartesian_product(self, value_sets, hp_names): + def get_cartesian_product(value_sets, hp_names): ''' Returns a grid for a subspace of the configuration with given hyperparameters and their grid values. @@ -619,3 +570,51 @@ class ConfigSpaceGrid: #TODO cdef? grid.append(config_dict) return grid + + + value_sets = [] # list of tuples: each tuple within is the grid values to be taken on by a Hyperparameter + hp_names = [] + + # Get HP names and allowed grid values they can take for the HPs at the top level of ConfigSpace tree + for hp_name in configuration_space._children['__HPOlib_configuration_space_root__']: + value_sets.append(get_value_set(num_steps_dict, hp_name)) + hp_names.append(hp_name) + + # Create a Cartesian product of above allowed values for the HPs. Hold them in an "unchecked" list because some of the conditionally dependent HPs may become active for some of the elements of the Cartesian product and in these cases creating a Configuration would throw an Error (see below). + unchecked_grid_pts = get_cartesian_product(value_sets, hp_names) # Creates a list of Configuration dicts + checked_grid_pts = [] + conditional_grid_lens = [] + + while len(unchecked_grid_pts) > 0: + try: + grid_point = Configuration(configuration_space, unchecked_grid_pts[0]) + checked_grid_pts.append(grid_point) + except ValueError as e: + value_sets = [] + hp_names = [] + new_active_hp_names = [] + + for hp_name in unchecked_grid_pts[0]: # "for" loop over currently active HP names + value_sets.append(tuple([unchecked_grid_pts[0][hp_name], ])) + hp_names.append(hp_name) + for new_hp_name in configuration_space._children[hp_name]: # Checks if the conditionally dependent children of already active HPs are now active + if new_hp_name not in new_active_hp_names and new_hp_name not in unchecked_grid_pts[0]: + all_cond_ = True + for cond in configuration_space._parent_conditions_of[new_hp_name]: + if not cond.evaluate(unchecked_grid_pts[0]): + all_cond_ = False + if all_cond_: + new_active_hp_names.append(new_hp_name) + + + for hp_name in new_active_hp_names: + value_sets.append(get_value_set(num_steps_dict, hp_name)) + hp_names.append(hp_name) + if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? #TODO + new_conditonal_grid = get_cartesian_product(value_sets, hp_names) + conditional_grid_lens.append(len(new_conditonal_grid)) + unchecked_grid_pts += new_conditonal_grid + del unchecked_grid_pts[0] + + return checked_grid_pts + diff --git a/test/test_util.py b/test/test_util.py index 4bb34b9b..52d2caed 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -36,7 +36,7 @@ EqualsCondition, AndConjunction, OrConjunction, LessThanCondition, GreaterThanCondition from ConfigSpace.read_and_write.pcs import read from ConfigSpace.util import impute_inactive_values, get_random_neighbor, \ - get_one_exchange_neighbourhood, deactivate_inactive_hyperparameters, fix_types, ConfigSpaceGrid + get_one_exchange_neighbourhood, deactivate_inactive_hyperparameters, fix_types, generate_grid import ConfigSpace.c_util @@ -308,7 +308,6 @@ def test_check_neighbouring_config_diamond_str(self): np.testing.assert_almost_equal(new_array, expected_array) - def test_fix_types(self): # Test categorical and ordinal for hyperparameter_type in [CategoricalHyperparameter, OrdinalHyperparameter]: @@ -357,21 +356,27 @@ def test_generate_grid(self): cs.add_hyperparameters([float1, int1, cat1, ord1, const1]) num_steps_dict = {'float1': 11, 'int1': 6} - #TODO uncomment below - # generated_grid = generate_grid(cs, num_steps_dict) - # - # # Check randomly pre-selected values in the generated_grid - # self.assertEqual(len(generated_grid), 396) # 2 * 1 * 11 * 6 * 3 total diff. possibilities for the Configuration (i.e., for each tuple of HPs) - # self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') - # self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') # - # self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) # - # self.assertAlmostEqual(generated_grid[55].get_dictionary()['float1'], -0.4, places=2) # The 2 most frequently changing HPs (int1 and ord1) have 3*6 = 18 different values for each value of float1, so the 4th value of float1 of -0.4 is reached after 3*18 = 54 values. - # self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) # 5th diff. value for int1 after 4*3 = 12 values. Reasoning as above. - # self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') # - # self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') # - # self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') # - - # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps for only some of the int and float HPs. The rest are taken from the 'q' member variables of these HPs. The conditional space tested has 2 levels of conditions. + generated_grid = generate_grid(cs, num_steps_dict) + + # Check randomly pre-selected values in the generated_grid + # 2 * 1 * 11 * 6 * 3 total diff. possible configurations + self.assertEqual(len(generated_grid), 396) + self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') + self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') + self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) + # The 2 most frequently changing HPs (int1 and ord1) have 3 * 6 = 18 different values for + # each value of float1, so the 4th value of float1 of -0.4 is reached after + # 3 * 18 = 54 values in the generated_grid (and remains the same for the next 18 values): + self.assertAlmostEqual(generated_grid[55].get_dictionary()['float1'], -0.4, places=2) + # 5th diff. value for int1 after 4 * 3 = 12 values. Reasoning as above. + self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) + self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') + self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') + self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') + + # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps + # for only some of the int and float HPs. The rest are taken from the 'q' member variables + # of these HPs. The conditional space tested has 2 levels of conditions. cs2 = ConfigurationSpace(seed=123) float1 = UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) int1 = UniformIntegerHyperparameter(name='int1', lower=0, upper=1000, log=False, q=500) @@ -380,27 +385,31 @@ def test_generate_grid(self): int2_cond = UniformIntegerHyperparameter(name='int2_cond', lower=10, upper=100, log=True) cs2.add_hyperparameters([int2_cond]) cond_1 = AndConjunction(LessThanCondition(int2_cond, float1, -0.5), - GreaterThanCondition(int2_cond, int1, 600)) + GreaterThanCondition(int2_cond, int1, 600)) cs2.add_conditions([cond_1]) cat1_cond = CategoricalHyperparameter(name='cat1_cond', choices=['apple', 'orange']) cs2.add_hyperparameters([cat1_cond]) cond_2 = AndConjunction(GreaterThanCondition(cat1_cond, int1, 300), - LessThanCondition(cat1_cond, int1, 700), - GreaterThanCondition(cat1_cond, float1, -0.5), - LessThanCondition(cat1_cond, float1, 0.5) - ) + LessThanCondition(cat1_cond, int1, 700), + GreaterThanCondition(cat1_cond, float1, -0.5), + LessThanCondition(cat1_cond, float1, 0.5) + ) cs2.add_conditions([cond_2]) - float2_cond = UniformFloatHyperparameter(name='float2_cond', lower=10., upper=100., log=True) + float2_cond = UniformFloatHyperparameter(name='float2_cond', + lower=10., upper=100., log=True) cs2.add_hyperparameters([float2_cond]) cond_3 = GreaterThanCondition(float2_cond, int2_cond, 50) cs2.add_conditions([cond_3]) - print(cs2) - num_steps_dict1 = {'float1': 4, 'int2_cond': 3, 'float2_cond': 3 } - configspace_grid = ConfigSpaceGrid(cs2) - generated_grid = configspace_grid.generate_grid(num_steps_dict1) + # print(cs2) + num_steps_dict1 = {'float1': 4, 'int2_cond': 3, 'float2_cond': 3} + generated_grid = generate_grid(cs2, num_steps_dict1) + # print(generated_grid) self.assertEqual(len(generated_grid), 18) - # RR: I manually generated the grid and verified the values were correct. Here, we test that a few randomly chosen values in the generated grid correspond to the ones I checked. + # RR: I manually generated the grid and verified the values were correct. + # Here, we test that a few randomly chosen values in the generated grid + # correspond to the ones I checked. self.assertEqual(generated_grid[3].get_dictionary()['int1'], 1000) self.assertEqual(generated_grid[12].get_dictionary()['cat1_cond'], 'orange') - self.assertAlmostEqual(generated_grid[-2].get_dictionary()['float2_cond'], 31.622776601683803, places=3) + self.assertAlmostEqual(generated_grid[-2].get_dictionary()['float2_cond'], + 31.622776601683803, places=3) From 60a495796ccf860bd18365bf1627481a27c2bd27 Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Wed, 24 Mar 2021 19:51:13 +0100 Subject: [PATCH 13/15] Updated based on Matthias' feedback --- ConfigSpace/util.pyx | 38 +++++++++--------- test/test_util.py | 95 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 112 insertions(+), 21 deletions(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index bfff3533..7d26370d 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -30,7 +30,7 @@ from collections import deque import copy -from typing import Union, Dict, Generator, List +from typing import Union, Dict, Generator, List, Tuple, Optional import numpy as np # type: ignore from ConfigSpace import Configuration, ConfigurationSpace @@ -39,6 +39,7 @@ from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ UniformFloatHyperparameter, UniformIntegerHyperparameter, Constant, \ OrdinalHyperparameter, NumericalHyperparameter import ConfigSpace.c_util +cimport cython def impute_inactive_values(configuration: Configuration, strategy: Union[str, float] = 'default') -> Configuration: @@ -437,11 +438,10 @@ def fix_types(configuration: dict, raise TypeError("Unknown hyperparameter type %s" % type(param)) return configuration -cimport cython @cython.boundscheck(True) # Activate bounds checking @cython.wraparound(True) # Activate negative indexing def generate_grid(configuration_space: ConfigurationSpace, - num_steps_dict: Union[None, Dict[str, int]] = None, + num_steps_dict: Optional[Dict[str, int]] = None, ) -> List[Configuration]: """ Generates a grid of Configurations for a given ConfigurationSpace. Can be used, for example, for grid search. @@ -460,11 +460,11 @@ def generate_grid(configuration_space: ConfigurationSpace, List containing Configurations. It is a cartesian product of tuples of HyperParameter values. Each tuple lists the possible values taken by the corresponding HyperParameter. Within the cartesian product, in each element, the ordering of HyperParameters is the same for the OrderedDict within the ConfigurationSpace. """ - def get_value_set(num_steps_dict, hp_name): + def get_value_set(num_steps_dict: Optional[Dict[str, int]], hp_name: str): ''' Gets values along the grid for a particular hyperparameter. - Uses the num_steps_dict to determine number of grid values for UniformFloatHyperparameter and UniformIntegerHyperparameter. If these values are not present in num_steps_dict, the quantization, q, objects of these classes will be used to divide the grid. NOTE: If q is None, this results in np.arange using None for its step size parameter which results in a quantization of 1. + Uses the num_steps_dict to determine number of grid values for UniformFloatHyperparameter and UniformIntegerHyperparameter. If these values are not present in num_steps_dict, the quantization factor, q, of these classes will be used to divide the grid. NOTE: When q is used if it is None, a ValueError is raised. Parameters ---------- @@ -496,17 +496,19 @@ def generate_grid(configuration_space: ConfigurationSpace, else: lower, upper = param.lower, param.upper - if param.name in num_steps_dict: + if num_steps_dict is not None and param.name in num_steps_dict: num_steps = num_steps_dict[param.name] grid_points = np.linspace(lower, upper, num_steps) else: - grid_points = np.arange(lower, upper + param.q, param.q) # check for log and for rounding issues + if param.q is not None: + grid_points = np.arange(lower, upper + param.q, param.q) # check for log and for rounding issues + else: + raise ValueError("num_steps_dict is None or doesn't contain the number of points to divide " + param.name + " into. And its quantization factor is None. Please provide/set one of these values.") if param.log: grid_points = np.exp(grid_points) # Avoiding rounding off issues - print(grid_points) if grid_points[0] < param.lower: grid_points[0] = param.lower if grid_points[-1] > param.upper: @@ -520,11 +522,14 @@ def generate_grid(configuration_space: ConfigurationSpace, else: lower, upper = param.lower, param.upper - if param.name in num_steps_dict: + if num_steps_dict is not None and param.name in num_steps_dict: num_steps = num_steps_dict[param.name] grid_points = np.linspace(lower, upper, num_steps) else: - grid_points = np.arange(lower, upper + param.q, param.q) # check for log and for rounding issues + if param.q is not None: + grid_points = np.arange(lower, upper + param.q, param.q) # check for log and for rounding issues + else: + raise ValueError("num_steps_dict is None or doesn't contain the number of points to divide " + param.name + " into. And its quantization factor is None. Please provide/set one of these values.") if param.log: grid_points = np.exp(grid_points) @@ -541,7 +546,7 @@ def generate_grid(configuration_space: ConfigurationSpace, else: raise TypeError("Unknown hyperparameter type %s" % type(param)) - def get_cartesian_product(value_sets, hp_names): + def get_cartesian_product(value_sets: List[Tuple], hp_names: List[str]): ''' Returns a grid for a subspace of the configuration with given hyperparameters and their grid values. @@ -563,7 +568,7 @@ def generate_grid(configuration_space: ConfigurationSpace, ''' grid = [] import itertools - for i, element in enumerate(itertools.product(*value_sets)): + for element in itertools.product(*value_sets): config_dict = {} for j, hp_name in enumerate(hp_names): config_dict[hp_name] = element[j] @@ -580,10 +585,9 @@ def generate_grid(configuration_space: ConfigurationSpace, value_sets.append(get_value_set(num_steps_dict, hp_name)) hp_names.append(hp_name) - # Create a Cartesian product of above allowed values for the HPs. Hold them in an "unchecked" list because some of the conditionally dependent HPs may become active for some of the elements of the Cartesian product and in these cases creating a Configuration would throw an Error (see below). - unchecked_grid_pts = get_cartesian_product(value_sets, hp_names) # Creates a list of Configuration dicts + # Create a Cartesian product of above allowed values for the HPs. Hold them in an "unchecked" deque because some of the conditionally dependent HPs may become active for some of the elements of the Cartesian product and in these cases creating a Configuration would throw an Error (see below). + unchecked_grid_pts = deque(get_cartesian_product(value_sets, hp_names)) # Creates a deque of Configuration dicts checked_grid_pts = [] - conditional_grid_lens = [] while len(unchecked_grid_pts) > 0: try: @@ -612,9 +616,7 @@ def generate_grid(configuration_space: ConfigurationSpace, hp_names.append(hp_name) if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? #TODO new_conditonal_grid = get_cartesian_product(value_sets, hp_names) - conditional_grid_lens.append(len(new_conditonal_grid)) unchecked_grid_pts += new_conditonal_grid - del unchecked_grid_pts[0] + unchecked_grid_pts.popleft() return checked_grid_pts - diff --git a/test/test_util.py b/test/test_util.py index 52d2caed..59244f93 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -345,6 +345,8 @@ def test_fix_types(self): def test_generate_grid(self): '''Test grid generation''' + + # Sub-test 1 cs = ConfigurationSpace(seed=1234) cat1 = CategoricalHyperparameter(name='cat1', choices=['T', 'F']) @@ -361,19 +363,81 @@ def test_generate_grid(self): # Check randomly pre-selected values in the generated_grid # 2 * 1 * 11 * 6 * 3 total diff. possible configurations self.assertEqual(len(generated_grid), 396) + # Check 1st and last generated configurations completely: self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') + self.assertEqual(generated_grid[0].get_dictionary()['const1'], 4) + self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) + self.assertEqual(generated_grid[0].get_dictionary()['int1'], 10) + self.assertEqual(generated_grid[0].get_dictionary()['ord1'], '1') + self.assertEqual(generated_grid[-1].get_dictionary()['cat1'], 'F') + self.assertEqual(generated_grid[-1].get_dictionary()['const1'], 4) + self.assertEqual(generated_grid[-1].get_dictionary()['float1'], 1.0) + self.assertEqual(generated_grid[-1].get_dictionary()['int1'], 100) + self.assertEqual(generated_grid[-1].get_dictionary()['ord1'], '3') self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) # The 2 most frequently changing HPs (int1 and ord1) have 3 * 6 = 18 different values for # each value of float1, so the 4th value of float1 of -0.4 is reached after # 3 * 18 = 54 values in the generated_grid (and remains the same for the next 18 values): - self.assertAlmostEqual(generated_grid[55].get_dictionary()['float1'], -0.4, places=2) + for i in range(18): + self.assertAlmostEqual(generated_grid[54 + i].get_dictionary()['float1'], -0.4, places=2) # 5th diff. value for int1 after 4 * 3 = 12 values. Reasoning as above. self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') + + # Sub-test 2 + # Test for extreme cases: only numerical + cs = ConfigurationSpace(seed=1234) + cs.add_hyperparameters([float1, int1]) + + num_steps_dict = {'float1': 11, 'int1': 6} + generated_grid = generate_grid(cs, num_steps_dict) + + self.assertEqual(len(generated_grid), 66) + # Check 1st and last generated configurations completely: + self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) + self.assertEqual(generated_grid[0].get_dictionary()['int1'], 10) + self.assertEqual(generated_grid[-1].get_dictionary()['float1'], 1.0) + self.assertEqual(generated_grid[-1].get_dictionary()['int1'], 100) + + + # Test: only categorical + cs = ConfigurationSpace(seed=1234) + cs.add_hyperparameters([cat1]) + + generated_grid = generate_grid(cs) + + self.assertEqual(len(generated_grid), 2) + # Check 1st and last generated configurations completely: + self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') + self.assertEqual(generated_grid[-1].get_dictionary()['cat1'], 'F') + + + # Test: only constant + cs = ConfigurationSpace(seed=1234) + cs.add_hyperparameters([const1]) + + generated_grid = generate_grid(cs) + + self.assertEqual(len(generated_grid), 1) + # Check 1st and only generated configuration completely: + self.assertEqual(generated_grid[0].get_dictionary()['const1'], 4) + + + # Test: no hyperparameters yet + cs = ConfigurationSpace(seed=1234) + + generated_grid = generate_grid(cs, num_steps_dict) + + # For the case of no hyperparameters, in get_cartesian_product, itertools.product() returns a single empty tuple element which leads to a single empty Configuration. + self.assertEqual(len(generated_grid), 1) + self.assertEqual(generated_grid[0].get_dictionary(), {}) + + + # Sub-test 3 # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps # for only some of the int and float HPs. The rest are taken from the 'q' member variables # of these HPs. The conditional space tested has 2 levels of conditions. @@ -397,19 +461,44 @@ def test_generate_grid(self): cs2.add_conditions([cond_2]) float2_cond = UniformFloatHyperparameter(name='float2_cond', lower=10., upper=100., log=True) + # 2nd level dependency in ConfigurationSpace tree being tested cs2.add_hyperparameters([float2_cond]) cond_3 = GreaterThanCondition(float2_cond, int2_cond, 50) cs2.add_conditions([cond_3]) - # print(cs2) num_steps_dict1 = {'float1': 4, 'int2_cond': 3, 'float2_cond': 3} generated_grid = generate_grid(cs2, num_steps_dict1) - # print(generated_grid) self.assertEqual(len(generated_grid), 18) # RR: I manually generated the grid and verified the values were correct. + # Check 1st and only generated configuration completely: + self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) + self.assertEqual(generated_grid[0].get_dictionary()['int1'], 0) + self.assertEqual(generated_grid[-1].get_dictionary()['float1'], -1.0) + self.assertEqual(generated_grid[-1].get_dictionary()['int1'], 1000) + self.assertEqual(generated_grid[-1].get_dictionary()['float2_cond'], 100.0) + self.assertEqual(generated_grid[-1].get_dictionary()['int2_cond'], 100) # Here, we test that a few randomly chosen values in the generated grid # correspond to the ones I checked. self.assertEqual(generated_grid[3].get_dictionary()['int1'], 1000) self.assertEqual(generated_grid[12].get_dictionary()['cat1_cond'], 'orange') self.assertAlmostEqual(generated_grid[-2].get_dictionary()['float2_cond'], 31.622776601683803, places=3) + + + # Sub-test 4 + # Test: only a single hyperparameter and num_steps_dict is None + cs = ConfigurationSpace(seed=1234) + cs.add_hyperparameters([float1]) + + num_steps_dict = {'float1': 11} + try: + generated_grid = generate_grid(cs) + except ValueError as e: + assert str(e) == "num_steps_dict is None or doesn't contain the number of points to divide float1 into. And its quantization factor is None. Please provide/set one of these values." + + generated_grid = generate_grid(cs, num_steps_dict) + + self.assertEqual(len(generated_grid), 11) + # Check 1st and last generated configurations completely: + self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) + self.assertEqual(generated_grid[-1].get_dictionary()['float1'], 1.0) From 79fdd84852027979c4869d6b968cd963c5ccc2e1 Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Wed, 24 Mar 2021 19:58:35 +0100 Subject: [PATCH 14/15] flake8 corrections --- test/test_util.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 59244f93..0d952638 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -380,14 +380,13 @@ def test_generate_grid(self): # each value of float1, so the 4th value of float1 of -0.4 is reached after # 3 * 18 = 54 values in the generated_grid (and remains the same for the next 18 values): for i in range(18): - self.assertAlmostEqual(generated_grid[54 + i].get_dictionary()['float1'], -0.4, places=2) + self.assertAlmostEqual(generated_grid[54+i].get_dictionary()['float1'], -0.4, places=2) # 5th diff. value for int1 after 4 * 3 = 12 values. Reasoning as above. self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') - # Sub-test 2 # Test for extreme cases: only numerical cs = ConfigurationSpace(seed=1234) @@ -403,7 +402,6 @@ def test_generate_grid(self): self.assertEqual(generated_grid[-1].get_dictionary()['float1'], 1.0) self.assertEqual(generated_grid[-1].get_dictionary()['int1'], 100) - # Test: only categorical cs = ConfigurationSpace(seed=1234) cs.add_hyperparameters([cat1]) @@ -415,7 +413,6 @@ def test_generate_grid(self): self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') self.assertEqual(generated_grid[-1].get_dictionary()['cat1'], 'F') - # Test: only constant cs = ConfigurationSpace(seed=1234) cs.add_hyperparameters([const1]) @@ -426,17 +423,16 @@ def test_generate_grid(self): # Check 1st and only generated configuration completely: self.assertEqual(generated_grid[0].get_dictionary()['const1'], 4) - # Test: no hyperparameters yet cs = ConfigurationSpace(seed=1234) generated_grid = generate_grid(cs, num_steps_dict) - # For the case of no hyperparameters, in get_cartesian_product, itertools.product() returns a single empty tuple element which leads to a single empty Configuration. + # For the case of no hyperparameters, in get_cartesian_product, itertools.product() returns + # a single empty tuple element which leads to a single empty Configuration. self.assertEqual(len(generated_grid), 1) self.assertEqual(generated_grid[0].get_dictionary(), {}) - # Sub-test 3 # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps # for only some of the int and float HPs. The rest are taken from the 'q' member variables @@ -484,7 +480,6 @@ def test_generate_grid(self): self.assertAlmostEqual(generated_grid[-2].get_dictionary()['float2_cond'], 31.622776601683803, places=3) - # Sub-test 4 # Test: only a single hyperparameter and num_steps_dict is None cs = ConfigurationSpace(seed=1234) @@ -494,7 +489,9 @@ def test_generate_grid(self): try: generated_grid = generate_grid(cs) except ValueError as e: - assert str(e) == "num_steps_dict is None or doesn't contain the number of points to divide float1 into. And its quantization factor is None. Please provide/set one of these values." + assert str(e) == "num_steps_dict is None or doesn't contain " \ + "the number of points to divide float1 into. And its quantization " \ + "factor is None. Please provide/set one of these values." generated_grid = generate_grid(cs, num_steps_dict) From 6aefd63443bf5b7945fb6cbc3d61dc7cb66713d1 Mon Sep 17 00:00:00 2001 From: Raghu Rajan Date: Tue, 6 Apr 2021 18:41:46 +0200 Subject: [PATCH 15/15] Small changes --- ConfigSpace/util.pyx | 18 ++++++++++++------ test/test_util.py | 37 ++++++++++++++----------------------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/ConfigSpace/util.pyx b/ConfigSpace/util.pyx index 7d26370d..72bcef70 100644 --- a/ConfigSpace/util.pyx +++ b/ConfigSpace/util.pyx @@ -568,11 +568,14 @@ def generate_grid(configuration_space: ConfigurationSpace, ''' grid = [] import itertools - for element in itertools.product(*value_sets): - config_dict = {} - for j, hp_name in enumerate(hp_names): - config_dict[hp_name] = element[j] - grid.append(config_dict) + if len(value_sets) == 0: + pass # Edge case + else: + for element in itertools.product(*value_sets): + config_dict = {} + for j, hp_name in enumerate(hp_names): + config_dict[hp_name] = element[j] + grid.append(config_dict) return grid @@ -594,6 +597,7 @@ def generate_grid(configuration_space: ConfigurationSpace, grid_point = Configuration(configuration_space, unchecked_grid_pts[0]) checked_grid_pts.append(grid_point) except ValueError as e: + assert str(e)[:23] == "Active hyperparameter '" and str(e)[-16:] == "' not specified!", "Caught exception contains unexpected message." value_sets = [] hp_names = [] new_active_hp_names = [] @@ -614,9 +618,11 @@ def generate_grid(configuration_space: ConfigurationSpace, for hp_name in new_active_hp_names: value_sets.append(get_value_set(num_steps_dict, hp_name)) hp_names.append(hp_name) - if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? #TODO + if len(new_active_hp_names) > 0: # this check might not be needed, as there is always going to be a new active HP when in this except block? new_conditonal_grid = get_cartesian_product(value_sets, hp_names) unchecked_grid_pts += new_conditonal_grid + else: + raise RuntimeError("Unexpected error: There should have been a newly activated hyperparameter for the current configuration values: " + str(unchecked_grid_pts[0]) + ". Please contact the developers with the code you ran and the stack trace.") unchecked_grid_pts.popleft() return checked_grid_pts diff --git a/test/test_util.py b/test/test_util.py index 0d952638..9b2c4aad 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -364,16 +364,10 @@ def test_generate_grid(self): # 2 * 1 * 11 * 6 * 3 total diff. possible configurations self.assertEqual(len(generated_grid), 396) # Check 1st and last generated configurations completely: - self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') - self.assertEqual(generated_grid[0].get_dictionary()['const1'], 4) - self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) - self.assertEqual(generated_grid[0].get_dictionary()['int1'], 10) - self.assertEqual(generated_grid[0].get_dictionary()['ord1'], '1') - self.assertEqual(generated_grid[-1].get_dictionary()['cat1'], 'F') - self.assertEqual(generated_grid[-1].get_dictionary()['const1'], 4) - self.assertEqual(generated_grid[-1].get_dictionary()['float1'], 1.0) - self.assertEqual(generated_grid[-1].get_dictionary()['int1'], 100) - self.assertEqual(generated_grid[-1].get_dictionary()['ord1'], '3') + first_expected_dict = {'cat1': 'T', 'const1': 4, 'float1': -1.0, 'int1': 10, 'ord1': '1'} + last_expected_dict = {'cat1': 'F', 'const1': 4, 'float1': 1.0, 'int1': 100, 'ord1': '3'} + self.assertEqual(generated_grid[0].get_dictionary(), first_expected_dict) + self.assertEqual(generated_grid[-1].get_dictionary(), last_expected_dict) self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) # The 2 most frequently changing HPs (int1 and ord1) have 3 * 6 = 18 different values for @@ -397,10 +391,10 @@ def test_generate_grid(self): self.assertEqual(len(generated_grid), 66) # Check 1st and last generated configurations completely: - self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) - self.assertEqual(generated_grid[0].get_dictionary()['int1'], 10) - self.assertEqual(generated_grid[-1].get_dictionary()['float1'], 1.0) - self.assertEqual(generated_grid[-1].get_dictionary()['int1'], 100) + first_expected_dict = {'float1': -1.0, 'int1': 10} + last_expected_dict = {'float1': 1.0, 'int1': 100} + self.assertEqual(generated_grid[0].get_dictionary(), first_expected_dict) + self.assertEqual(generated_grid[-1].get_dictionary(), last_expected_dict) # Test: only categorical cs = ConfigurationSpace(seed=1234) @@ -430,8 +424,7 @@ def test_generate_grid(self): # For the case of no hyperparameters, in get_cartesian_product, itertools.product() returns # a single empty tuple element which leads to a single empty Configuration. - self.assertEqual(len(generated_grid), 1) - self.assertEqual(generated_grid[0].get_dictionary(), {}) + self.assertEqual(len(generated_grid), 0) # Sub-test 3 # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps @@ -466,13 +459,11 @@ def test_generate_grid(self): self.assertEqual(len(generated_grid), 18) # RR: I manually generated the grid and verified the values were correct. - # Check 1st and only generated configuration completely: - self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) - self.assertEqual(generated_grid[0].get_dictionary()['int1'], 0) - self.assertEqual(generated_grid[-1].get_dictionary()['float1'], -1.0) - self.assertEqual(generated_grid[-1].get_dictionary()['int1'], 1000) - self.assertEqual(generated_grid[-1].get_dictionary()['float2_cond'], 100.0) - self.assertEqual(generated_grid[-1].get_dictionary()['int2_cond'], 100) + # Check 1st and last generated configurations completely: + first_expected_dict = {'float1': -1.0, 'int1': 0} + last_expected_dict = {'float1': -1.0, 'int1': 1000, 'int2_cond': 100, 'float2_cond': 100.0} + self.assertEqual(generated_grid[0].get_dictionary(), first_expected_dict) + self.assertEqual(generated_grid[-1].get_dictionary(), last_expected_dict) # Here, we test that a few randomly chosen values in the generated grid # correspond to the ones I checked. self.assertEqual(generated_grid[3].get_dictionary()['int1'], 1000)