From a7575ded83c6bea740b77d09adac70c9745a8370 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 17:15:11 +0200 Subject: [PATCH 01/38] Add fit_by_exhaustive_search to NNClassifier --- src/safeds/ml/nn/_model.py | 101 ++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 3d9086d61..911644126 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +from concurrent.futures import ProcessPoolExecutor, wait, ALL_COMPLETED from typing import TYPE_CHECKING, Generic, Self, TypeVar from safeds._config import _init_default_device @@ -13,8 +14,9 @@ from safeds.exceptions import ( FeatureDataMismatchError, InvalidModelStructureError, - ModelNotFittedError, + ModelNotFittedError, LearningError, ) +from safeds.ml.metrics import ClassificationMetrics from safeds.ml.nn.converters import ( InputConversionImageToColumn, InputConversionImageToImage, @@ -562,6 +564,103 @@ def fit( copied_model._model.eval() return copied_model + def fit_by_exhaustive_search( + self, + train_data: IFT, + optimization_metric: ClassifierMetric, + positive_class = None, + epoch_size: int = 25, + batch_size: int = 1, + learning_rate: float = 0.001, + ) -> Self: + if not self._contains_choices(): + raise FittingWithoutChoiceError + + list_of_models = self._get_models_for_all_choices() + list_of_fitted_models = [] + + if isinstance(IFT, TimeSeriesDataset): + raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + if isinstance(IFT, ImageDataset): + raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + + with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: + futures = [] + for model in list_of_models: + futures.append( + executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) + [done, _] = wait(futures, return_when=ALL_COMPLETED) + for future in done: + list_of_fitted_models.append(future.result()) + executor.shutdown() + + + target_col = train_data.target + test_data = train_data.to_table().remove_columns([target_col.name]) + + best_model = None + best_metric_value = None + for fitted_model in list_of_fitted_models: + if best_model is None: + best_model = fitted_model + match optimization_metric.value: + case "accuracy": + best_metric_value = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) + case "precision": + best_metric_value = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + case "recall": + best_metric_value = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + case "f1score": + best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + else: + match optimization_metric.value: + case "accuracy": + error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) + if error_of_fitted_model > best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + case "precision": + error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + if error_of_fitted_model > best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + case "recall": + error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) + if error_of_fitted_model > best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + case "f1score": + error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) + if error_of_fitted_model > best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + best_model._is_fitted = True + return best_model + + def _get_models_for_all_choices(self) -> list[Self]: + + all_possible_layer_combinations: list[list] = [[]] + for layer in self._layers: + if not layer._contains_choices(): + for item in all_possible_layer_combinations: + item.append(layer) + else: + updated_combinations = [] + versions_of_one_layer = layer._get_layers_for_all_choices() + for version in versions_of_one_layer: + copy_of_all_current_possible_combinations = copy.deepcopy(all_possible_layer_combinations) + for combination in copy_of_all_current_possible_combinations: + combination.append(version) + updated_combinations.append(combination) + all_possible_layer_combinations = updated_combinations + + models = [] + for combination in all_possible_layer_combinations: + copied_model = copy.deepcopy(self) + copied_model._layers = combination + models.append(copied_model) + return models + def predict(self, test_data: IPT) -> IFT: """ Make a prediction for the given test data. From 2ac1169eaa635a93297a60070e284f575174fe52 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 17:16:43 +0200 Subject: [PATCH 02/38] Add fit_by_exhaustive_search to NNRegressor --- src/safeds/ml/nn/_model.py | 99 +++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 911644126..df6061c62 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -16,7 +16,7 @@ InvalidModelStructureError, ModelNotFittedError, LearningError, ) -from safeds.ml.metrics import ClassificationMetrics +from safeds.ml.metrics import ClassificationMetrics, RegressionMetrics from safeds.ml.nn.converters import ( InputConversionImageToColumn, InputConversionImageToImage, @@ -260,6 +260,103 @@ def fit( copied_model._model.eval() return copied_model + def fit_by_exhaustive_search( + self, + train_data: IFT, + optimization_metric: RegressorMetric, + epoch_size: int = 25, + batch_size: int = 1, + learning_rate: float = 0.001, + ) -> Self: + if not self._contains_choices(): + raise FittingWithoutChoiceError + + list_of_models = self._get_models_for_all_choices() + list_of_fitted_models = [] + + if isinstance(IFT, TimeSeriesDataset): + raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + if isinstance(IFT, ImageDataset): + raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + + with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: + futures = [] + for model in list_of_models: + futures.append( + executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) + [done, _] = wait(futures, return_when=ALL_COMPLETED) + for future in done: + list_of_fitted_models.append(future.result()) + executor.shutdown() + + + target_col = train_data.target + test_data = train_data.to_table().remove_columns([target_col.name]) + + best_model = None + best_metric_value = None + for fitted_model in list_of_fitted_models: + if best_model is None: + best_model = fitted_model + match optimization_metric.value: + case "mean_squared_error": + best_metric_value = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) + case "mean_absolute_error": + best_metric_value = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) + case "median_absolute_deviation": + best_metric_value = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) + case "coefficient_of_determination": + best_metric_value = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) + else: + match optimization_metric.value: + case "mean_squared_error": + error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) + if error_of_fitted_model < best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + case "mean_absolute_error": + error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) + if error_of_fitted_model < best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + case "median_absolute_deviation": + error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) + if error_of_fitted_model < best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + case "coefficient_of_determination": + error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) + if error_of_fitted_model > best_metric_value: + best_model = fitted_model + best_metric_value = error_of_fitted_model + best_model._is_fitted = True + return best_model + + def _get_models_for_all_choices(self) -> list[Self]: + + all_possible_layer_combinations: list[list] = [[]] + for layer in self._layers: + if not layer._contains_choices(): + for item in all_possible_layer_combinations: + item.append(layer) + else: + updated_combinations = [] + versions_of_one_layer = layer._get_layers_for_all_choices() + for version in versions_of_one_layer: + copy_of_all_current_possible_combinations = copy.deepcopy(all_possible_layer_combinations) + for combination in copy_of_all_current_possible_combinations: + combination.append(version) + updated_combinations.append(combination) + all_possible_layer_combinations = updated_combinations + + models = [] + for combination in all_possible_layer_combinations: + copied_model = copy.deepcopy(self) + copied_model._layers = combination + models.append(copied_model) + return models + + def predict(self, test_data: IPT) -> IFT: """ Make a prediction for the given test data. From 79a887b892311a45b429a770d07605e1d506ba11 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 17:21:13 +0200 Subject: [PATCH 03/38] Add Classifier and Regressor Metric --- src/safeds/ml/metrics/__init__.py | 7 +++++++ src/safeds/ml/metrics/_classifier_metric.py | 10 ++++++++++ src/safeds/ml/metrics/_regressor_metric.py | 10 ++++++++++ 3 files changed, 27 insertions(+) create mode 100644 src/safeds/ml/metrics/_classifier_metric.py create mode 100644 src/safeds/ml/metrics/_regressor_metric.py diff --git a/src/safeds/ml/metrics/__init__.py b/src/safeds/ml/metrics/__init__.py index aa465cff0..50928710f 100644 --- a/src/safeds/ml/metrics/__init__.py +++ b/src/safeds/ml/metrics/__init__.py @@ -4,6 +4,9 @@ import apipkg +from ._classifier_metric import ClassifierMetric +from ._regressor_metric import RegressorMetric + if TYPE_CHECKING: from ._classification_metrics import ClassificationMetrics from ._regression_metrics import RegressionMetrics @@ -12,11 +15,15 @@ __name__, { "ClassificationMetrics": "._classification_metrics:ClassificationMetrics", + "ClassifierMetric": "._classifier_metrics:ClassifierMetric", "RegressionMetrics": "._regression_metrics:RegressionMetrics", + "RegressorMetric": "._regressor_metric:RegressorMetric", }, ) __all__ = [ "ClassificationMetrics", + "ClassifierMetric", "RegressionMetrics", + "RegressorMetric", ] diff --git a/src/safeds/ml/metrics/_classifier_metric.py b/src/safeds/ml/metrics/_classifier_metric.py new file mode 100644 index 000000000..4f69c2607 --- /dev/null +++ b/src/safeds/ml/metrics/_classifier_metric.py @@ -0,0 +1,10 @@ +from enum import Enum + + +class ClassifierMetric(Enum): + """An Enum of possible Metrics for a Classifier.""" + + ACCURACY = "accuracy" + PRECISION = "precision" + RECALL = "recall" + F1_SCORE = "f1_score" diff --git a/src/safeds/ml/metrics/_regressor_metric.py b/src/safeds/ml/metrics/_regressor_metric.py new file mode 100644 index 000000000..421ce4b08 --- /dev/null +++ b/src/safeds/ml/metrics/_regressor_metric.py @@ -0,0 +1,10 @@ +from enum import Enum + + +class RegressorMetric(Enum): + """An Enum of possible Metrics for a Regressor.""" + + MEAN_SQUARED_ERROR = "mean_squared_error" + MEAN_ABSOLUTE_ERROR = "mean_absolute_error" + MEDIAN_ABSOLUTE_DEVIATION = "median_absolute_deviation" + COEFFICIENT_OF_DETERMINATION = "coefficient_of_determination" From a5c3713b9080d8cbd00d219bdbf1de13f1e15baa Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 17:27:16 +0200 Subject: [PATCH 04/38] add FittingWithChoiceError and FittingWithoutChoiceError --- src/safeds/exceptions/__init__.py | 4 ++++ src/safeds/exceptions/_ml.py | 18 ++++++++++++++++++ src/safeds/ml/nn/_model.py | 24 ++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/safeds/exceptions/__init__.py b/src/safeds/exceptions/__init__.py index dabbc3afa..105edbab7 100644 --- a/src/safeds/exceptions/__init__.py +++ b/src/safeds/exceptions/__init__.py @@ -17,6 +17,8 @@ DatasetMissesDataError, DatasetMissesFeaturesError, FeatureDataMismatchError, + FittingWithChoiceError, + FittingWithoutChoiceError, InputSizeError, InvalidFitDataError, InvalidModelStructureError, @@ -72,6 +74,8 @@ class OutOfBoundsError(SafeDsError): "DatasetMissesFeaturesError", "TargetDataMismatchError", "FeatureDataMismatchError", + "FittingWithChoiceError", + "FittingWithoutChoiceError", "InvalidFitDataError", "InputSizeError", "InvalidModelStructureError", diff --git a/src/safeds/exceptions/_ml.py b/src/safeds/exceptions/_ml.py index b7600df34..a468ed190 100644 --- a/src/safeds/exceptions/_ml.py +++ b/src/safeds/exceptions/_ml.py @@ -41,6 +41,24 @@ class DatasetMissesDataError(ValueError): def __init__(self) -> None: super().__init__("Dataset contains no rows") +class FittingWithChoiceError(Exception): + """Raised when a model is fitted with a choice object as a parameter.""" + + def __init__(self) -> None: + super().__init__( + "Error occurred while fitting: Trying to fit with a Choice Parameter. Please use " + "fit_by_exhaustive_search() instead.", + ) + + +class FittingWithoutChoiceError(Exception): + """Raised when a model is fitted by exhaustive search without a choice object as a parameter.""" + + def __init__(self) -> None: + super().__init__( + "Error occurred while fitting: Trying to fit by exhaustive search without a Choice " + "Parameter. Please use fit() instead.", + ) class InvalidFitDataError(Exception): """Raised when a Neural Network is fitted on invalid data.""" diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index df6061c62..c84c4ae4b 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -14,9 +14,9 @@ from safeds.exceptions import ( FeatureDataMismatchError, InvalidModelStructureError, - ModelNotFittedError, LearningError, + ModelNotFittedError, LearningError, FittingWithoutChoiceError, FittingWithChoiceError, ) -from safeds.ml.metrics import ClassificationMetrics, RegressionMetrics +from safeds.ml.metrics import ClassificationMetrics, RegressionMetrics, RegressorMetric, ClassifierMetric from safeds.ml.nn.converters import ( InputConversionImageToColumn, InputConversionImageToImage, @@ -212,6 +212,9 @@ def fit( _init_default_device() + if self._contains_choices(): + raise FittingWithChoiceError + if not self._input_conversion._is_fit_data_valid(train_data): raise FeatureDataMismatchError @@ -412,6 +415,13 @@ def input_size(self) -> int | ModelImageSize | None: # TODO: raise if not fitted, don't return None return self._input_size + def _contains_choices(self) -> bool: + """Whether the model contains choices in any layer.""" + for layer in self._layers: + if layer._contains_choices(): + return True + return False + class NeuralNetworkClassifier(Generic[IFT, IPT]): """ @@ -604,6 +614,9 @@ def fit( _init_default_device() + if self._contains_choices(): + raise FittingWithChoiceError + if not self._input_conversion._is_fit_data_valid(train_data): raise FeatureDataMismatchError @@ -815,3 +828,10 @@ def input_size(self) -> int | ModelImageSize | None: """The input size of the model.""" # TODO: raise if not fitted, don't return None return self._input_size + + def _contains_choices(self) -> bool: + """Whether the model contains choices in any layer.""" + for layer in self._layers: + if layer._contains_choices(): + return True + return False From a3b9593be3ae8964780dce3b625885599adcaacb Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 19:01:25 +0200 Subject: [PATCH 05/38] Add choices to layers --- .../ml/nn/layers/_convolutional2d_layer.py | 6 +++++ src/safeds/ml/nn/layers/_dropout_layer.py | 6 +++++ src/safeds/ml/nn/layers/_flatten_layer.py | 6 +++++ src/safeds/ml/nn/layers/_forward_layer.py | 22 ++++++++++++++++--- src/safeds/ml/nn/layers/_gru_layer.py | 22 ++++++++++++++++--- src/safeds/ml/nn/layers/_layer.py | 8 +++++++ src/safeds/ml/nn/layers/_lstm_layer.py | 22 ++++++++++++++++--- src/safeds/ml/nn/layers/_pooling2d_layer.py | 6 +++++ 8 files changed, 89 insertions(+), 9 deletions(-) diff --git a/src/safeds/ml/nn/layers/_convolutional2d_layer.py b/src/safeds/ml/nn/layers/_convolutional2d_layer.py index dd42f2d97..2adb7baf3 100644 --- a/src/safeds/ml/nn/layers/_convolutional2d_layer.py +++ b/src/safeds/ml/nn/layers/_convolutional2d_layer.py @@ -125,6 +125,12 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size self._output_size = None + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[Convolutional2DLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash( self._output_channel, diff --git a/src/safeds/ml/nn/layers/_dropout_layer.py b/src/safeds/ml/nn/layers/_dropout_layer.py index 1814e4383..cd15ad219 100644 --- a/src/safeds/ml/nn/layers/_dropout_layer.py +++ b/src/safeds/ml/nn/layers/_dropout_layer.py @@ -87,6 +87,12 @@ def output_size(self) -> int | ModelImageSize: def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[DropoutLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash(self._input_size, self._probability) diff --git a/src/safeds/ml/nn/layers/_flatten_layer.py b/src/safeds/ml/nn/layers/_flatten_layer.py index a84551c2b..0697f3e65 100644 --- a/src/safeds/ml/nn/layers/_flatten_layer.py +++ b/src/safeds/ml/nn/layers/_flatten_layer.py @@ -76,6 +76,12 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size self._output_size = None + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[FlattenLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash(self._input_size, self._output_size) diff --git a/src/safeds/ml/nn/layers/_forward_layer.py b/src/safeds/ml/nn/layers/_forward_layer.py index e420b78ec..7e38d74df 100644 --- a/src/safeds/ml/nn/layers/_forward_layer.py +++ b/src/safeds/ml/nn/layers/_forward_layer.py @@ -7,6 +7,7 @@ from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer +from ...hyperparameters import Choice if TYPE_CHECKING: from torch import nn @@ -28,13 +29,18 @@ class ForwardLayer(Layer): If output_size < 1 """ - def __init__(self, neuron_count: int): - _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) + def __init__(self, neuron_count: int | Choice[int]) -> None: + if isinstance(neuron_count, Choice): + for val in neuron_count: + _check_bounds("neuron_count", val, lower_bound=_ClosedBound(1)) + else: + _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) self._input_size: int | None = None self._output_size = neuron_count def _get_internal_layer(self, **kwargs: Any) -> nn.Module: + assert not self._contains_choices() from ._internal_layers import _InternalForwardLayer # Slow import on global level if "activation_function" not in kwargs: @@ -65,7 +71,7 @@ def input_size(self) -> int: return self._input_size @property - def output_size(self) -> int: + def output_size(self) -> int | Choice[int]: """ Get the output_size of this layer. @@ -82,6 +88,16 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size + def _contains_choices(self) -> bool: + return isinstance(self._output_size, Choice) + + def _get_layers_for_all_choices(self) -> list[ForwardLayer]: + assert self._contains_choices() + layers = [] + for val in self._output_size: + layers.append(ForwardLayer(neuron_count=val)) + return layers + def __hash__(self) -> int: return _structural_hash(self._input_size, self._output_size) diff --git a/src/safeds/ml/nn/layers/_gru_layer.py b/src/safeds/ml/nn/layers/_gru_layer.py index e74fec417..da5f6d3b9 100644 --- a/src/safeds/ml/nn/layers/_gru_layer.py +++ b/src/safeds/ml/nn/layers/_gru_layer.py @@ -8,6 +8,7 @@ from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer +from ...hyperparameters import Choice if TYPE_CHECKING: from torch import nn @@ -29,13 +30,18 @@ class GRULayer(Layer): If output_size < 1 """ - def __init__(self, neuron_count: int): - _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) + def __init__(self, neuron_count: int | Choice[int]): + if isinstance(neuron_count, Choice): + for val in neuron_count: + _check_bounds("neuron_count", val, lower_bound=_ClosedBound(1)) + else: + _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) self._input_size: int | None = None self._output_size = neuron_count def _get_internal_layer(self, **kwargs: Any) -> nn.Module: + assert not self._contains_choices() from ._internal_layers import _InternalGRULayer # Slow import on global level if "activation_function" not in kwargs: @@ -65,7 +71,7 @@ def input_size(self) -> int: return self._input_size @property - def output_size(self) -> int: + def output_size(self) -> int | Choice[int]: """ Get the output_size of this layer. @@ -82,6 +88,16 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size + def _contains_choices(self) -> bool: + return isinstance(self._output_size, Choice) + + def _get_layers_for_all_choices(self) -> list[GRULayer]: + assert self._contains_choices() + layers = [] + for val in self._output_size: + layers.append(GRULayer(neuron_count=val)) + return layers + def __hash__(self) -> int: return _structural_hash( self._input_size, diff --git a/src/safeds/ml/nn/layers/_layer.py b/src/safeds/ml/nn/layers/_layer.py index 058036688..2dd94c18f 100644 --- a/src/safeds/ml/nn/layers/_layer.py +++ b/src/safeds/ml/nn/layers/_layer.py @@ -32,6 +32,14 @@ def output_size(self) -> int | ModelImageSize: def _set_input_size(self, input_size: int | ModelImageSize) -> None: pass # pragma: no cover + @abstractmethod + def _contains_choices(self) -> bool: + pass # pragma: no cover + + @abstractmethod + def _get_layers_for_all_choices(self) -> list[Layer]: + pass # pragma: no cover + @abstractmethod def __hash__(self) -> int: pass # pragma: no cover diff --git a/src/safeds/ml/nn/layers/_lstm_layer.py b/src/safeds/ml/nn/layers/_lstm_layer.py index 330809474..6de577b49 100644 --- a/src/safeds/ml/nn/layers/_lstm_layer.py +++ b/src/safeds/ml/nn/layers/_lstm_layer.py @@ -8,6 +8,7 @@ from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer +from ...hyperparameters import Choice if TYPE_CHECKING: from torch import nn @@ -29,13 +30,18 @@ class LSTMLayer(Layer): If output_size < 1 """ - def __init__(self, neuron_count: int): - _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) + def __init__(self, neuron_count: int | Choice[int]): + if isinstance(neuron_count, Choice): + for val in neuron_count: + _check_bounds("neuron_count", val, lower_bound=_ClosedBound(1)) + else: + _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) self._input_size: int | None = None self._output_size = neuron_count def _get_internal_layer(self, **kwargs: Any) -> nn.Module: + assert not self._contains_choices() from ._internal_layers import _InternalLSTMLayer # Slow import on global level if "activation_function" not in kwargs: @@ -65,7 +71,7 @@ def input_size(self) -> int: return self._input_size @property - def output_size(self) -> int: + def output_size(self) -> int | Choice[int]: """ Get the output_size of this layer. @@ -82,6 +88,16 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size + def _contains_choices(self) -> bool: + return isinstance(self._output_size, Choice) + + def _get_layers_for_all_choices(self) -> list[LSTMLayer]: + assert self._contains_choices() + layers = [] + for val in self._output_size: + layers.append(LSTMLayer(neuron_count=val)) + return layers + def __hash__(self) -> int: return _structural_hash( self._input_size, diff --git a/src/safeds/ml/nn/layers/_pooling2d_layer.py b/src/safeds/ml/nn/layers/_pooling2d_layer.py index d658ed848..01bc9f0da 100644 --- a/src/safeds/ml/nn/layers/_pooling2d_layer.py +++ b/src/safeds/ml/nn/layers/_pooling2d_layer.py @@ -102,6 +102,12 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size self._output_size = None + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[_Pooling2DLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash( self._strategy, From 41dd053e475350df21d5d7f13500612925b8072a Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 20:42:47 +0200 Subject: [PATCH 06/38] linter fixes --- src/safeds/ml/metrics/__init__.py | 2 +- src/safeds/ml/nn/_model.py | 11 +++-------- src/safeds/ml/nn/layers/_forward_layer.py | 2 +- src/safeds/ml/nn/layers/_gru_layer.py | 2 +- src/safeds/ml/nn/layers/_lstm_layer.py | 2 +- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/safeds/ml/metrics/__init__.py b/src/safeds/ml/metrics/__init__.py index 50928710f..dc0a93657 100644 --- a/src/safeds/ml/metrics/__init__.py +++ b/src/safeds/ml/metrics/__init__.py @@ -15,7 +15,7 @@ __name__, { "ClassificationMetrics": "._classification_metrics:ClassificationMetrics", - "ClassifierMetric": "._classifier_metrics:ClassifierMetric", + "ClassifierMetric": "._classifier_metric:ClassifierMetric", "RegressionMetrics": "._regression_metrics:RegressionMetrics", "RegressorMetric": "._regressor_metric:RegressorMetric", }, diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index c84c4ae4b..3589c9ad7 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -417,10 +417,7 @@ def input_size(self) -> int | ModelImageSize | None: def _contains_choices(self) -> bool: """Whether the model contains choices in any layer.""" - for layer in self._layers: - if layer._contains_choices(): - return True - return False + return any(layer._contains_choices() for layer in self._layers) class NeuralNetworkClassifier(Generic[IFT, IPT]): @@ -831,7 +828,5 @@ def input_size(self) -> int | ModelImageSize | None: def _contains_choices(self) -> bool: """Whether the model contains choices in any layer.""" - for layer in self._layers: - if layer._contains_choices(): - return True - return False + return any(layer._contains_choices() for layer in self._layers) + diff --git a/src/safeds/ml/nn/layers/_forward_layer.py b/src/safeds/ml/nn/layers/_forward_layer.py index 7e38d74df..bc4a312fb 100644 --- a/src/safeds/ml/nn/layers/_forward_layer.py +++ b/src/safeds/ml/nn/layers/_forward_layer.py @@ -7,7 +7,7 @@ from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer -from ...hyperparameters import Choice +from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn diff --git a/src/safeds/ml/nn/layers/_gru_layer.py b/src/safeds/ml/nn/layers/_gru_layer.py index da5f6d3b9..1aefc7291 100644 --- a/src/safeds/ml/nn/layers/_gru_layer.py +++ b/src/safeds/ml/nn/layers/_gru_layer.py @@ -8,7 +8,7 @@ from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer -from ...hyperparameters import Choice +from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn diff --git a/src/safeds/ml/nn/layers/_lstm_layer.py b/src/safeds/ml/nn/layers/_lstm_layer.py index 6de577b49..3872d524f 100644 --- a/src/safeds/ml/nn/layers/_lstm_layer.py +++ b/src/safeds/ml/nn/layers/_lstm_layer.py @@ -8,7 +8,7 @@ from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer -from ...hyperparameters import Choice +from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn From b542fd6bc1a04dfd2f8bbee2f40b461068afc9ce Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 21:11:49 +0200 Subject: [PATCH 07/38] linter fixes --- src/safeds/ml/nn/_model.py | 11 ++++++----- src/safeds/ml/nn/layers/_forward_layer.py | 2 ++ src/safeds/ml/nn/layers/_gru_layer.py | 2 ++ src/safeds/ml/nn/layers/_layer.py | 8 +++++--- src/safeds/ml/nn/layers/_lstm_layer.py | 2 ++ 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 3589c9ad7..717598e78 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -277,11 +277,10 @@ def fit_by_exhaustive_search( list_of_models = self._get_models_for_all_choices() list_of_fitted_models = [] - if isinstance(IFT, TimeSeriesDataset): + if isinstance(train_data, TimeSeriesDataset): raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover - if isinstance(IFT, ImageDataset): + if isinstance(train_data, ImageDataset): raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover - with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: futures = [] for model in list_of_models: @@ -332,6 +331,7 @@ def fit_by_exhaustive_search( if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model + assert best_model is not None # just for linter best_model._is_fitted = True return best_model @@ -686,9 +686,9 @@ def fit_by_exhaustive_search( list_of_models = self._get_models_for_all_choices() list_of_fitted_models = [] - if isinstance(IFT, TimeSeriesDataset): + if isinstance(train_data, TimeSeriesDataset): raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover - if isinstance(IFT, ImageDataset): + if isinstance(train_data, ImageDataset): raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: @@ -741,6 +741,7 @@ def fit_by_exhaustive_search( if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model + assert best_model is not None # just for linter best_model._is_fitted = True return best_model diff --git a/src/safeds/ml/nn/layers/_forward_layer.py b/src/safeds/ml/nn/layers/_forward_layer.py index bc4a312fb..64cddd930 100644 --- a/src/safeds/ml/nn/layers/_forward_layer.py +++ b/src/safeds/ml/nn/layers/_forward_layer.py @@ -41,6 +41,7 @@ def __init__(self, neuron_count: int | Choice[int]) -> None: def _get_internal_layer(self, **kwargs: Any) -> nn.Module: assert not self._contains_choices() + assert not isinstance(self._output_size, Choice) from ._internal_layers import _InternalForwardLayer # Slow import on global level if "activation_function" not in kwargs: @@ -93,6 +94,7 @@ def _contains_choices(self) -> bool: def _get_layers_for_all_choices(self) -> list[ForwardLayer]: assert self._contains_choices() + assert isinstance(self._output_size, Choice) # just for linter layers = [] for val in self._output_size: layers.append(ForwardLayer(neuron_count=val)) diff --git a/src/safeds/ml/nn/layers/_gru_layer.py b/src/safeds/ml/nn/layers/_gru_layer.py index 1aefc7291..24d7c3993 100644 --- a/src/safeds/ml/nn/layers/_gru_layer.py +++ b/src/safeds/ml/nn/layers/_gru_layer.py @@ -42,6 +42,7 @@ def __init__(self, neuron_count: int | Choice[int]): def _get_internal_layer(self, **kwargs: Any) -> nn.Module: assert not self._contains_choices() + assert not isinstance(self._output_size, Choice) # just for linter from ._internal_layers import _InternalGRULayer # Slow import on global level if "activation_function" not in kwargs: @@ -93,6 +94,7 @@ def _contains_choices(self) -> bool: def _get_layers_for_all_choices(self) -> list[GRULayer]: assert self._contains_choices() + assert isinstance(self._output_size, Choice) # just for linter layers = [] for val in self._output_size: layers.append(GRULayer(neuron_count=val)) diff --git a/src/safeds/ml/nn/layers/_layer.py b/src/safeds/ml/nn/layers/_layer.py index 2dd94c18f..9fadb07d9 100644 --- a/src/safeds/ml/nn/layers/_layer.py +++ b/src/safeds/ml/nn/layers/_layer.py @@ -1,7 +1,9 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self + +from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn @@ -25,7 +27,7 @@ def input_size(self) -> int | ModelImageSize: @property @abstractmethod - def output_size(self) -> int | ModelImageSize: + def output_size(self) -> int | ModelImageSize | Choice[int]: pass # pragma: no cover @abstractmethod @@ -37,7 +39,7 @@ def _contains_choices(self) -> bool: pass # pragma: no cover @abstractmethod - def _get_layers_for_all_choices(self) -> list[Layer]: + def _get_layers_for_all_choices(self) -> list[Self]: pass # pragma: no cover @abstractmethod diff --git a/src/safeds/ml/nn/layers/_lstm_layer.py b/src/safeds/ml/nn/layers/_lstm_layer.py index 3872d524f..3df578ace 100644 --- a/src/safeds/ml/nn/layers/_lstm_layer.py +++ b/src/safeds/ml/nn/layers/_lstm_layer.py @@ -42,6 +42,7 @@ def __init__(self, neuron_count: int | Choice[int]): def _get_internal_layer(self, **kwargs: Any) -> nn.Module: assert not self._contains_choices() + assert not isinstance(self._output_size, Choice) from ._internal_layers import _InternalLSTMLayer # Slow import on global level if "activation_function" not in kwargs: @@ -93,6 +94,7 @@ def _contains_choices(self) -> bool: def _get_layers_for_all_choices(self) -> list[LSTMLayer]: assert self._contains_choices() + assert isinstance(self._output_size, Choice) # just for linter layers = [] for val in self._output_size: layers.append(LSTMLayer(neuron_count=val)) From 8b7a2cfc351e41a3e4c344b9aefbe36807a94b61 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 21:31:53 +0200 Subject: [PATCH 08/38] linter fixes --- src/safeds/ml/nn/_model.py | 18 +++++++++--------- src/safeds/ml/nn/layers/_layer.py | 3 +-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 717598e78..5b8fe5ba7 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -275,7 +275,7 @@ def fit_by_exhaustive_search( raise FittingWithoutChoiceError list_of_models = self._get_models_for_all_choices() - list_of_fitted_models = [] + list_of_fitted_models: list[Self] = [] if isinstance(train_data, TimeSeriesDataset): raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover @@ -302,32 +302,32 @@ def fit_by_exhaustive_search( best_model = fitted_model match optimization_metric.value: case "mean_squared_error": - best_metric_value = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) + best_metric_value = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] case "mean_absolute_error": - best_metric_value = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) + best_metric_value = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] case "median_absolute_deviation": - best_metric_value = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) + best_metric_value = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] case "coefficient_of_determination": - best_metric_value = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) + best_metric_value = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] else: match optimization_metric.value: case "mean_squared_error": - error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) + error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "mean_absolute_error": - error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) + error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "median_absolute_deviation": - error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) + error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "coefficient_of_determination": - error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) + error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model diff --git a/src/safeds/ml/nn/layers/_layer.py b/src/safeds/ml/nn/layers/_layer.py index 9fadb07d9..a527277e3 100644 --- a/src/safeds/ml/nn/layers/_layer.py +++ b/src/safeds/ml/nn/layers/_layer.py @@ -3,11 +3,10 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Self -from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn - + from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize From b718f749afe0c589398eb58571c5be5ada298c44 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 21:40:21 +0200 Subject: [PATCH 09/38] linter fixes --- src/safeds/ml/nn/_model.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 5b8fe5ba7..a787c5353 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -2,7 +2,7 @@ import copy from concurrent.futures import ProcessPoolExecutor, wait, ALL_COMPLETED -from typing import TYPE_CHECKING, Generic, Self, TypeVar +from typing import TYPE_CHECKING, Generic, Self, TypeVar, Any from safeds._config import _init_default_device from safeds._validation import _check_bounds, _ClosedBound @@ -675,7 +675,7 @@ def fit_by_exhaustive_search( self, train_data: IFT, optimization_metric: ClassifierMetric, - positive_class = None, + positive_class: Any = None, epoch_size: int = 25, batch_size: int = 1, learning_rate: float = 0.001, @@ -684,7 +684,7 @@ def fit_by_exhaustive_search( raise FittingWithoutChoiceError list_of_models = self._get_models_for_all_choices() - list_of_fitted_models = [] + list_of_fitted_models: list[Self] = [] if isinstance(train_data, TimeSeriesDataset): raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover @@ -712,32 +712,32 @@ def fit_by_exhaustive_search( best_model = fitted_model match optimization_metric.value: case "accuracy": - best_metric_value = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) + best_metric_value = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] case "precision": - best_metric_value = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + best_metric_value = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] case "recall": - best_metric_value = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + best_metric_value = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] case "f1score": - best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] else: match optimization_metric.value: case "accuracy": - error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) + error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "precision": - error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) + error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "recall": - error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) + error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "f1score": - error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) + error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model From 04c3ac7aade30f4cce8a52c76e2a369cd1f52ae0 Mon Sep 17 00:00:00 2001 From: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> Date: Thu, 11 Jul 2024 19:41:59 +0000 Subject: [PATCH 10/38] style: apply automated linter fixes --- src/safeds/exceptions/_ml.py | 2 + src/safeds/ml/nn/_model.py | 57 +++++++++---------- .../ml/nn/layers/_convolutional2d_layer.py | 2 +- src/safeds/ml/nn/layers/_dropout_layer.py | 2 +- src/safeds/ml/nn/layers/_flatten_layer.py | 2 +- src/safeds/ml/nn/layers/_forward_layer.py | 4 +- src/safeds/ml/nn/layers/_gru_layer.py | 6 +- src/safeds/ml/nn/layers/_layer.py | 2 +- src/safeds/ml/nn/layers/_lstm_layer.py | 2 +- src/safeds/ml/nn/layers/_pooling2d_layer.py | 2 +- 10 files changed, 40 insertions(+), 41 deletions(-) diff --git a/src/safeds/exceptions/_ml.py b/src/safeds/exceptions/_ml.py index a468ed190..977d516d5 100644 --- a/src/safeds/exceptions/_ml.py +++ b/src/safeds/exceptions/_ml.py @@ -41,6 +41,7 @@ class DatasetMissesDataError(ValueError): def __init__(self) -> None: super().__init__("Dataset contains no rows") + class FittingWithChoiceError(Exception): """Raised when a model is fitted with a choice object as a parameter.""" @@ -60,6 +61,7 @@ def __init__(self) -> None: "Parameter. Please use fit() instead.", ) + class InvalidFitDataError(Exception): """Raised when a Neural Network is fitted on invalid data.""" diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index a787c5353..66872e13b 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -1,8 +1,8 @@ from __future__ import annotations import copy -from concurrent.futures import ProcessPoolExecutor, wait, ALL_COMPLETED -from typing import TYPE_CHECKING, Generic, Self, TypeVar, Any +from concurrent.futures import ALL_COMPLETED, ProcessPoolExecutor, wait +from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar from safeds._config import _init_default_device from safeds._validation import _check_bounds, _ClosedBound @@ -13,10 +13,13 @@ from safeds.data.tabular.transformation import OneHotEncoder from safeds.exceptions import ( FeatureDataMismatchError, + FittingWithChoiceError, + FittingWithoutChoiceError, InvalidModelStructureError, - ModelNotFittedError, LearningError, FittingWithoutChoiceError, FittingWithChoiceError, + LearningError, + ModelNotFittedError, ) -from safeds.ml.metrics import ClassificationMetrics, RegressionMetrics, RegressorMetric, ClassifierMetric +from safeds.ml.metrics import ClassificationMetrics, ClassifierMetric, RegressionMetrics, RegressorMetric from safeds.ml.nn.converters import ( InputConversionImageToColumn, InputConversionImageToImage, @@ -284,14 +287,12 @@ def fit_by_exhaustive_search( with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: futures = [] for model in list_of_models: - futures.append( - executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) + futures.append(executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) [done, _] = wait(futures, return_when=ALL_COMPLETED) for future in done: list_of_fitted_models.append(future.result()) executor.shutdown() - target_col = train_data.target test_data = train_data.to_table().remove_columns([target_col.name]) @@ -302,36 +303,36 @@ def fit_by_exhaustive_search( best_model = fitted_model match optimization_metric.value: case "mean_squared_error": - best_metric_value = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + best_metric_value = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] case "mean_absolute_error": - best_metric_value = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + best_metric_value = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] case "median_absolute_deviation": - best_metric_value = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + best_metric_value = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] case "coefficient_of_determination": - best_metric_value = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + best_metric_value = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] else: match optimization_metric.value: case "mean_squared_error": - error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "mean_absolute_error": - error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "median_absolute_deviation": - error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "coefficient_of_determination": - error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model - assert best_model is not None # just for linter + assert best_model is not None # just for linter best_model._is_fitted = True return best_model @@ -359,7 +360,6 @@ def _get_models_for_all_choices(self) -> list[Self]: models.append(copied_model) return models - def predict(self, test_data: IPT) -> IFT: """ Make a prediction for the given test data. @@ -694,14 +694,12 @@ def fit_by_exhaustive_search( with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: futures = [] for model in list_of_models: - futures.append( - executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) + futures.append(executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) [done, _] = wait(futures, return_when=ALL_COMPLETED) for future in done: list_of_fitted_models.append(future.result()) executor.shutdown() - target_col = train_data.target test_data = train_data.to_table().remove_columns([target_col.name]) @@ -712,36 +710,36 @@ def fit_by_exhaustive_search( best_model = fitted_model match optimization_metric.value: case "accuracy": - best_metric_value = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + best_metric_value = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] case "precision": - best_metric_value = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + best_metric_value = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] case "recall": - best_metric_value = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + best_metric_value = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] case "f1score": - best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] else: match optimization_metric.value: case "accuracy": - error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data),expected=target_col) # type: ignore[arg-type] + error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "precision": - error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data),expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "recall": - error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model case "f1score": - error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: best_model = fitted_model best_metric_value = error_of_fitted_model - assert best_model is not None # just for linter + assert best_model is not None # just for linter best_model._is_fitted = True return best_model @@ -830,4 +828,3 @@ def input_size(self) -> int | ModelImageSize | None: def _contains_choices(self) -> bool: """Whether the model contains choices in any layer.""" return any(layer._contains_choices() for layer in self._layers) - diff --git a/src/safeds/ml/nn/layers/_convolutional2d_layer.py b/src/safeds/ml/nn/layers/_convolutional2d_layer.py index 2adb7baf3..b67bd5330 100644 --- a/src/safeds/ml/nn/layers/_convolutional2d_layer.py +++ b/src/safeds/ml/nn/layers/_convolutional2d_layer.py @@ -129,7 +129,7 @@ def _contains_choices(self) -> bool: return False def _get_layers_for_all_choices(self) -> list[Convolutional2DLayer]: - raise NotImplementedError # pragma: no cover + raise NotImplementedError # pragma: no cover def __hash__(self) -> int: return _structural_hash( diff --git a/src/safeds/ml/nn/layers/_dropout_layer.py b/src/safeds/ml/nn/layers/_dropout_layer.py index cd15ad219..cb8fa548f 100644 --- a/src/safeds/ml/nn/layers/_dropout_layer.py +++ b/src/safeds/ml/nn/layers/_dropout_layer.py @@ -91,7 +91,7 @@ def _contains_choices(self) -> bool: return False def _get_layers_for_all_choices(self) -> list[DropoutLayer]: - raise NotImplementedError # pragma: no cover + raise NotImplementedError # pragma: no cover def __hash__(self) -> int: return _structural_hash(self._input_size, self._probability) diff --git a/src/safeds/ml/nn/layers/_flatten_layer.py b/src/safeds/ml/nn/layers/_flatten_layer.py index 0697f3e65..1b2671712 100644 --- a/src/safeds/ml/nn/layers/_flatten_layer.py +++ b/src/safeds/ml/nn/layers/_flatten_layer.py @@ -80,7 +80,7 @@ def _contains_choices(self) -> bool: return False def _get_layers_for_all_choices(self) -> list[FlattenLayer]: - raise NotImplementedError # pragma: no cover + raise NotImplementedError # pragma: no cover def __hash__(self) -> int: return _structural_hash(self._input_size, self._output_size) diff --git a/src/safeds/ml/nn/layers/_forward_layer.py b/src/safeds/ml/nn/layers/_forward_layer.py index 64cddd930..9324af01b 100644 --- a/src/safeds/ml/nn/layers/_forward_layer.py +++ b/src/safeds/ml/nn/layers/_forward_layer.py @@ -4,10 +4,10 @@ from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer -from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn @@ -94,7 +94,7 @@ def _contains_choices(self) -> bool: def _get_layers_for_all_choices(self) -> list[ForwardLayer]: assert self._contains_choices() - assert isinstance(self._output_size, Choice) # just for linter + assert isinstance(self._output_size, Choice) # just for linter layers = [] for val in self._output_size: layers.append(ForwardLayer(neuron_count=val)) diff --git a/src/safeds/ml/nn/layers/_gru_layer.py b/src/safeds/ml/nn/layers/_gru_layer.py index 24d7c3993..5a0ab33ad 100644 --- a/src/safeds/ml/nn/layers/_gru_layer.py +++ b/src/safeds/ml/nn/layers/_gru_layer.py @@ -5,10 +5,10 @@ from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer -from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn @@ -42,7 +42,7 @@ def __init__(self, neuron_count: int | Choice[int]): def _get_internal_layer(self, **kwargs: Any) -> nn.Module: assert not self._contains_choices() - assert not isinstance(self._output_size, Choice) # just for linter + assert not isinstance(self._output_size, Choice) # just for linter from ._internal_layers import _InternalGRULayer # Slow import on global level if "activation_function" not in kwargs: @@ -94,7 +94,7 @@ def _contains_choices(self) -> bool: def _get_layers_for_all_choices(self) -> list[GRULayer]: assert self._contains_choices() - assert isinstance(self._output_size, Choice) # just for linter + assert isinstance(self._output_size, Choice) # just for linter layers = [] for val in self._output_size: layers.append(GRULayer(neuron_count=val)) diff --git a/src/safeds/ml/nn/layers/_layer.py b/src/safeds/ml/nn/layers/_layer.py index a527277e3..d469af9aa 100644 --- a/src/safeds/ml/nn/layers/_layer.py +++ b/src/safeds/ml/nn/layers/_layer.py @@ -3,9 +3,9 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Self - if TYPE_CHECKING: from torch import nn + from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize diff --git a/src/safeds/ml/nn/layers/_lstm_layer.py b/src/safeds/ml/nn/layers/_lstm_layer.py index 3df578ace..48fb2f58f 100644 --- a/src/safeds/ml/nn/layers/_lstm_layer.py +++ b/src/safeds/ml/nn/layers/_lstm_layer.py @@ -5,10 +5,10 @@ from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer -from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from torch import nn diff --git a/src/safeds/ml/nn/layers/_pooling2d_layer.py b/src/safeds/ml/nn/layers/_pooling2d_layer.py index 01bc9f0da..401e95f50 100644 --- a/src/safeds/ml/nn/layers/_pooling2d_layer.py +++ b/src/safeds/ml/nn/layers/_pooling2d_layer.py @@ -106,7 +106,7 @@ def _contains_choices(self) -> bool: return False def _get_layers_for_all_choices(self) -> list[_Pooling2DLayer]: - raise NotImplementedError # pragma: no cover + raise NotImplementedError # pragma: no cover def __hash__(self) -> int: return _structural_hash( From 3a817ced2e6a79f49b286de9f700522cb49d7dd5 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Jul 2024 23:16:32 +0200 Subject: [PATCH 11/38] added tests fir invalid outputsizes in choices --- tests/safeds/ml/nn/layers/test_forward_layer.py | 6 ++++-- tests/safeds/ml/nn/layers/test_gru_layer.py | 6 ++++-- tests/safeds/ml/nn/layers/test_lstm_layer.py | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/safeds/ml/nn/layers/test_forward_layer.py b/tests/safeds/ml/nn/layers/test_forward_layer.py index 0ecd3bd05..7f1fcc1ae 100644 --- a/tests/safeds/ml/nn/layers/test_forward_layer.py +++ b/tests/safeds/ml/nn/layers/test_forward_layer.py @@ -4,6 +4,7 @@ import pytest from safeds.data.image.typing import ImageSize from safeds.exceptions import OutOfBoundsError +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.layers import ForwardLayer from torch import nn @@ -66,10 +67,11 @@ def test_should_raise_if_unknown_activation_function_is_passed(activation_functi "output_size", [ 0, + Choice(0), ], - ids=["output_size_out_of_bounds"], + ids=["invalid_int", "invalid_choice"], ) -def test_should_raise_if_output_size_out_of_bounds(output_size: int) -> None: +def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int]) -> None: with pytest.raises(OutOfBoundsError): ForwardLayer(neuron_count=output_size) diff --git a/tests/safeds/ml/nn/layers/test_gru_layer.py b/tests/safeds/ml/nn/layers/test_gru_layer.py index 4a6f366e4..c3efe4b3c 100644 --- a/tests/safeds/ml/nn/layers/test_gru_layer.py +++ b/tests/safeds/ml/nn/layers/test_gru_layer.py @@ -4,6 +4,7 @@ import pytest from safeds.data.image.typing import ImageSize from safeds.exceptions import OutOfBoundsError +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.layers import GRULayer from torch import nn @@ -54,10 +55,11 @@ def test_should_raise_if_unknown_activation_function_is_passed(activation_functi "output_size", [ 0, + Choice(0), ], - ids=["output_size_out_of_bounds"], + ids=["invalid_int", "invalid_choice"], ) -def test_should_raise_if_output_size_out_of_bounds(output_size: int) -> None: +def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int]) -> None: with pytest.raises(OutOfBoundsError): GRULayer(neuron_count=output_size) diff --git a/tests/safeds/ml/nn/layers/test_lstm_layer.py b/tests/safeds/ml/nn/layers/test_lstm_layer.py index 8d58e5dd8..f7783bfba 100644 --- a/tests/safeds/ml/nn/layers/test_lstm_layer.py +++ b/tests/safeds/ml/nn/layers/test_lstm_layer.py @@ -4,6 +4,7 @@ import pytest from safeds.data.image.typing import ImageSize from safeds.exceptions import OutOfBoundsError +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.layers import LSTMLayer from torch import nn @@ -66,10 +67,11 @@ def test_should_raise_if_unknown_activation_function_is_passed(activation_functi "output_size", [ 0, + Choice(0), ], - ids=["output_size_out_of_bounds"], + ids=["invalid_int", "invalid_choice"], ) -def test_should_raise_if_output_size_out_of_bounds(output_size: int) -> None: +def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int]) -> None: with pytest.raises(OutOfBoundsError): LSTMLayer(neuron_count=output_size) From 07572fc61985c527c8e18d05efa85663c113aa30 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:14:27 +0200 Subject: [PATCH 12/38] add __eq__ to choice --- src/safeds/ml/hyperparameters/_choice.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/safeds/ml/hyperparameters/_choice.py b/src/safeds/ml/hyperparameters/_choice.py index 6d0f59db2..2b5a2e1b3 100644 --- a/src/safeds/ml/hyperparameters/_choice.py +++ b/src/safeds/ml/hyperparameters/_choice.py @@ -61,3 +61,10 @@ def __len__(self) -> int: The number of values in this choice. """ return len(self.elements) + + def __eq__(self, other): + if not isinstance(other, Choice): + return NotImplemented + if self is other: + return True + return (self is other) or (self.elements == other.elements) From 5f13bfa0edbdf201dbfba0f9daf7a436283d7d6e Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:25:58 +0200 Subject: [PATCH 13/38] add gru_layer tests --- src/safeds/ml/nn/layers/_gru_layer.py | 2 +- tests/safeds/ml/nn/layers/test_gru_layer.py | 43 ++++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/safeds/ml/nn/layers/_gru_layer.py b/src/safeds/ml/nn/layers/_gru_layer.py index 5a0ab33ad..3a0fae379 100644 --- a/src/safeds/ml/nn/layers/_gru_layer.py +++ b/src/safeds/ml/nn/layers/_gru_layer.py @@ -85,7 +85,7 @@ def output_size(self) -> int | Choice[int]: def _set_input_size(self, input_size: int | ModelImageSize) -> None: if isinstance(input_size, ModelImageSize): - raise TypeError("The input_size of a forward layer has to be of type int.") + raise TypeError("The input_size of a gru layer has to be of type int.") self._input_size = input_size diff --git a/tests/safeds/ml/nn/layers/test_gru_layer.py b/tests/safeds/ml/nn/layers/test_gru_layer.py index c3efe4b3c..137a58174 100644 --- a/tests/safeds/ml/nn/layers/test_gru_layer.py +++ b/tests/safeds/ml/nn/layers/test_gru_layer.py @@ -69,16 +69,17 @@ def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int [ 1, 20, + Choice(1, 20) ], - ids=["one", "twenty"], + ids=["one", "twenty", "choice"], ) -def test_should_raise_if_output_size_doesnt_match(output_size: int) -> None: +def test_should_raise_if_output_size_doesnt_match(output_size: int | Choice[int]) -> None: assert GRULayer(neuron_count=output_size).output_size == output_size def test_should_raise_if_input_size_is_set_with_image_size() -> None: layer = GRULayer(1) - with pytest.raises(TypeError, match=r"The input_size of a forward layer has to be of type int."): + with pytest.raises(TypeError, match=r"The input_size of a gru layer has to be of type int."): layer._set_input_size(ImageSize(1, 2, 3)) @@ -104,14 +105,29 @@ def test_should_raise_if_activation_function_not_set() -> None: GRULayer(neuron_count=1), False, ), + ( + GRULayer(neuron_count=Choice(2)), + GRULayer(neuron_count=Choice(2)), + True, + ), + ( + GRULayer(neuron_count=Choice(2)), + GRULayer(neuron_count=Choice(1)), + False, + ), + ( + GRULayer(neuron_count=Choice(2)), + GRULayer(neuron_count=2), + False, + ), ], - ids=["equal", "not equal"], + ids=["equal", "not equal", "equal choices", "not equal choices", "choice and int"], ) -def test_should_compare_forward_layers(layer1: GRULayer, layer2: GRULayer, equal: bool) -> None: +def test_should_compare_gru_layers(layer1: GRULayer, layer2: GRULayer, equal: bool) -> None: assert (layer1.__eq__(layer2)) == equal -def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: +def test_should_assert_that_gru_layer_is_equal_to_itself() -> None: layer = GRULayer(neuron_count=1) assert layer.__eq__(layer) @@ -121,9 +137,9 @@ def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: [ (GRULayer(neuron_count=1), None), ], - ids=["ForwardLayer vs. None"], + ids=["GRULayer vs. None"], ) -def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: GRULayer, other: Any) -> None: +def test_should_return_not_implemented_if_other_is_not_gru_layer(layer: GRULayer, other: Any) -> None: assert (layer.__eq__(other)) is NotImplemented @@ -137,7 +153,7 @@ def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: GRUL ], ids=["equal"], ) -def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: GRULayer, layer2: GRULayer) -> None: +def test_should_assert_that_equal_gru_layers_have_equal_hash(layer1: GRULayer, layer2: GRULayer) -> None: assert layer1.__hash__() == layer2.__hash__() @@ -151,7 +167,7 @@ def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: GRULaye ], ids=["not equal"], ) -def test_should_assert_that_different_forward_layers_have_different_hash( +def test_should_assert_that_different_gru_layers_have_different_hash( layer1: GRULayer, layer2: GRULayer, ) -> None: @@ -189,3 +205,10 @@ def test_internal_layer_should_raise_error() -> None: layer = GRULayer(1) with pytest.raises(ValueError, match="The input_size is not yet set."): layer._get_internal_layer(activation_function="relu") + + +def test_should_get_all_possible_combinations_of_gru_layer() -> None: + layer = GRULayer(Choice(1, 2)) + possible_layers = layer._get_layers_for_all_choices() + assert possible_layers[0] == GRULayer(1) + assert possible_layers[1] == GRULayer(2) From 2dc29f203c0bacc6033062c649c5c1ae8329aafb Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:34:33 +0200 Subject: [PATCH 14/38] update choice to remove duplicates and raise EmptyChoiceError when empty --- src/safeds/exceptions/__init__.py | 5 +++- src/safeds/exceptions/_ml.py | 7 +++++ src/safeds/ml/hyperparameters/_choice.py | 5 +++- .../safeds/ml/hyperparameters/test_choice.py | 26 +++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/safeds/exceptions/__init__.py b/src/safeds/exceptions/__init__.py index 105edbab7..f8cf282c5 100644 --- a/src/safeds/exceptions/__init__.py +++ b/src/safeds/exceptions/__init__.py @@ -16,6 +16,7 @@ from ._ml import ( DatasetMissesDataError, DatasetMissesFeaturesError, + EmptyChoiceError, FeatureDataMismatchError, FittingWithChoiceError, FittingWithoutChoiceError, @@ -72,7 +73,8 @@ class OutOfBoundsError(SafeDsError): # ML exceptions "DatasetMissesDataError", "DatasetMissesFeaturesError", - "TargetDataMismatchError", + + "EmptyChoiceError", "FeatureDataMismatchError", "FittingWithChoiceError", "FittingWithoutChoiceError", @@ -83,4 +85,5 @@ class OutOfBoundsError(SafeDsError): "ModelNotFittedError", "PlainTableError", "PredictionError", + "TargetDataMismatchError", ] diff --git a/src/safeds/exceptions/_ml.py b/src/safeds/exceptions/_ml.py index 977d516d5..4f0462188 100644 --- a/src/safeds/exceptions/_ml.py +++ b/src/safeds/exceptions/_ml.py @@ -42,6 +42,13 @@ def __init__(self) -> None: super().__init__("Dataset contains no rows") +class EmptyChoiceError(ValueError): + """Raised when a choice object is created, but no arguments are provided.""" + + def __init__(self) -> None: + super().__init__("Please provide at least one Value in a Choice Parameter") + + class FittingWithChoiceError(Exception): """Raised when a model is fitted with a choice object as a parameter.""" diff --git a/src/safeds/ml/hyperparameters/_choice.py b/src/safeds/ml/hyperparameters/_choice.py index 2b5a2e1b3..6a1f169a7 100644 --- a/src/safeds/ml/hyperparameters/_choice.py +++ b/src/safeds/ml/hyperparameters/_choice.py @@ -16,13 +16,16 @@ class Choice(Collection[T]): def __init__(self, *args: T) -> None: """ Create a new choice. - + Duplicate values will be removed. Parameters ---------- *args: The values to choose from. """ self.elements = list(args) + if len(args) < 1: + raise EmptyChoiceError + self.elements = list(dict.fromkeys(args)) def __contains__(self, value: Any) -> bool: """ diff --git a/tests/safeds/ml/hyperparameters/test_choice.py b/tests/safeds/ml/hyperparameters/test_choice.py index 8adcd5952..55d52f3e2 100644 --- a/tests/safeds/ml/hyperparameters/test_choice.py +++ b/tests/safeds/ml/hyperparameters/test_choice.py @@ -61,3 +61,29 @@ class TestLen: ) def test_should_return_number_of_values(self, choice: Choice, expected: int) -> None: assert len(choice) == expected + + +class TestEq: + @pytest.mark.parametrize( + ("choice1", "choice2", "equal"), + [ + ( + Choice(1), + Choice(1), + True, + ), + ( + Choice(1), + Choice(2), + False, + ), + ( + Choice(1, 2, 3), + Choice(1, 2, 3), + True, + ), + ], + ids=["equal", "not_equal", "equal with multiple values"], + ) + def test_should_compare_choices(self, choice1: Choice[int], choice2: Choice[int], equal: bool) -> None: + assert (choice1.__eq__(choice2)) == equal From d018cf23c554f82ec34d10a28192b904255290b3 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:35:10 +0200 Subject: [PATCH 15/38] forgot to import whoops --- src/safeds/ml/hyperparameters/_choice.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/safeds/ml/hyperparameters/_choice.py b/src/safeds/ml/hyperparameters/_choice.py index 6a1f169a7..cdb37445c 100644 --- a/src/safeds/ml/hyperparameters/_choice.py +++ b/src/safeds/ml/hyperparameters/_choice.py @@ -3,6 +3,8 @@ from collections.abc import Collection from typing import TYPE_CHECKING, TypeVar +from safeds.exceptions import EmptyChoiceError + if TYPE_CHECKING: from collections.abc import Iterator from typing import Any From 87224eed3c93539e27c5185a816c34a2f043019d Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:39:35 +0200 Subject: [PATCH 16/38] add choice tests --- tests/safeds/ml/hyperparameters/test_choice.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/safeds/ml/hyperparameters/test_choice.py b/tests/safeds/ml/hyperparameters/test_choice.py index 55d52f3e2..758f8a081 100644 --- a/tests/safeds/ml/hyperparameters/test_choice.py +++ b/tests/safeds/ml/hyperparameters/test_choice.py @@ -3,12 +3,20 @@ from typing import TYPE_CHECKING import pytest + +from safeds.exceptions import EmptyChoiceError from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from typing import Any +class TestInit: + def test_should_iterate_values(self) -> None: + with pytest.raises(EmptyChoiceError): + Choice() + + class TestContains: @pytest.mark.parametrize( ("choice", "value", "expected"), @@ -35,11 +43,9 @@ class TestIter: @pytest.mark.parametrize( ("choice", "expected"), [ - (Choice(), []), (Choice(1, 2, 3), [1, 2, 3]), ], ids=[ - "empty", "non-empty", ], ) @@ -51,11 +57,9 @@ class TestLen: @pytest.mark.parametrize( ("choice", "expected"), [ - (Choice(), 0), (Choice(1, 2, 3), 3), ], ids=[ - "empty", "non-empty", ], ) From 1c3da1c69406aa4879084039f005477061d4e826 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:47:31 +0200 Subject: [PATCH 17/38] add lstm layer tests --- src/safeds/ml/nn/layers/_lstm_layer.py | 2 +- tests/safeds/ml/nn/layers/test_lstm_layer.py | 43 +++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/safeds/ml/nn/layers/_lstm_layer.py b/src/safeds/ml/nn/layers/_lstm_layer.py index 48fb2f58f..6202d4fd2 100644 --- a/src/safeds/ml/nn/layers/_lstm_layer.py +++ b/src/safeds/ml/nn/layers/_lstm_layer.py @@ -85,7 +85,7 @@ def output_size(self) -> int | Choice[int]: def _set_input_size(self, input_size: int | ModelImageSize) -> None: if isinstance(input_size, ModelImageSize): - raise TypeError("The input_size of a forward layer has to be of type int.") + raise TypeError("The input_size of a lstm layer has to be of type int.") self._input_size = input_size diff --git a/tests/safeds/ml/nn/layers/test_lstm_layer.py b/tests/safeds/ml/nn/layers/test_lstm_layer.py index f7783bfba..12063e9fe 100644 --- a/tests/safeds/ml/nn/layers/test_lstm_layer.py +++ b/tests/safeds/ml/nn/layers/test_lstm_layer.py @@ -81,16 +81,17 @@ def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int [ 1, 20, + Choice(1, 20) ], - ids=["one", "twenty"], + ids=["one", "twenty", "choice"], ) -def test_should_raise_if_output_size_doesnt_match(output_size: int) -> None: +def test_should_raise_if_output_size_doesnt_match(output_size: int | Choice[int]) -> None: assert LSTMLayer(neuron_count=output_size).output_size == output_size def test_should_raise_if_input_size_is_set_with_image_size() -> None: layer = LSTMLayer(1) - with pytest.raises(TypeError, match=r"The input_size of a forward layer has to be of type int."): + with pytest.raises(TypeError, match=r"The input_size of a lstm layer has to be of type int."): layer._set_input_size(ImageSize(1, 2, 3)) @@ -116,14 +117,29 @@ def test_should_raise_if_activation_function_not_set() -> None: LSTMLayer(neuron_count=1), False, ), +( + LSTMLayer(neuron_count=Choice(2)), + LSTMLayer(neuron_count=Choice(2)), + True, + ), + ( + LSTMLayer(neuron_count=Choice(2)), + LSTMLayer(neuron_count=Choice(1)), + False, + ), + ( + LSTMLayer(neuron_count=Choice(2)), + LSTMLayer(neuron_count=2), + False, + ), ], - ids=["equal", "not equal"], + ids=["equal", "not equal", "equal choices", "not equal choices", "choice and int"], ) -def test_should_compare_forward_layers(layer1: LSTMLayer, layer2: LSTMLayer, equal: bool) -> None: +def test_should_compare_lstm_layers(layer1: LSTMLayer, layer2: LSTMLayer, equal: bool) -> None: assert (layer1.__eq__(layer2)) == equal -def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: +def test_should_assert_that_lstm_layer_is_equal_to_itself() -> None: layer = LSTMLayer(neuron_count=1) assert layer.__eq__(layer) @@ -133,9 +149,9 @@ def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: [ (LSTMLayer(neuron_count=1), None), ], - ids=["ForwardLayer vs. None"], + ids=["LSTMLayer vs. None"], ) -def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: LSTMLayer, other: Any) -> None: +def test_should_return_not_implemented_if_other_is_not_lstm_layer(layer: LSTMLayer, other: Any) -> None: assert (layer.__eq__(other)) is NotImplemented @@ -149,7 +165,7 @@ def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: LSTM ], ids=["equal"], ) -def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: LSTMLayer, layer2: LSTMLayer) -> None: +def test_should_assert_that_equal_lstm_layers_have_equal_hash(layer1: LSTMLayer, layer2: LSTMLayer) -> None: assert layer1.__hash__() == layer2.__hash__() @@ -163,7 +179,7 @@ def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: LSTMLay ], ids=["not equal"], ) -def test_should_assert_that_different_forward_layers_have_different_hash( +def test_should_assert_that_different_lstm_layers_have_different_hash( layer1: LSTMLayer, layer2: LSTMLayer, ) -> None: @@ -179,3 +195,10 @@ def test_should_assert_that_different_forward_layers_have_different_hash( ) def test_should_assert_that_layer_size_is_greater_than_normal_object(layer: LSTMLayer) -> None: assert sys.getsizeof(layer) > sys.getsizeof(object()) + + +def test_should_get_all_possible_combinations_of_gru_layer() -> None: + layer = LSTMLayer(Choice(1, 2)) + possible_layers = layer._get_layers_for_all_choices() + assert possible_layers[0] == LSTMLayer(1) + assert possible_layers[1] == LSTMLayer(2) From 00a36fe3a1c3074e37f6fcb16e9aae704d1875be Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:51:55 +0200 Subject: [PATCH 18/38] add tests --- .../safeds/ml/nn/layers/test_forward_layer.py | 21 ++++++++++++++++--- tests/safeds/ml/nn/layers/test_lstm_layer.py | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/safeds/ml/nn/layers/test_forward_layer.py b/tests/safeds/ml/nn/layers/test_forward_layer.py index 7f1fcc1ae..f155931d3 100644 --- a/tests/safeds/ml/nn/layers/test_forward_layer.py +++ b/tests/safeds/ml/nn/layers/test_forward_layer.py @@ -81,10 +81,11 @@ def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int [ 1, 20, + Choice(1, 20) ], - ids=["one", "twenty"], + ids=["one", "twenty", "choice"], ) -def test_should_return_output_size(output_size: int) -> None: +def test_should_return_output_size(output_size: int | Choice[int]) -> None: assert ForwardLayer(neuron_count=output_size).output_size == output_size @@ -116,8 +117,22 @@ def test_should_raise_if_activation_function_not_set() -> None: ForwardLayer(neuron_count=1), False, ), + ( ForwardLayer(neuron_count=Choice(2)), + ForwardLayer(neuron_count=Choice(2)), + True, + ), + ( + ForwardLayer(neuron_count=Choice(2)), + ForwardLayer(neuron_count=Choice(1)), + False, + ), + ( + ForwardLayer(neuron_count=Choice(2)), + ForwardLayer(neuron_count=2), + False, + ), ], - ids=["equal", "not equal"], + ids=["equal", "not equal", "equal choices", "not equal choices", "choice and int"], ) def test_should_compare_forward_layers(layer1: ForwardLayer, layer2: ForwardLayer, equal: bool) -> None: assert (layer1.__eq__(layer2)) == equal diff --git a/tests/safeds/ml/nn/layers/test_lstm_layer.py b/tests/safeds/ml/nn/layers/test_lstm_layer.py index 12063e9fe..fb036de45 100644 --- a/tests/safeds/ml/nn/layers/test_lstm_layer.py +++ b/tests/safeds/ml/nn/layers/test_lstm_layer.py @@ -117,7 +117,7 @@ def test_should_raise_if_activation_function_not_set() -> None: LSTMLayer(neuron_count=1), False, ), -( + ( LSTMLayer(neuron_count=Choice(2)), LSTMLayer(neuron_count=Choice(2)), True, From 4601b0a50764ac1c327d1ac917b3b8443d6df2b8 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:54:41 +0200 Subject: [PATCH 19/38] add tests --- tests/safeds/ml/nn/layers/test_forward_layer.py | 7 +++++++ tests/safeds/ml/nn/layers/test_lstm_layer.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/safeds/ml/nn/layers/test_forward_layer.py b/tests/safeds/ml/nn/layers/test_forward_layer.py index f155931d3..9b2f8420a 100644 --- a/tests/safeds/ml/nn/layers/test_forward_layer.py +++ b/tests/safeds/ml/nn/layers/test_forward_layer.py @@ -194,3 +194,10 @@ def test_should_assert_that_different_forward_layers_have_different_hash( ) def test_should_assert_that_layer_size_is_greater_than_normal_object(layer: ForwardLayer) -> None: assert sys.getsizeof(layer) > sys.getsizeof(object()) + + +def test_should_get_all_possible_combinations_of_forward_layer() -> None: + layer = ForwardLayer(Choice(1, 2)) + possible_layers = layer._get_layers_for_all_choices() + assert possible_layers[0] == ForwardLayer(1) + assert possible_layers[1] == ForwardLayer(2) diff --git a/tests/safeds/ml/nn/layers/test_lstm_layer.py b/tests/safeds/ml/nn/layers/test_lstm_layer.py index fb036de45..996f92705 100644 --- a/tests/safeds/ml/nn/layers/test_lstm_layer.py +++ b/tests/safeds/ml/nn/layers/test_lstm_layer.py @@ -197,7 +197,7 @@ def test_should_assert_that_layer_size_is_greater_than_normal_object(layer: LSTM assert sys.getsizeof(layer) > sys.getsizeof(object()) -def test_should_get_all_possible_combinations_of_gru_layer() -> None: +def test_should_get_all_possible_combinations_of_lstm_layer() -> None: layer = LSTMLayer(Choice(1, 2)) possible_layers = layer._get_layers_for_all_choices() assert possible_layers[0] == LSTMLayer(1) From 4b8d3f6d67f6fad738c45203d5a9519adeced9c1 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 21:59:07 +0200 Subject: [PATCH 20/38] linter fix --- src/safeds/ml/hyperparameters/_choice.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/safeds/ml/hyperparameters/_choice.py b/src/safeds/ml/hyperparameters/_choice.py index cdb37445c..191037ff5 100644 --- a/src/safeds/ml/hyperparameters/_choice.py +++ b/src/safeds/ml/hyperparameters/_choice.py @@ -17,8 +17,8 @@ class Choice(Collection[T]): def __init__(self, *args: T) -> None: """ - Create a new choice. - Duplicate values will be removed. + Create a new choice. Duplicate values will be removed. + Parameters ---------- *args: @@ -67,7 +67,7 @@ def __len__(self) -> int: """ return len(self.elements) - def __eq__(self, other): + def __eq__(self, other) -> bool: if not isinstance(other, Choice): return NotImplemented if self is other: From fa141e837f2b76424fed268077e215e7406f8216 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 13 Jul 2024 00:24:32 +0200 Subject: [PATCH 21/38] linter fix --- src/safeds/ml/hyperparameters/_choice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/safeds/ml/hyperparameters/_choice.py b/src/safeds/ml/hyperparameters/_choice.py index 191037ff5..09530ded2 100644 --- a/src/safeds/ml/hyperparameters/_choice.py +++ b/src/safeds/ml/hyperparameters/_choice.py @@ -67,7 +67,7 @@ def __len__(self) -> int: """ return len(self.elements) - def __eq__(self, other) -> bool: + def __eq__(self, other: object) -> bool: if not isinstance(other, Choice): return NotImplemented if self is other: From 87549b7ecbd9cad1f26f16c6ef3febff7b80d149 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 13 Jul 2024 19:00:09 +0200 Subject: [PATCH 22/38] add tests for fit_by_exhaustive_search and restructure model tests --- src/safeds/ml/nn/_model.py | 57 +- tests/safeds/ml/nn/test_model.py | 1064 ++++++++++++++++-------------- 2 files changed, 628 insertions(+), 493 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 66872e13b..9fcedb2f8 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -274,16 +274,23 @@ def fit_by_exhaustive_search( batch_size: int = 1, learning_rate: float = 0.001, ) -> Self: + + _init_default_device() + if not self._contains_choices(): raise FittingWithoutChoiceError - list_of_models = self._get_models_for_all_choices() - list_of_fitted_models: list[Self] = [] - if isinstance(train_data, TimeSeriesDataset): raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover if isinstance(train_data, ImageDataset): raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + + _check_bounds("epoch_size", epoch_size, lower_bound=_ClosedBound(1)) + _check_bounds("batch_size", batch_size, lower_bound=_ClosedBound(1)) + + list_of_models = self._get_models_for_all_choices() + list_of_fitted_models: list[Self] = [] + with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: futures = [] for model in list_of_models: @@ -293,8 +300,17 @@ def fit_by_exhaustive_search( list_of_fitted_models.append(future.result()) executor.shutdown() + # Cross Validation + [train_split, test_split] = train_data.to_table().split_rows(0.75) + train_data = train_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ) + test_data = test_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ).features target_col = train_data.target - test_data = train_data.to_table().remove_columns([target_col.name]) best_model = None best_metric_value = None @@ -355,9 +371,8 @@ def _get_models_for_all_choices(self) -> list[Self]: models = [] for combination in all_possible_layer_combinations: - copied_model = copy.deepcopy(self) - copied_model._layers = combination - models.append(copied_model) + new_model = NeuralNetworkRegressor(input_conversion=self._input_conversion, layers=combination) + models.append(new_model) return models def predict(self, test_data: IPT) -> IFT: @@ -680,17 +695,23 @@ def fit_by_exhaustive_search( batch_size: int = 1, learning_rate: float = 0.001, ) -> Self: + + _init_default_device() + if not self._contains_choices(): raise FittingWithoutChoiceError - list_of_models = self._get_models_for_all_choices() - list_of_fitted_models: list[Self] = [] - if isinstance(train_data, TimeSeriesDataset): raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover if isinstance(train_data, ImageDataset): raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + _check_bounds("epoch_size", epoch_size, lower_bound=_ClosedBound(1)) + _check_bounds("batch_size", batch_size, lower_bound=_ClosedBound(1)) + + list_of_models = self._get_models_for_all_choices() + list_of_fitted_models: list[Self] = [] + with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: futures = [] for model in list_of_models: @@ -700,8 +721,17 @@ def fit_by_exhaustive_search( list_of_fitted_models.append(future.result()) executor.shutdown() + # Cross Validation + [train_split, test_split] = train_data.to_table().split_rows(0.75) + train_data = train_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ) + test_data = test_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ).features target_col = train_data.target - test_data = train_data.to_table().remove_columns([target_col.name]) best_model = None best_metric_value = None @@ -762,9 +792,8 @@ def _get_models_for_all_choices(self) -> list[Self]: models = [] for combination in all_possible_layer_combinations: - copied_model = copy.deepcopy(self) - copied_model._layers = combination - models.append(copied_model) + new_model = NeuralNetworkClassifier(input_conversion=self._input_conversion, layers=combination) + models.append(new_model) return models def predict(self, test_data: IPT) -> IFT: diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index 43fc67aa6..a1cc21b7e 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -10,8 +10,10 @@ InvalidFitDataError, InvalidModelStructureError, ModelNotFittedError, - OutOfBoundsError, + OutOfBoundsError, FittingWithChoiceError, FittingWithoutChoiceError, ) +from safeds.ml.hyperparameters import Choice +from safeds.ml.metrics import ClassifierMetric, RegressorMetric from safeds.ml.nn import ( NeuralNetworkClassifier, NeuralNetworkRegressor, @@ -41,302 +43,355 @@ @pytest.mark.parametrize("device", get_devices(), ids=get_devices_ids()) class TestClassificationModel: - def test_should_return_input_size(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - ) - - assert model.input_size == 1 - - @pytest.mark.parametrize( - "epoch_size", - [ - 0, - ], - ids=["epoch_size_out_of_bounds"], - ) - def test_should_raise_if_epoch_size_out_of_bounds(self, epoch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkClassifier( + class TestFit: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( InputConversionTable(), - [ForwardLayer(1)], + [ForwardLayer(neuron_count=1)], ).fit( Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - epoch_size=epoch_size, ) - @pytest.mark.parametrize( - "batch_size", - [ - 0, - ], - ids=["batch_size_out_of_bounds"], - ) - def test_should_raise_if_batch_size_out_of_bounds(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkClassifier( + assert model.input_size == 1 + + def test_should_raise_if_epoch_size_out_of_bounds(self, device: Device) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + epoch_size=invalid_epoch_size, + ) + + def test_should_raise_if_batch_size_out_of_bounds(self, device: Device) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + batch_size=invalid_batch_size, + ) + + def test_should_raise_if_fit_function_returns_wrong_datatype(self, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkClassifier( InputConversionTable(), - [ForwardLayer(neuron_count=1)], + [ForwardLayer(neuron_count=2), ForwardLayer(neuron_count=1)], ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - batch_size=batch_size, + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), ) + assert isinstance(fitted_model, NeuralNetworkClassifier) - def test_should_raise_if_fit_function_returns_wrong_datatype(self, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - ) - assert isinstance(fitted_model, NeuralNetworkClassifier) - - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1, 0, 1, 0, 1, 0], "b": [0, 1, 0, 12, 3, 3]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - predictions = fitted_model.predict(Table.from_dict({"b": [1, 0]})) - assert isinstance(predictions, TabularDataset) + def test_should_raise_when_fitting_with_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(1, 2))]) + with pytest.raises(FittingWithChoiceError): + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a")) - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_predict_function_returns_wrong_datatype_for_multiclass_classification( - self, - batch_size: int, - device: Device, - ) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=3)], - ).fit( - Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), LSTMLayer(neuron_count=3)], - ).fit( - Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - predictions = fitted_model.predict(Table.from_dict({"b": [1, 4, 124]})) - assert isinstance(predictions, TabularDataset) - - def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): - NeuralNetworkClassifier( + def test_should_raise_if_is_fitted_is_set_correctly_for_binary_classification(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( InputConversionTable(), [ForwardLayer(neuron_count=1)], - ).predict( - Table.from_dict({"a": [1]}), ) - - def test_should_raise_if_is_fitted_is_set_correctly_for_binary_classification(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - model_2 = NeuralNetworkClassifier( - InputConversionTable(), - [LSTMLayer(neuron_count=1)], - ) - assert not model.is_fitted - assert not model_2.is_fitted - model = model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - ) - model_2 = model_2.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - ) - assert model.is_fitted - assert model_2.is_fitted - - def test_should_raise_if_is_fitted_is_set_correctly_for_multiclass_classification(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - ) - model_2 = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), LSTMLayer(neuron_count=3)], - ) - assert not model.is_fitted - assert not model_2.is_fitted - model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - model_2 = model_2.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - assert model.is_fitted - assert model_2.is_fitted - - def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - ) - model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - model.predict( - Table.from_dict({"a": [1], "c": [2]}), + model_2 = NeuralNetworkClassifier( + InputConversionTable(), + [LSTMLayer(neuron_count=1)], + ) + assert not model.is_fitted + assert not model_2.is_fitted + model = model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + ) + model_2 = model_2.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), ) + assert model.is_fitted + assert model_2.is_fitted - def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=1)], - ) - learned_model = model.fit( - Table.from_dict({"a": [0.1, 0, 0.2], "b": [0, 0.15, 0.5]}).to_tabular_dataset("b"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - learned_model.fit(Table.from_dict({"k": [0.1, 0, 0.2], "l": [0, 0.15, 0.5]}).to_tabular_dataset("k")) + def test_should_raise_if_is_fitted_is_set_correctly_for_multiclass_classification(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], + ) + model_2 = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), LSTMLayer(neuron_count=3)], + ) + assert not model.is_fitted + assert not model_2.is_fitted + model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + model_2 = model_2.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + assert model.is_fitted + assert model_2.is_fitted - @pytest.mark.parametrize( - ("table", "reason"), - [ - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", + def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=1)], + ) + learned_model = model.fit( + Table.from_dict({"a": [0.1, 0, 0.2], "b": [0, 0.15, 0.5]}).to_tabular_dataset("b"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + learned_model.fit(Table.from_dict({"k": [0.1, 0, 0.2], "l": [0, 0.15, 0.5]}).to_tabular_dataset("k")) + + @pytest.mark.parametrize( + ("table", "reason"), + [ + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), - ), - ], - ids=[ - "missing value feature", - "non-numerical feature", - "missing value and non-numerical features", - "missing value target", - "non-numerical target", - ], - ) - def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=4), ForwardLayer(1)], - ) - with pytest.raises( - InvalidFitDataError, - match=reason, - ): - model.fit(table) - - # def test_should_raise_if_table_size_and_input_size_mismatch(self, device: Device) -> None: - # configure_test_with_device(device) - # model = NeuralNetworkClassifier( - # InputConversionTable(), - # [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - # ) - # with pytest.raises( - # InputSizeError, - # ): - # model.fit( - # Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5], "c": [3, 33, 333]}).to_tabular_dataset("a"), - # ) - - def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), + ), + ], + ids=[ + "missing value feature", + "non-numerical feature", + "missing value and non-numerical features", + "missing value target", + "non-numerical target", + ], ) + def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=4), ForwardLayer(1)], + ) + with pytest.raises( + InvalidFitDataError, + match=reason, + ): + model.fit(table) + + def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) - class Test: - self.was_called = False + class Test: + self.was_called = False - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True - def callback_was_called(self) -> bool: - return self.was_called + def callback_was_called(self) -> bool: + return self.was_called - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb) + obj = Test() + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_batch_completion=obj.cb) - assert obj.callback_was_called() is True + assert obj.callback_was_called() is True - def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) + def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) + + class Test: + self.was_called = False - class Test: - self.was_called = False + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + def callback_was_called(self) -> bool: + return self.was_called - def callback_was_called(self) -> bool: - return self.was_called + obj = Test() + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_epoch_completion=obj.cb) - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb) + assert obj.callback_was_called() is True + + class TestFitByExhaustiveSearch: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), + ClassifierMetric.ACCURACY, + ) + assert model.input_size == 1 + + def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, + device: Device) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), + ClassifierMetric.ACCURACY, + epoch_size=invalid_epoch_size, + ) + + def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, + device: Device) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), + ClassifierMetric.ACCURACY, + batch_size=invalid_batch_size, + ) + + def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(1)]) + with pytest.raises(FittingWithoutChoiceError): + model.fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), + ClassifierMetric.ACCURACY) + + def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) + assert not model.is_fitted + fitted_model = model.fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), + ClassifierMetric.ACCURACY) + assert fitted_model.is_fitted + + def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatype(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) + fitted_model = model.fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), + ClassifierMetric.ACCURACY) + assert isinstance(fitted_model, NeuralNetworkClassifier) + + class TestPredict: + + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1, 0, 1, 0, 1, 0], "b": [0, 1, 0, 12, 3, 3]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + predictions = fitted_model.predict(Table.from_dict({"b": [1, 0]})) + assert isinstance(predictions, TabularDataset) - assert obj.callback_was_called() is True + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_predict_function_returns_wrong_datatype_for_multiclass_classification( + self, + batch_size: int, + device: Device, + ) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=3)], + ).fit( + Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=8), LSTMLayer(neuron_count=3)], + ).fit( + Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + predictions = fitted_model.predict(Table.from_dict({"b": [1, 4, 124]})) + assert isinstance(predictions, TabularDataset) + + def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: + configure_test_with_device(device) + with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).predict( + Table.from_dict({"a": [1]}), + ) + + def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], + ) + model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + model.predict( + Table.from_dict({"a": [1], "c": [2]}), + ) @pytest.mark.parametrize( ("input_conversion", "layers", "error_msg"), @@ -517,252 +572,303 @@ def test_should_be_pickleable(self, device: Device) -> None: @pytest.mark.parametrize("device", get_devices(), ids=get_devices_ids()) class TestRegressionModel: - def test_should_return_input_size(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - ) - - assert model.input_size == 1 - - @pytest.mark.parametrize( - "epoch_size", - [ - 0, - ], - ids=["epoch_size_out_of_bounds"], - ) - def test_should_raise_if_epoch_size_out_of_bounds(self, epoch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkRegressor( + class TestFit: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( InputConversionTable(), [ForwardLayer(neuron_count=1)], ).fit( Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - epoch_size=epoch_size, ) - @pytest.mark.parametrize( - "batch_size", - [ - 0, - ], - ids=["batch_size_out_of_bounds"], - ) - def test_should_raise_if_batch_size_out_of_bounds(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkRegressor( + assert model.input_size == 1 + + def test_should_raise_if_epoch_size_out_of_bounds(self, device: Device) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + epoch_size=invalid_epoch_size, + ) + + def test_should_raise_if_batch_size_out_of_bounds(self, device: Device) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + batch_size=invalid_batch_size, + ) + + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_fit_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkRegressor( InputConversionTable(), [ForwardLayer(neuron_count=1)], ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), batch_size=batch_size, ) + assert isinstance(fitted_model, NeuralNetworkRegressor) - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_fit_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - assert isinstance(fitted_model, NeuralNetworkRegressor) + def test_should_raise_when_fitting_with_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(1, 2))]) + with pytest.raises(FittingWithChoiceError): + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a")) - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - predictions = fitted_model.predict(Table.from_dict({"b": [5, 6, 7]})) - assert isinstance(predictions, TabularDataset) - def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): - NeuralNetworkRegressor( + def test_should_raise_if_is_fitted_is_set_correctly(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( InputConversionTable(), [ForwardLayer(neuron_count=1)], - ).predict( - Table.from_dict({"a": [1]}), ) - - def test_should_raise_if_is_fitted_is_set_correctly(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - assert not model.is_fitted - model = model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - ) - assert model.is_fitted - - def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - model.predict( - Table.from_dict({"a": [1], "c": [2]}), + assert not model.is_fitted + model = model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), ) + assert model.is_fitted - def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - trained_model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("b"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - trained_model.fit( - Table.from_dict({"k": [1, 0, 2], "l": [0, 15, 5]}).to_tabular_dataset("l"), + def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], ) - - @pytest.mark.parametrize( - ("table", "reason"), - [ - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", + trained_model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("b"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + trained_model.fit( + Table.from_dict({"k": [1, 0, 2], "l": [0, 15, 5]}).to_tabular_dataset("l"), + ) + + @pytest.mark.parametrize( + ("table", "reason"), + [ + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), - ), - ], - ids=[ - "missing value feature", - "non-numerical feature", - "missing value and non-numerical features", - "missing value target", - "non-numerical target", - ], - ) - def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=4), ForwardLayer(1)], - ) - with pytest.raises( - InvalidFitDataError, - match=reason, - ): - model.fit(table) - - # def test_should_raise_if_table_size_and_input_size_mismatch(self, device: Device) -> None: - # configure_test_with_device(device) - # model = NeuralNetworkRegressor( - # InputConversionTable(), - # [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - # ) - # with pytest.raises( - # InputSizeError, - # ): - # model.fit( - # Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5], "c": [3, 33, 333]}).to_tabular_dataset("a"), - # ) - - def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), + ), + ], + ids=[ + "missing value feature", + "non-numerical feature", + "missing value and non-numerical features", + "missing value target", + "non-numerical target", + ], ) + def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=4), ForwardLayer(1)], + ) + with pytest.raises( + InvalidFitDataError, + match=reason, + ): + model.fit(table) + + def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) - class Test: - self.was_called = False + class Test: + self.was_called = False - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True - def callback_was_called(self) -> bool: - return self.was_called + def callback_was_called(self) -> bool: + return self.was_called - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb) + obj = Test() + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_batch_completion=obj.cb) - assert obj.callback_was_called() is True + assert obj.callback_was_called() is True - def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) + def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) + + class Test: + self.was_called = False - class Test: - self.was_called = False + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + def callback_was_called(self) -> bool: + return self.was_called - def callback_was_called(self) -> bool: - return self.was_called + obj = Test() + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_epoch_completion=obj.cb) - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb) + assert obj.callback_was_called() is True - assert obj.callback_was_called() is True + class TestFitByExhaustiveSearch: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), + RegressorMetric.MEAN_SQUARED_ERROR, + ) + assert model.input_size == 1 + + def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, device: Device) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(Choice(1, 3))], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), + RegressorMetric.MEAN_ABSOLUTE_ERROR, + epoch_size=invalid_epoch_size, + ) + + def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, device: Device) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(1, 3))], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), + RegressorMetric.MEDIAN_ABSOLUTE_DEVIATION, + batch_size=invalid_batch_size, + ) + + def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(1)]) + with pytest.raises(FittingWithoutChoiceError): + model.fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), + RegressorMetric.COEFFICIENT_OF_DETERMINATION) + + def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) + assert not model.is_fitted + fitted_model = model.fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), + RegressorMetric.MEAN_ABSOLUTE_ERROR) + assert fitted_model.is_fitted + + def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatype(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) + fitted_model = model.fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), + RegressorMetric.MEAN_ABSOLUTE_ERROR) + assert isinstance(fitted_model, NeuralNetworkRegressor) + + class TestPredict: + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + predictions = fitted_model.predict(Table.from_dict({"b": [5, 6, 7]})) + assert isinstance(predictions, TabularDataset) + + def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: + configure_test_with_device(device) + with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).predict( + Table.from_dict({"a": [1]}), + ) + + def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) + model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + model.predict( + Table.from_dict({"a": [1], "c": [2]}), + ) @pytest.mark.parametrize( ("input_conversion", "layers", "error_msg"), From af9e9f7ecd4e5735d01434dec17733a2d8f2ece1 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 13 Jul 2024 19:03:29 +0200 Subject: [PATCH 23/38] linter fix --- src/safeds/ml/nn/_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 9fcedb2f8..fd286ffc6 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -373,7 +373,7 @@ def _get_models_for_all_choices(self) -> list[Self]: for combination in all_possible_layer_combinations: new_model = NeuralNetworkRegressor(input_conversion=self._input_conversion, layers=combination) models.append(new_model) - return models + return models # type: ignore[return-value] def predict(self, test_data: IPT) -> IFT: """ @@ -794,7 +794,7 @@ def _get_models_for_all_choices(self) -> list[Self]: for combination in all_possible_layer_combinations: new_model = NeuralNetworkClassifier(input_conversion=self._input_conversion, layers=combination) models.append(new_model) - return models + return models # type: ignore[return-value] def predict(self, test_data: IPT) -> IFT: """ From bf4802c97de8abc075ace6080e05d40a703fd846 Mon Sep 17 00:00:00 2001 From: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:05:10 +0000 Subject: [PATCH 24/38] style: apply automated linter fixes --- src/safeds/exceptions/__init__.py | 1 - src/safeds/ml/nn/_model.py | 4 +- .../safeds/ml/hyperparameters/test_choice.py | 1 - .../safeds/ml/nn/layers/test_forward_layer.py | 9 +-- tests/safeds/ml/nn/layers/test_gru_layer.py | 6 +- tests/safeds/ml/nn/layers/test_lstm_layer.py | 6 +- tests/safeds/ml/nn/test_model.py | 74 ++++++++++++------- 7 files changed, 54 insertions(+), 47 deletions(-) diff --git a/src/safeds/exceptions/__init__.py b/src/safeds/exceptions/__init__.py index f71b1854d..802afdb57 100644 --- a/src/safeds/exceptions/__init__.py +++ b/src/safeds/exceptions/__init__.py @@ -75,7 +75,6 @@ class OutOfBoundsError(SafeDsError): # ML exceptions "DatasetMissesDataError", "DatasetMissesFeaturesError", - "EmptyChoiceError", "FeatureDataMismatchError", "FittingWithChoiceError", diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index fd286ffc6..66c612ea9 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -373,7 +373,7 @@ def _get_models_for_all_choices(self) -> list[Self]: for combination in all_possible_layer_combinations: new_model = NeuralNetworkRegressor(input_conversion=self._input_conversion, layers=combination) models.append(new_model) - return models # type: ignore[return-value] + return models # type: ignore[return-value] def predict(self, test_data: IPT) -> IFT: """ @@ -794,7 +794,7 @@ def _get_models_for_all_choices(self) -> list[Self]: for combination in all_possible_layer_combinations: new_model = NeuralNetworkClassifier(input_conversion=self._input_conversion, layers=combination) models.append(new_model) - return models # type: ignore[return-value] + return models # type: ignore[return-value] def predict(self, test_data: IPT) -> IFT: """ diff --git a/tests/safeds/ml/hyperparameters/test_choice.py b/tests/safeds/ml/hyperparameters/test_choice.py index 758f8a081..ce4213044 100644 --- a/tests/safeds/ml/hyperparameters/test_choice.py +++ b/tests/safeds/ml/hyperparameters/test_choice.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING import pytest - from safeds.exceptions import EmptyChoiceError from safeds.ml.hyperparameters import Choice diff --git a/tests/safeds/ml/nn/layers/test_forward_layer.py b/tests/safeds/ml/nn/layers/test_forward_layer.py index 9b2f8420a..e396e7308 100644 --- a/tests/safeds/ml/nn/layers/test_forward_layer.py +++ b/tests/safeds/ml/nn/layers/test_forward_layer.py @@ -78,11 +78,7 @@ def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int @pytest.mark.parametrize( "output_size", - [ - 1, - 20, - Choice(1, 20) - ], + [1, 20, Choice(1, 20)], ids=["one", "twenty", "choice"], ) def test_should_return_output_size(output_size: int | Choice[int]) -> None: @@ -117,7 +113,8 @@ def test_should_raise_if_activation_function_not_set() -> None: ForwardLayer(neuron_count=1), False, ), - ( ForwardLayer(neuron_count=Choice(2)), + ( + ForwardLayer(neuron_count=Choice(2)), ForwardLayer(neuron_count=Choice(2)), True, ), diff --git a/tests/safeds/ml/nn/layers/test_gru_layer.py b/tests/safeds/ml/nn/layers/test_gru_layer.py index 137a58174..409b5880e 100644 --- a/tests/safeds/ml/nn/layers/test_gru_layer.py +++ b/tests/safeds/ml/nn/layers/test_gru_layer.py @@ -66,11 +66,7 @@ def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int @pytest.mark.parametrize( "output_size", - [ - 1, - 20, - Choice(1, 20) - ], + [1, 20, Choice(1, 20)], ids=["one", "twenty", "choice"], ) def test_should_raise_if_output_size_doesnt_match(output_size: int | Choice[int]) -> None: diff --git a/tests/safeds/ml/nn/layers/test_lstm_layer.py b/tests/safeds/ml/nn/layers/test_lstm_layer.py index 996f92705..fc04c6eac 100644 --- a/tests/safeds/ml/nn/layers/test_lstm_layer.py +++ b/tests/safeds/ml/nn/layers/test_lstm_layer.py @@ -78,11 +78,7 @@ def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int @pytest.mark.parametrize( "output_size", - [ - 1, - 20, - Choice(1, 20) - ], + [1, 20, Choice(1, 20)], ids=["one", "twenty", "choice"], ) def test_should_raise_if_output_size_doesnt_match(output_size: int | Choice[int]) -> None: diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index a1cc21b7e..e20edc692 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -7,10 +7,12 @@ from safeds.data.tabular.containers import Table from safeds.exceptions import ( FeatureDataMismatchError, + FittingWithChoiceError, + FittingWithoutChoiceError, InvalidFitDataError, InvalidModelStructureError, ModelNotFittedError, - OutOfBoundsError, FittingWithChoiceError, FittingWithoutChoiceError, + OutOfBoundsError, ) from safeds.ml.hyperparameters import Choice from safeds.ml.metrics import ClassifierMetric, RegressorMetric @@ -162,7 +164,8 @@ def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: ( Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), re.escape( - "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']", + ), ), ( Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), @@ -179,7 +182,8 @@ def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: ( Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), re.escape( - "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']", + ), ), ], ids=[ @@ -220,8 +224,9 @@ def callback_was_called(self) -> bool: return self.was_called obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - callback_on_batch_completion=obj.cb) + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb, + ) assert obj.callback_was_called() is True @@ -243,8 +248,9 @@ def callback_was_called(self) -> bool: return self.was_called obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - callback_on_epoch_completion=obj.cb) + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb, + ) assert obj.callback_was_called() is True @@ -260,8 +266,9 @@ def test_should_return_input_size(self, device: Device) -> None: ) assert model.input_size == 1 - def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, - device: Device) -> None: + def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, device: Device, + ) -> None: invalid_epoch_size = 0 configure_test_with_device(device) with pytest.raises(OutOfBoundsError): @@ -274,8 +281,9 @@ def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_sea epoch_size=invalid_epoch_size, ) - def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, - device: Device) -> None: + def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, device: Device, + ) -> None: invalid_batch_size = 0 configure_test_with_device(device) with pytest.raises(OutOfBoundsError): @@ -293,8 +301,8 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(1)]) with pytest.raises(FittingWithoutChoiceError): model.fit_by_exhaustive_search( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY) + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), ClassifierMetric.ACCURACY, + ) def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> None: configure_test_with_device(device) @@ -302,7 +310,8 @@ def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> assert not model.is_fitted fitted_model = model.fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY) + ClassifierMetric.ACCURACY, + ) assert fitted_model.is_fitted def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatype(self, device: Device) -> None: @@ -310,7 +319,8 @@ def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatyp model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) fitted_model = model.fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY) + ClassifierMetric.ACCURACY, + ) assert isinstance(fitted_model, NeuralNetworkClassifier) class TestPredict: @@ -633,7 +643,6 @@ def test_should_raise_when_fitting_with_choice(self, device: Device) -> None: with pytest.raises(FittingWithChoiceError): model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a")) - def test_should_raise_if_is_fitted_is_set_correctly(self, device: Device) -> None: configure_test_with_device(device) model = NeuralNetworkRegressor( @@ -673,7 +682,8 @@ def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: ( Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), re.escape( - "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']", + ), ), ( Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), @@ -690,7 +700,8 @@ def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: ( Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), re.escape( - "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']", + ), ), ], ids=[ @@ -731,8 +742,9 @@ def callback_was_called(self) -> bool: return self.was_called obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - callback_on_batch_completion=obj.cb) + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb, + ) assert obj.callback_was_called() is True @@ -754,8 +766,9 @@ def callback_was_called(self) -> bool: return self.was_called obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - callback_on_epoch_completion=obj.cb) + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb, + ) assert obj.callback_was_called() is True @@ -771,7 +784,9 @@ def test_should_return_input_size(self, device: Device) -> None: ) assert model.input_size == 1 - def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, device: Device) -> None: + def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, device: Device, + ) -> None: invalid_epoch_size = 0 configure_test_with_device(device) with pytest.raises(OutOfBoundsError): @@ -784,7 +799,9 @@ def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_sea epoch_size=invalid_epoch_size, ) - def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search(self, device: Device) -> None: + def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, device: Device, + ) -> None: invalid_batch_size = 0 configure_test_with_device(device) with pytest.raises(OutOfBoundsError): @@ -803,7 +820,8 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev with pytest.raises(FittingWithoutChoiceError): model.fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), - RegressorMetric.COEFFICIENT_OF_DETERMINATION) + RegressorMetric.COEFFICIENT_OF_DETERMINATION, + ) def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> None: configure_test_with_device(device) @@ -811,7 +829,8 @@ def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> assert not model.is_fitted fitted_model = model.fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_ABSOLUTE_ERROR) + RegressorMetric.MEAN_ABSOLUTE_ERROR, + ) assert fitted_model.is_fitted def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatype(self, device: Device) -> None: @@ -819,7 +838,8 @@ def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatyp model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) fitted_model = model.fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_ABSOLUTE_ERROR) + RegressorMetric.MEAN_ABSOLUTE_ERROR, + ) assert isinstance(fitted_model, NeuralNetworkRegressor) class TestPredict: From da408be7b9fd192bacf39fd2fa87c57d08f4cfd9 Mon Sep 17 00:00:00 2001 From: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:06:39 +0000 Subject: [PATCH 25/38] style: apply automated linter fixes --- tests/safeds/ml/nn/test_model.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index e20edc692..133bbfe5b 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -225,7 +225,8 @@ def callback_was_called(self) -> bool: obj = Test() model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb, + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_batch_completion=obj.cb, ) assert obj.callback_was_called() is True @@ -249,7 +250,8 @@ def callback_was_called(self) -> bool: obj = Test() model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb, + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_epoch_completion=obj.cb, ) assert obj.callback_was_called() is True @@ -267,7 +269,8 @@ def test_should_return_input_size(self, device: Device) -> None: assert model.input_size == 1 def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search( - self, device: Device, + self, + device: Device, ) -> None: invalid_epoch_size = 0 configure_test_with_device(device) @@ -282,7 +285,8 @@ def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_sea ) def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search( - self, device: Device, + self, + device: Device, ) -> None: invalid_batch_size = 0 configure_test_with_device(device) @@ -301,7 +305,8 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(1)]) with pytest.raises(FittingWithoutChoiceError): model.fit_by_exhaustive_search( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), ClassifierMetric.ACCURACY, + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), + ClassifierMetric.ACCURACY, ) def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> None: @@ -743,7 +748,8 @@ def callback_was_called(self) -> bool: obj = Test() model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb, + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_batch_completion=obj.cb, ) assert obj.callback_was_called() is True @@ -767,7 +773,8 @@ def callback_was_called(self) -> bool: obj = Test() model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb, + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_epoch_completion=obj.cb, ) assert obj.callback_was_called() is True @@ -785,7 +792,8 @@ def test_should_return_input_size(self, device: Device) -> None: assert model.input_size == 1 def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search( - self, device: Device, + self, + device: Device, ) -> None: invalid_epoch_size = 0 configure_test_with_device(device) @@ -800,7 +808,8 @@ def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_sea ) def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search( - self, device: Device, + self, + device: Device, ) -> None: invalid_batch_size = 0 configure_test_with_device(device) From 7abb1e9527b180383ed4d86f07b3d44175850dcc Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 13 Jul 2024 19:13:30 +0200 Subject: [PATCH 26/38] set context of processpoolexecutor --- src/safeds/ml/nn/_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 66c612ea9..33d887597 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import multiprocessing as mp from concurrent.futures import ALL_COMPLETED, ProcessPoolExecutor, wait from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar @@ -291,7 +292,7 @@ def fit_by_exhaustive_search( list_of_models = self._get_models_for_all_choices() list_of_fitted_models: list[Self] = [] - with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: + with ProcessPoolExecutor(max_workers=len(list_of_models), mp_context=mp.get_context("spawn")) as executor: futures = [] for model in list_of_models: futures.append(executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) @@ -712,7 +713,7 @@ def fit_by_exhaustive_search( list_of_models = self._get_models_for_all_choices() list_of_fitted_models: list[Self] = [] - with ProcessPoolExecutor(max_workers=len(list_of_models)) as executor: + with ProcessPoolExecutor(max_workers=len(list_of_models), mp_context=mp.get_context("spawn")) as executor: futures = [] for model in list_of_models: futures.append(executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) From 0e6e53af12ff8e08ea8012241b6533fcdb60b102 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 14 Jul 2024 00:14:07 +0200 Subject: [PATCH 27/38] add tests for all metrics --- tests/safeds/ml/nn/test_model.py | 64 ++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index 133bbfe5b..709684879 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -1,5 +1,6 @@ import pickle import re +from typing import Any import pytest from safeds.data.image.typing import ImageSize @@ -309,23 +310,38 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev ClassifierMetric.ACCURACY, ) - def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> None: + @pytest.mark.parametrize( + ["metric", "positive_class"], + [ + ( + ClassifierMetric.ACCURACY, + None, + ), + ( + ClassifierMetric.PRECISION, + 0, + ), + ( + ClassifierMetric.F1_SCORE, + 0, + ), + ( + ClassifierMetric.RECALL, + 0, + ), + ], + ids=["accuracy", "precision", "f1score", "recall"] + ) + def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type(self, metric: ClassifierMetric, positive_class: Any, device: Device) -> None: configure_test_with_device(device) model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) assert not model.is_fitted fitted_model = model.fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY, + optimization_metric=metric, + positive_class=positive_class, ) assert fitted_model.is_fitted - - def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatype(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) - fitted_model = model.fit_by_exhaustive_search( - Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY, - ) assert isinstance(fitted_model, NeuralNetworkClassifier) class TestPredict: @@ -803,7 +819,7 @@ def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_sea [ForwardLayer(Choice(1, 3))], ).fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_ABSOLUTE_ERROR, + RegressorMetric.MEAN_SQUARED_ERROR, epoch_size=invalid_epoch_size, ) @@ -819,7 +835,7 @@ def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_sea [ForwardLayer(neuron_count=Choice(1, 3))], ).fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), - RegressorMetric.MEDIAN_ABSOLUTE_DEVIATION, + RegressorMetric.MEAN_SQUARED_ERROR, batch_size=invalid_batch_size, ) @@ -829,26 +845,28 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev with pytest.raises(FittingWithoutChoiceError): model.fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), - RegressorMetric.COEFFICIENT_OF_DETERMINATION, + RegressorMetric.MEAN_SQUARED_ERROR, ) - def test_should_assert_that_is_fitted_is_set_correctly(self, device: Device) -> None: + @pytest.mark.parametrize( + "metric", + [ + RegressorMetric.MEAN_SQUARED_ERROR, + RegressorMetric.MEAN_ABSOLUTE_ERROR, + RegressorMetric.MEDIAN_ABSOLUTE_DEVIATION, + RegressorMetric.COEFFICIENT_OF_DETERMINATION, + ], + ids=["mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination"], + ) + def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type(self, metric: RegressorMetric, device: Device) -> None: configure_test_with_device(device) model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) assert not model.is_fitted fitted_model = model.fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_ABSOLUTE_ERROR, + optimization_metric=metric, ) assert fitted_model.is_fitted - - def test_should_raise_if_fit_by_exhaustive_search_function_returns_wrong_datatype(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) - fitted_model = model.fit_by_exhaustive_search( - Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_ABSOLUTE_ERROR, - ) assert isinstance(fitted_model, NeuralNetworkRegressor) class TestPredict: From 5f51cd7985972e164ddff9e1b1eaffe22ba71e7f Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 14 Jul 2024 00:29:46 +0200 Subject: [PATCH 28/38] linter fix --- tests/safeds/ml/nn/test_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index 709684879..97a9481d4 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -311,7 +311,7 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev ) @pytest.mark.parametrize( - ["metric", "positive_class"], + ("metric", "positive_class"), [ ( ClassifierMetric.ACCURACY, From 02c0f0590493145e32019797b70df9ba703f1fb8 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 14 Jul 2024 00:37:35 +0200 Subject: [PATCH 29/38] codecov --- src/safeds/ml/nn/_model.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 33d887597..aab5efdfc 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -332,23 +332,23 @@ def fit_by_exhaustive_search( case "mean_squared_error": error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "mean_absolute_error": error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "median_absolute_deviation": error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "coefficient_of_determination": error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover assert best_model is not None # just for linter best_model._is_fitted = True return best_model @@ -746,30 +746,30 @@ def fit_by_exhaustive_search( best_metric_value = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] case "recall": best_metric_value = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] - case "f1score": + case "f1_score": best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] else: match optimization_metric.value: case "accuracy": error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "precision": error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "recall": error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model - case "f1score": + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + case "f1_score": error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model - best_metric_value = error_of_fitted_model + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover assert best_model is not None # just for linter best_model._is_fitted = True return best_model From 7ff98d2189a0a0ebfd19b9fa3d6cbae0f6a67ded Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 14 Jul 2024 00:41:35 +0200 Subject: [PATCH 30/38] add missing image test, dont know why it went missing --- tests/safeds/ml/nn/test_model.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index 97a9481d4..e8a0944be 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -567,6 +567,11 @@ def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: [FlattenLayer()], r"A NeuralNetworkClassifier cannot be used with a InputConversionImage that uses a VariableImageSize.", ), + ( + InputConversionImageToImage(VariableImageSize(1, 1, 1)), + [FlattenLayer()], + r"A NeuralNetworkClassifier cannot be used with images as output.", + ), ], ) def test_should_raise_if_model_has_invalid_structure( From 84033cf9ace019c4d1093be30ddba71fb17ed8c8 Mon Sep 17 00:00:00 2001 From: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> Date: Sat, 13 Jul 2024 22:43:10 +0000 Subject: [PATCH 31/38] style: apply automated linter fixes --- src/safeds/ml/nn/_model.py | 32 ++++++++++++++++---------------- tests/safeds/ml/nn/test_model.py | 17 +++++++++++++---- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index aab5efdfc..599808109 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -332,23 +332,23 @@ def fit_by_exhaustive_search( case "mean_squared_error": error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "mean_absolute_error": error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "median_absolute_deviation": error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "coefficient_of_determination": error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover assert best_model is not None # just for linter best_model._is_fitted = True return best_model @@ -753,23 +753,23 @@ def fit_by_exhaustive_search( case "accuracy": error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "precision": error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "recall": error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover case "f1_score": error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: - best_model = fitted_model # pragma: no cover - best_metric_value = error_of_fitted_model # pragma: no cover + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover assert best_model is not None # just for linter best_model._is_fitted = True return best_model diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index e8a0944be..4ae3c5908 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -330,9 +330,11 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev 0, ), ], - ids=["accuracy", "precision", "f1score", "recall"] + ids=["accuracy", "precision", "f1score", "recall"], ) - def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type(self, metric: ClassifierMetric, positive_class: Any, device: Device) -> None: + def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( + self, metric: ClassifierMetric, positive_class: Any, device: Device, + ) -> None: configure_test_with_device(device) model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) assert not model.is_fitted @@ -861,9 +863,16 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev RegressorMetric.MEDIAN_ABSOLUTE_DEVIATION, RegressorMetric.COEFFICIENT_OF_DETERMINATION, ], - ids=["mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination"], + ids=[ + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", + ], ) - def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type(self, metric: RegressorMetric, device: Device) -> None: + def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( + self, metric: RegressorMetric, device: Device, + ) -> None: configure_test_with_device(device) model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) assert not model.is_fitted From 3d0415eb8517dc51b4029f33fd8378c900a3acc6 Mon Sep 17 00:00:00 2001 From: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> Date: Sat, 13 Jul 2024 22:44:40 +0000 Subject: [PATCH 32/38] style: apply automated linter fixes --- tests/safeds/ml/nn/test_model.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index 4ae3c5908..a7f221cc5 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -333,7 +333,10 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev ids=["accuracy", "precision", "f1score", "recall"], ) def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( - self, metric: ClassifierMetric, positive_class: Any, device: Device, + self, + metric: ClassifierMetric, + positive_class: Any, + device: Device, ) -> None: configure_test_with_device(device) model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) @@ -871,7 +874,9 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev ], ) def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( - self, metric: RegressorMetric, device: Device, + self, + metric: RegressorMetric, + device: Device, ) -> None: configure_test_with_device(device) model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) From 3985dfbd63dd8e56e450e709c71f37b98a0eb930 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 15 Jul 2024 13:00:46 +0200 Subject: [PATCH 33/38] add docs for fit_by_exhaustive_search --- src/safeds/ml/nn/_model.py | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 599808109..678dd3da2 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -275,7 +275,36 @@ def fit_by_exhaustive_search( batch_size: int = 1, learning_rate: float = 0.001, ) -> Self: + """ + Use the hyperparameter choices to create multiple models and fit them. + + **Note:** This model is not modified. + Parameters + ---------- + train_data: + The data the network should be trained on. + optimization_metric: + The metric that should be used for determining the performance of a model. + epoch_size: + The number of times the training cycle should be done. + batch_size: + The size of data batches that should be loaded at one time. + learning_rate: + The learning rate of the neural network. + + Returns + ------- + best_model: + The model that performed the best out of all possible models given the Choices of hyperparameters. + + Raises + ------ + FittingWithoutChoiceError + When calling this method on a model without hyperparameter choices. + LearningError + If the training data contains invalid values or if the training failed. Currently raised, when calling this on RNNs or CNNs as well. + """ _init_default_device() if not self._contains_choices(): @@ -696,7 +725,38 @@ def fit_by_exhaustive_search( batch_size: int = 1, learning_rate: float = 0.001, ) -> Self: + """ + Use the hyperparameter choices to create multiple models and fit them. + + **Note:** This model is not modified. + Parameters + ---------- + train_data: + The data the network should be trained on. + optimization_metric: + The metric that should be used for determining the performance of a model. + positive_class: + The class to be considered positive. Only needs to be provided when choosing precision, recall or f1_score as the optimization metric. + epoch_size: + The number of times the training cycle should be done. + batch_size: + The size of data batches that should be loaded at one time. + learning_rate: + The learning rate of the neural network. + + Returns + ------- + best_model: + The model that performed the best out of all possible models given the Choices of hyperparameters. + + Raises + ------ + FittingWithoutChoiceError + When calling this method on a model without hyperparameter choices. + LearningError + If the training data contains invalid values or if the training failed. Currently raised, when calling this on RNNs or CNNs as well. + """ _init_default_device() if not self._contains_choices(): From 6dcd34f6132c522a5d4ec74be0db0384e07c561c Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 15 Jul 2024 13:22:28 +0200 Subject: [PATCH 34/38] change Metric Enums to Literals --- src/safeds/ml/nn/_model.py | 14 +++++------ tests/safeds/ml/nn/test_model.py | 40 ++++++++++++++++---------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 678dd3da2..5d4a8baa4 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -3,7 +3,7 @@ import copy import multiprocessing as mp from concurrent.futures import ALL_COMPLETED, ProcessPoolExecutor, wait -from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar +from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar, Literal from safeds._config import _init_default_device from safeds._validation import _check_bounds, _ClosedBound @@ -270,7 +270,7 @@ def fit( def fit_by_exhaustive_search( self, train_data: IFT, - optimization_metric: RegressorMetric, + optimization_metric: Literal["mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination"], epoch_size: int = 25, batch_size: int = 1, learning_rate: float = 0.001, @@ -347,7 +347,7 @@ def fit_by_exhaustive_search( for fitted_model in list_of_fitted_models: if best_model is None: best_model = fitted_model - match optimization_metric.value: + match optimization_metric: case "mean_squared_error": best_metric_value = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] case "mean_absolute_error": @@ -357,7 +357,7 @@ def fit_by_exhaustive_search( case "coefficient_of_determination": best_metric_value = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] else: - match optimization_metric.value: + match optimization_metric: case "mean_squared_error": error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model < best_metric_value: @@ -719,7 +719,7 @@ def fit( def fit_by_exhaustive_search( self, train_data: IFT, - optimization_metric: ClassifierMetric, + optimization_metric: Literal["accuracy", "precision", "recall", "f1_score"], positive_class: Any = None, epoch_size: int = 25, batch_size: int = 1, @@ -799,7 +799,7 @@ def fit_by_exhaustive_search( for fitted_model in list_of_fitted_models: if best_model is None: best_model = fitted_model - match optimization_metric.value: + match optimization_metric: case "accuracy": best_metric_value = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] case "precision": @@ -809,7 +809,7 @@ def fit_by_exhaustive_search( case "f1_score": best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] else: - match optimization_metric.value: + match optimization_metric: case "accuracy": error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] if error_of_fitted_model > best_metric_value: diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index a7f221cc5..ce264b0e4 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -1,6 +1,6 @@ import pickle import re -from typing import Any +from typing import Any, Literal import pytest from safeds.data.image.typing import ImageSize @@ -265,7 +265,7 @@ def test_should_return_input_size(self, device: Device) -> None: [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], ).fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY, + "accuracy", ) assert model.input_size == 1 @@ -281,7 +281,7 @@ def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_sea [ForwardLayer(Choice(2, 4)), ForwardLayer(1)], ).fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY, + "accuracy", epoch_size=invalid_epoch_size, ) @@ -297,7 +297,7 @@ def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_sea [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], ).fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY, + "accuracy", batch_size=invalid_batch_size, ) @@ -307,34 +307,34 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev with pytest.raises(FittingWithoutChoiceError): model.fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), - ClassifierMetric.ACCURACY, + "accuracy", ) @pytest.mark.parametrize( ("metric", "positive_class"), [ ( - ClassifierMetric.ACCURACY, + "accuracy", None, ), ( - ClassifierMetric.PRECISION, + "precision", 0, ), ( - ClassifierMetric.F1_SCORE, + "recall", 0, ), ( - ClassifierMetric.RECALL, + "f1_score", 0, ), ], - ids=["accuracy", "precision", "f1score", "recall"], + ids=["accuracy", "precision", "recall", "f1_score"], ) def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( self, - metric: ClassifierMetric, + metric: Literal["accuracy", "precision", "recall", "f1_score"], positive_class: Any, device: Device, ) -> None: @@ -813,7 +813,7 @@ def test_should_return_input_size(self, device: Device) -> None: [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], ).fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_SQUARED_ERROR, + "mean_squared_error" ) assert model.input_size == 1 @@ -829,7 +829,7 @@ def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_sea [ForwardLayer(Choice(1, 3))], ).fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_SQUARED_ERROR, + "mean_squared_error", epoch_size=invalid_epoch_size, ) @@ -845,7 +845,7 @@ def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_sea [ForwardLayer(neuron_count=Choice(1, 3))], ).fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_SQUARED_ERROR, + "mean_squared_error", batch_size=invalid_batch_size, ) @@ -855,16 +855,16 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev with pytest.raises(FittingWithoutChoiceError): model.fit_by_exhaustive_search( Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), - RegressorMetric.MEAN_SQUARED_ERROR, + "mean_squared_error", ) @pytest.mark.parametrize( "metric", [ - RegressorMetric.MEAN_SQUARED_ERROR, - RegressorMetric.MEAN_ABSOLUTE_ERROR, - RegressorMetric.MEDIAN_ABSOLUTE_DEVIATION, - RegressorMetric.COEFFICIENT_OF_DETERMINATION, + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", ], ids=[ "mean_squared_error", @@ -875,7 +875,7 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev ) def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( self, - metric: RegressorMetric, + metric: Literal["mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination"], device: Device, ) -> None: configure_test_with_device(device) From d00c673f4117bb3dcd5594e19f2db1341e8dedb9 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 15 Jul 2024 13:24:11 +0200 Subject: [PATCH 35/38] Delete Metric Enums --- src/safeds/ml/metrics/__init__.py | 7 ------- src/safeds/ml/metrics/_classifier_metric.py | 10 ---------- src/safeds/ml/metrics/_regressor_metric.py | 10 ---------- 3 files changed, 27 deletions(-) delete mode 100644 src/safeds/ml/metrics/_classifier_metric.py delete mode 100644 src/safeds/ml/metrics/_regressor_metric.py diff --git a/src/safeds/ml/metrics/__init__.py b/src/safeds/ml/metrics/__init__.py index dc0a93657..aa465cff0 100644 --- a/src/safeds/ml/metrics/__init__.py +++ b/src/safeds/ml/metrics/__init__.py @@ -4,9 +4,6 @@ import apipkg -from ._classifier_metric import ClassifierMetric -from ._regressor_metric import RegressorMetric - if TYPE_CHECKING: from ._classification_metrics import ClassificationMetrics from ._regression_metrics import RegressionMetrics @@ -15,15 +12,11 @@ __name__, { "ClassificationMetrics": "._classification_metrics:ClassificationMetrics", - "ClassifierMetric": "._classifier_metric:ClassifierMetric", "RegressionMetrics": "._regression_metrics:RegressionMetrics", - "RegressorMetric": "._regressor_metric:RegressorMetric", }, ) __all__ = [ "ClassificationMetrics", - "ClassifierMetric", "RegressionMetrics", - "RegressorMetric", ] diff --git a/src/safeds/ml/metrics/_classifier_metric.py b/src/safeds/ml/metrics/_classifier_metric.py deleted file mode 100644 index 4f69c2607..000000000 --- a/src/safeds/ml/metrics/_classifier_metric.py +++ /dev/null @@ -1,10 +0,0 @@ -from enum import Enum - - -class ClassifierMetric(Enum): - """An Enum of possible Metrics for a Classifier.""" - - ACCURACY = "accuracy" - PRECISION = "precision" - RECALL = "recall" - F1_SCORE = "f1_score" diff --git a/src/safeds/ml/metrics/_regressor_metric.py b/src/safeds/ml/metrics/_regressor_metric.py deleted file mode 100644 index 421ce4b08..000000000 --- a/src/safeds/ml/metrics/_regressor_metric.py +++ /dev/null @@ -1,10 +0,0 @@ -from enum import Enum - - -class RegressorMetric(Enum): - """An Enum of possible Metrics for a Regressor.""" - - MEAN_SQUARED_ERROR = "mean_squared_error" - MEAN_ABSOLUTE_ERROR = "mean_absolute_error" - MEDIAN_ABSOLUTE_DEVIATION = "median_absolute_deviation" - COEFFICIENT_OF_DETERMINATION = "coefficient_of_determination" From 99b258f23347416a3296a8f3081620fb5d725bdf Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 15 Jul 2024 13:28:29 +0200 Subject: [PATCH 36/38] remove imports of deleted enums --- src/safeds/ml/nn/_model.py | 2 +- tests/safeds/ml/nn/test_model.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 5d4a8baa4..c55cafcff 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -20,7 +20,7 @@ LearningError, ModelNotFittedError, ) -from safeds.ml.metrics import ClassificationMetrics, ClassifierMetric, RegressionMetrics, RegressorMetric +from safeds.ml.metrics import ClassificationMetrics, RegressionMetrics from safeds.ml.nn.converters import ( InputConversionImageToColumn, InputConversionImageToImage, diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index ce264b0e4..a0ae83257 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -16,7 +16,6 @@ OutOfBoundsError, ) from safeds.ml.hyperparameters import Choice -from safeds.ml.metrics import ClassifierMetric, RegressorMetric from safeds.ml.nn import ( NeuralNetworkClassifier, NeuralNetworkRegressor, From 1048d2c528e9f76c833b9f64798a981136093843 Mon Sep 17 00:00:00 2001 From: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> Date: Mon, 15 Jul 2024 11:30:06 +0000 Subject: [PATCH 37/38] style: apply automated linter fixes --- src/safeds/ml/nn/_model.py | 6 ++++-- tests/safeds/ml/nn/test_model.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index c55cafcff..c11ddc66a 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -3,7 +3,7 @@ import copy import multiprocessing as mp from concurrent.futures import ALL_COMPLETED, ProcessPoolExecutor, wait -from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar, Literal +from typing import TYPE_CHECKING, Any, Generic, Literal, Self, TypeVar from safeds._config import _init_default_device from safeds._validation import _check_bounds, _ClosedBound @@ -270,7 +270,9 @@ def fit( def fit_by_exhaustive_search( self, train_data: IFT, - optimization_metric: Literal["mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination"], + optimization_metric: Literal[ + "mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination", + ], epoch_size: int = 25, batch_size: int = 1, learning_rate: float = 0.001, diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index a0ae83257..86db213ae 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -812,7 +812,7 @@ def test_should_return_input_size(self, device: Device) -> None: [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], ).fit_by_exhaustive_search( Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), - "mean_squared_error" + "mean_squared_error", ) assert model.input_size == 1 @@ -874,7 +874,9 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev ) def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( self, - metric: Literal["mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination"], + metric: Literal[ + "mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination", + ], device: Device, ) -> None: configure_test_with_device(device) From 7903fade49a35a7b80c7e49ff31f63e3460a4624 Mon Sep 17 00:00:00 2001 From: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> Date: Mon, 15 Jul 2024 11:31:40 +0000 Subject: [PATCH 38/38] style: apply automated linter fixes --- src/safeds/ml/nn/_model.py | 5 ++++- tests/safeds/ml/nn/test_model.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index c11ddc66a..57e9d10a6 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -271,7 +271,10 @@ def fit_by_exhaustive_search( self, train_data: IFT, optimization_metric: Literal[ - "mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination", + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", ], epoch_size: int = 25, batch_size: int = 1, diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index 86db213ae..fc7b87a89 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -875,7 +875,10 @@ def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, dev def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( self, metric: Literal[ - "mean_squared_error", "mean_absolute_error", "median_absolute_deviation", "coefficient_of_determination", + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", ], device: Device, ) -> None: