diff --git a/docs/source/refs.bib b/docs/source/refs.bib index cac8159e..62ef87f0 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -175,3 +175,22 @@ @article{zou2005 journal = {Journal of the Royal Statistical Society Series B}, doi = {10.1111/j.1467-9868.2005.00527.x} } + +@article{ruder2017, + title = {An overview of gradient descent optimization algorithms}, + url = {http://arxiv.org/abs/1609.04747}, + journaltitle = {{arXiv}:1609.04747 [cs]}, + author = {Ruder, Sebastian}, + urldate = {2021-12-09}, + date = {2017-06-15}, + langid = {english}, + eprinttype = {arxiv}, + eprint = {1609.04747}, +} + +@article{defazio2014, + title = {{SAGA}: A Fast Incremental Gradient Method With Support for Non-Strongly Convex Composite Objectives}, + pages = {15}, + author = {Defazio, Aaron and Bach, Francis and Lacoste-Julien, Simon}, + langid = {english}, +} diff --git a/modopt/opt/algorithms/__init__.py b/modopt/opt/algorithms/__init__.py new file mode 100644 index 00000000..aa645e47 --- /dev/null +++ b/modopt/opt/algorithms/__init__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +r"""OPTIMISATION ALGOTITHMS. + +This module contains class implementations of various optimisation algoritms. + +:Authors: Samuel Farrens , + Zaccharie Ramzi + +:Notes: + +Input classes must have the following properties: + + * **Gradient Operators** + + Must have the following methods: + + * ``get_grad()`` - calculate the gradient + + Must have the following variables: + + * ``grad`` - the gradient + + * **Linear Operators** + + Must have the following methods: + + * ``op()`` - operator + * ``adj_op()`` - adjoint operator + + * **Proximity Operators** + + Must have the following methods: + + * ``op()`` - operator + +The following notation is used to implement the algorithms: + + * x_old is used in place of :math:`x_{n}`. + * x_new is used in place of :math:`x_{n+1}`. + * x_prox is used in place of :math:`\tilde{x}_{n+1}`. + * x_temp is used for intermediate operations. + +""" + +from modopt.opt.algorithms.base import SetUp +from modopt.opt.algorithms.forward_backward import (FISTA, POGM, + ForwardBackward, + GenForwardBackward) +from modopt.opt.algorithms.gradient_descent import (AdaGenericGradOpt, + ADAMGradOpt, + GenericGradOpt, + MomentumGradOpt, + RMSpropGradOpt, + SAGAOptGradOpt, + VanillaGenericGradOpt) +from modopt.opt.algorithms.primal_dual import Condat diff --git a/modopt/opt/algorithms/base.py b/modopt/opt/algorithms/base.py new file mode 100644 index 00000000..9191642e --- /dev/null +++ b/modopt/opt/algorithms/base.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- +"""Base SetUp for optimisation algorithms.""" + +from inspect import getmro + +import numpy as np +from progressbar import ProgressBar + +from modopt.base import backend +from modopt.base.observable import MetricObserver, Observable +from modopt.interface.errors import warn + + +class SetUp(Observable): + r"""Algorithm Set-Up. + + This class contains methods for checking the set-up of an optimisation + algotithm and produces warnings if they do not comply. + + Parameters + ---------- + metric_call_period : int, optional + Metric call period (default is ``5``) + metrics : dict, optional + Metrics to be used (default is ``\{\}``) + verbose : bool, optional + Option for verbose output (default is ``False``) + progress : bool, optional + Option to display progress bar (default is ``True``) + step_size : int, optional + Generic step size parameter to override default algorithm + parameter name (`e.g.` `step_size` will override the value set for + `beta_param` in `ForwardBackward`) + use_gpu : bool, optional + Option to use available GPU + + See Also + -------- + modopt.base.observable.MetricObserver : + Definition of Metrics. + """ + + def __init__( + self, + metric_call_period=5, + metrics=None, + verbose=False, + progress=True, + step_size=None, + compute_backend='numpy', + **dummy_kwargs, + ): + self.idx = 0 + self.converge = False + self.verbose = verbose + self.progress = progress + self.metrics = metrics + self.step_size = step_size + self._op_parents = ( + 'GradParent', + 'ProximityParent', + 'LinearParent', + 'costObj', + ) + + self.metric_call_period = metric_call_period + + # Declaration of observers for metrics + super().__init__(['cv_metrics']) + + for name, dic in self.metrics.items(): + observer = MetricObserver( + name, + dic['metric'], + dic['mapping'], + dic['cst_kwargs'], + dic['early_stopping'], + ) + self.add_observer('cv_metrics', observer) + + xp, compute_backend = backend.get_backend(compute_backend) + self.xp = xp + self.compute_backend = compute_backend + + @property + def metrics(self): + """Metrics.""" + return self._metrics + + @metrics.setter + def metrics(self, metrics): + + if isinstance(metrics, type(None)): + self._metrics = {} + elif isinstance(metrics, dict): + self._metrics = metrics + else: + raise TypeError( + 'Metrics must be a dictionary, not {0}.'.format(type(metrics)), + ) + + def any_convergence_flag(self): + """Check convergence flag. + + Return if any matrices values matched the convergence criteria. + + Returns + ------- + bool + True if any convergence criteria met + + """ + return any( + obs.converge_flag for obs in self._observers['cv_metrics'] + ) + + def copy_data(self, input_data): + """Copy Data. + + Set directive for copying data. + + Parameters + ---------- + input_data : numpy.ndarray + Input data + + Returns + ------- + numpy.ndarray + Copy of input data + + """ + return self.xp.copy(backend.change_backend( + input_data, + self.compute_backend, + )) + + def _check_input_data(self, input_data): + """Check input data type. + + This method checks if the input data is a numpy array + + Parameters + ---------- + input_data : numpy.ndarray + Input data array + + Raises + ------ + TypeError + For invalid input type + + """ + if not (isinstance(input_data, (self.xp.ndarray, np.ndarray))): + raise TypeError( + 'Input data must be a numpy array or backend array', + ) + + def _check_param(self, param_val): + """Check algorithm parameters. + + This method checks if the specified algorithm parameters are floats + + Parameters + ---------- + param_val : float + Parameter value + + Raises + ------ + TypeError + For invalid input type + + """ + if not isinstance(param_val, float): + raise TypeError('Algorithm parameter must be a float value.') + + def _check_param_update(self, param_update): + """Check algorithm parameter update methods. + + This method checks if the specified algorithm parameters are floats + + Parameters + ---------- + param_update : function + Callable function + + Raises + ------ + TypeError + For invalid input type + + """ + param_conditions = ( + not isinstance(param_update, type(None)) + and not callable(param_update) + ) + + if param_conditions: + raise TypeError( + 'Algorithm parameter update must be a callabale function.', + ) + + def _check_operator(self, operator): + """Check set-Up. + + This method checks algorithm operator against the expected parent + classes + + Parameters + ---------- + operator : str + Algorithm operator to check + + """ + if not isinstance(operator, type(None)): + tree = [op_obj.__name__ for op_obj in getmro(operator.__class__)] + + if not any(parent in tree for parent in self._op_parents): + message = '{0} does not inherit an operator parent.' + warn(message.format(str(operator.__class__))) + + def _compute_metrics(self): + """Compute metrics during iteration. + + This method create the args necessary for metrics computation, then + call the observers to compute metrics + + """ + kwargs = self.get_notify_observers_kwargs() + self.notify_observers('cv_metrics', **kwargs) + + def _iterations(self, max_iter, progbar=None): + """Iterate method. + + Iterate the update step of the given algorithm. + + Parameters + ---------- + max_iter : int + Maximum number of iterations + progbar : progressbar.ProgressBar + Progress bar (default is ``None``) + + """ + for idx in range(max_iter): + self.idx = idx + self._update() + + # Calling metrics every metric_call_period cycle + # Also calculate at the end (max_iter or at convergence) + # We do not call metrics if metrics is empty or metric call + # period is None + if self.metrics and self.metric_call_period is not None: + + metric_conditions = ( + self.idx % self.metric_call_period == 0 + or self.idx == (max_iter - 1) + or self.converge, + ) + + if metric_conditions: + self._compute_metrics() + + if self.converge: + if self.verbose: + print(' - Converged!') + break + + if not isinstance(progbar, type(None)): + progbar.update(idx) + + def _run_alg(self, max_iter): + """Run algorithm. + + Run the update step of a given algorithm up to the maximum number of + iterations. + + Parameters + ---------- + max_iter : int + Maximum number of iterations + + """ + if self.progress: + with ProgressBar( + redirect_stdout=True, + max_value=max_iter, + ) as progbar: + self._iterations(max_iter, progbar=progbar) + else: + self._iterations(max_iter) diff --git a/modopt/opt/algorithms.py b/modopt/opt/algorithms/forward_backward.py similarity index 64% rename from modopt/opt/algorithms.py rename to modopt/opt/algorithms/forward_backward.py index 125ac84c..5c6fb404 100644 --- a/modopt/opt/algorithms.py +++ b/modopt/opt/algorithms/forward_backward.py @@ -1,337 +1,14 @@ # -*- coding: utf-8 -*- - -r"""OPTIMISATION ALGOTITHMS. - -This module contains class implementations of various optimisation algoritms. - -:Authors: Samuel Farrens , - Zaccharie Ramzi - -:Notes: - -Input classes must have the following properties: - - * **Gradient Operators** - - Must have the following methods: - - * ``get_grad()`` - calculate the gradient - - Must have the following variables: - - * ``grad`` - the gradient - - * **Linear Operators** - - Must have the following methods: - - * ``op()`` - operator - * ``adj_op()`` - adjoint operator - - * **Proximity Operators** - - Must have the following methods: - - * ``op()`` - operator - -The following notation is used to implement the algorithms: - - * x_old is used in place of :math:`x_{n}`. - * x_new is used in place of :math:`x_{n+1}`. - * x_prox is used in place of :math:`\tilde{x}_{n+1}`. - * x_temp is used for intermediate operations. - -""" - -from inspect import getmro +"""Forward-Backward Algorithms.""" import numpy as np -from progressbar import ProgressBar from modopt.base import backend -from modopt.base.observable import MetricObserver, Observable -from modopt.interface.errors import warn +from modopt.opt.algorithms.base import SetUp from modopt.opt.cost import costObj from modopt.opt.linear import Identity -class SetUp(Observable): - r"""Algorithm Set-Up. - - This class contains methods for checking the set-up of an optimisation - algotithm and produces warnings if they do not comply. - - Parameters - ---------- - metric_call_period : int, optional - Metric call period (default is ``5``) - metrics : dict, optional - Metrics to be used (default is ``\{\}``) - verbose : bool, optional - Option for verbose output (default is ``False``) - progress : bool, optional - Option to display progress bar (default is ``True``) - step_size : int, optional - Generic step size parameter to override default algorithm - parameter name (`e.g.` `step_size` will override the value set for - `beta_param` in `ForwardBackward`) - use_gpu : bool, optional - Option to use available GPU - - """ - - def __init__( - self, - metric_call_period=5, - metrics=None, - verbose=False, - progress=True, - step_size=None, - compute_backend='numpy', - **dummy_kwargs, - ): - - self.converge = False - self.verbose = verbose - self.progress = progress - self.metrics = metrics - self.step_size = step_size - self._op_parents = ( - 'GradParent', - 'ProximityParent', - 'LinearParent', - 'costObj', - ) - - self.metric_call_period = metric_call_period - - # Declaration of observers for metrics - super().__init__(['cv_metrics']) - - for name, dic in self.metrics.items(): - observer = MetricObserver( - name, - dic['metric'], - dic['mapping'], - dic['cst_kwargs'], - dic['early_stopping'], - ) - self.add_observer('cv_metrics', observer) - - xp, compute_backend = backend.get_backend(compute_backend) - self.xp = xp - self.compute_backend = compute_backend - - @property - def metrics(self): - """Metrics.""" - return self._metrics - - @metrics.setter - def metrics(self, metrics): - - if isinstance(metrics, type(None)): - self._metrics = {} - elif isinstance(metrics, dict): - self._metrics = metrics - else: - raise TypeError( - 'Metrics must be a dictionary, not {0}.'.format(type(metrics)), - ) - - def any_convergence_flag(self): - """Check convergence flag. - - Return if any matrices values matched the convergence criteria. - - Returns - ------- - bool - True if any convergence criteria met - - """ - return any( - obs.converge_flag for obs in self._observers['cv_metrics'] - ) - - def copy_data(self, input_data): - """Copy Data. - - Set directive for copying data. - - Parameters - ---------- - input_data : numpy.ndarray - Input data - - Returns - ------- - numpy.ndarray - Copy of input data - - """ - return self.xp.copy(backend.change_backend( - input_data, - self.compute_backend, - )) - - def _check_input_data(self, input_data): - """Check input data type. - - This method checks if the input data is a numpy array - - Parameters - ---------- - input_data : numpy.ndarray - Input data array - - Raises - ------ - TypeError - For invalid input type - - """ - if not (isinstance(input_data, (self.xp.ndarray, np.ndarray))): - raise TypeError( - 'Input data must be a numpy array or backend array', - ) - - def _check_param(self, param_val): - """Check algorithm parameters. - - This method checks if the specified algorithm parameters are floats - - Parameters - ---------- - param_val : float - Parameter value - - Raises - ------ - TypeError - For invalid input type - - """ - if not isinstance(param_val, float): - raise TypeError('Algorithm parameter must be a float value.') - - def _check_param_update(self, param_update): - """Check algorithm parameter update methods. - - This method checks if the specified algorithm parameters are floats - - Parameters - ---------- - param_update : function - Callable function - - Raises - ------ - TypeError - For invalid input type - - """ - param_conditions = ( - not isinstance(param_update, type(None)) - and not callable(param_update) - ) - - if param_conditions: - raise TypeError( - 'Algorithm parameter update must be a callabale function.', - ) - - def _check_operator(self, operator): - """Check set-Up. - - This method checks algorithm operator against the expected parent - classes - - Parameters - ---------- - operator : str - Algorithm operator to check - - """ - if not isinstance(operator, type(None)): - tree = [op_obj.__name__ for op_obj in getmro(operator.__class__)] - - if not any(parent in tree for parent in self._op_parents): - message = '{0} does not inherit an operator parent.' - warn(message.format(str(operator.__class__))) - - def _compute_metrics(self): - """Compute metrics during iteration. - - This method create the args necessary for metrics computation, then - call the observers to compute metrics - - """ - kwargs = self.get_notify_observers_kwargs() - self.notify_observers('cv_metrics', **kwargs) - - def _iterations(self, max_iter, progbar=None): - """Iterate method. - - Iterate the update step of the given algorithm. - - Parameters - ---------- - max_iter : int - Maximum number of iterations - progbar : progressbar.ProgressBar - Progress bar (default is ``None``) - - """ - for idx in range(max_iter): - self.idx = idx - self._update() - - # Calling metrics every metric_call_period cycle - # Also calculate at the end (max_iter or at convergence) - # We do not call metrics if metrics is empty or metric call - # period is None - if self.metrics and self.metric_call_period is not None: - - metric_conditions = ( - self.idx % self.metric_call_period == 0 - or self.idx == (max_iter - 1) - or self.converge, - ) - - if metric_conditions: - self._compute_metrics() - - if self.converge: - if self.verbose: - print(' - Converged!') - break - - if not isinstance(progbar, type(None)): - progbar.update(idx) - - def _run_alg(self, max_iter): - """Run algorithm. - - Run the update step of a given algorithm up to the maximum number of - iterations. - - Parameters - ---------- - max_iter : int - Maximum number of iterations - - """ - if self.progress: - with ProgressBar( - redirect_stdout=True, - max_value=max_iter, - ) as progbar: - self._iterations(max_iter, progbar=progbar) - else: - self._iterations(max_iter) - - class FISTA(object): """FISTA. @@ -645,6 +322,13 @@ class ForwardBackward(SetUp): The `beta_param` can also be set using the keyword `step_size`, which will override the value of `beta_param`. + The following state variable are available for metrics measurememts at + each iteration : + + * `x_new` : new estimate of x + * `z_new` : new estimate of z (adjoint representation of x). + * `idx` : index of the iteration. + See Also -------- FISTA : complementary class @@ -865,6 +549,13 @@ class GenForwardBackward(SetUp): The `gamma_param` can also be set using the keyword `step_size`, which will override the value of `gamma_param`. + The following state variable are available for metrics measurememts at + each iteration : + + * `x_new` : new estimate of x + * `z_new` : new estimate of z (adjoint representation of x). + * `idx` : index of the iteration. + See Also -------- SetUp : parent class @@ -1100,267 +791,6 @@ def retrieve_outputs(self): self.metrics = metrics -class Condat(SetUp): - """Condat optimisation. - - This class implements algorithm 3.1 from :cite:`condat2013` - - Parameters - ---------- - x : numpy.ndarray - Initial guess for the primal variable - y : numpy.ndarray - Initial guess for the dual variable - grad : class instance - Gradient operator class - prox : class instance - Proximity primal operator class - prox_dual : class instance - Proximity dual operator class - linear : class instance, optional - Linear operator class (default is ``None``) - cost : class or str, optional - Cost function class (default is 'auto'); Use 'auto' to automatically - generate a costObj instance - reweight : class instance, optional - Reweighting class - rho : float, optional - Relaxation parameter (default is ``0.5``) - sigma : float, optional - Proximal dual parameter (default is ``1.0``) - tau : float, optional - Proximal primal paramater (default is ``1.0``) - rho_update : function, optional - Relaxation parameter update method (default is ``None``) - sigma_update : function, optional - Proximal dual parameter update method (default is ``None``) - tau_update : function, optional - Proximal primal parameter update method (default is ``None``) - auto_iterate : bool, optional - Option to automatically begin iterations upon initialisation (default - is ``True``) - max_iter : int, optional - Maximum number of iterations (default is ``150``) - n_rewightings : int, optional - Number of reweightings to perform (default is ``1``) - - Notes - ----- - The `tau_param` can also be set using the keyword `step_size`, which will - override the value of `tau_param`. - - See Also - -------- - SetUp : parent class - - """ - - def __init__( - self, - x, - y, - grad, - prox, - prox_dual, - linear=None, - cost='auto', - reweight=None, - rho=0.5, - sigma=1.0, - tau=1.0, - rho_update=None, - sigma_update=None, - tau_update=None, - auto_iterate=True, - max_iter=150, - n_rewightings=1, - metric_call_period=5, - metrics=None, - **kwargs, - ): - - # Set default algorithm properties - super().__init__( - metric_call_period=metric_call_period, - metrics=metrics, - **kwargs, - ) - - # Set the initial variable values - for input_data in (x, y): - self._check_input_data(input_data) - - self._x_old = self.xp.copy(x) - self._y_old = self.xp.copy(y) - - # Set the algorithm operators - for operator in (grad, prox, prox_dual, linear, cost): - self._check_operator(operator) - - self._grad = grad - self._prox = prox - self._prox_dual = prox_dual - self._reweight = reweight - if isinstance(linear, type(None)): - self._linear = Identity() - else: - self._linear = linear - if cost == 'auto': - self._cost_func = costObj([ - self._grad, - self._prox, - self._prox_dual, - ]) - else: - self._cost_func = cost - - # Set the algorithm parameters - for param_val in (rho, sigma, tau): - self._check_param(param_val) - - self._rho = rho - self._sigma = sigma - self._tau = self.step_size or tau - - # Set the algorithm parameter update methods - for param_update in (rho_update, sigma_update, tau_update): - self._check_param_update(param_update) - - self._rho_update = rho_update - self._sigma_update = sigma_update - self._tau_update = tau_update - - # Automatically run the algorithm - if auto_iterate: - self.iterate(max_iter=max_iter, n_rewightings=n_rewightings) - - def _update_param(self): - """Update parameters. - - This method updates the values of the algorthm parameters with the - methods provided - - """ - # Update relaxation parameter. - if not isinstance(self._rho_update, type(None)): - self._rho = self._rho_update(self._rho) - - # Update proximal dual parameter. - if not isinstance(self._sigma_update, type(None)): - self._sigma = self._sigma_update(self._sigma) - - # Update proximal primal parameter. - if not isinstance(self._tau_update, type(None)): - self._tau = self._tau_update(self._tau) - - def _update(self): - """Update. - - This method updates the current reconstruction - - Notes - ----- - Implements equation 9 (algorithm 3.1) from :cite:`condat2013` - - - primal proximity operator set up for positivity constraint - - """ - # Step 1 from eq.9. - self._grad.get_grad(self._x_old) - - x_prox = self._prox.op( - self._x_old - self._tau * self._grad.grad - self._tau - * self._linear.adj_op(self._y_old), - ) - - # Step 2 from eq.9. - y_temp = ( - self._y_old + self._sigma - * self._linear.op(2 * x_prox - self._x_old) - ) - - y_prox = ( - y_temp - self._sigma - * self._prox_dual.op( - y_temp / self._sigma, - extra_factor=(1.0 / self._sigma), - ) - ) - - # Step 3 from eq.9. - self._x_new = self._rho * x_prox + (1 - self._rho) * self._x_old - self._y_new = self._rho * y_prox + (1 - self._rho) * self._y_old - - del x_prox, y_prox, y_temp - - # Update old values for next iteration. - self.xp.copyto(self._x_old, self._x_new) - self.xp.copyto(self._y_old, self._y_new) - - # Update parameter values for next iteration. - self._update_param() - - # Test cost function for convergence. - if self._cost_func: - self.converge = ( - self.any_convergence_flag() - or self._cost_func.get_cost(self._x_new, self._y_new) - ) - - def iterate(self, max_iter=150, n_rewightings=1): - """Iterate. - - This method calls update until either convergence criteria is met or - the maximum number of iterations is reached - - Parameters - ---------- - max_iter : int, optional - Maximum number of iterations (default is ``150``) - n_rewightings : int, optional - Number of reweightings to perform (default is ``1``) - - """ - self._run_alg(max_iter) - - if not isinstance(self._reweight, type(None)): - for _ in range(n_rewightings): - self._reweight.reweight(self._linear.op(self._x_new)) - self._run_alg(max_iter) - - # retrieve metrics results - self.retrieve_outputs() - # rename outputs as attributes - self.x_final = self._x_new - self.y_final = self._y_new - - def get_notify_observers_kwargs(self): - """Notify observers. - - Return the mapping between the metrics call and the iterated - variables. - - Returns - ------- - notify_observers_kwargs : dict, - The mapping between the iterated variables - - """ - return {'x_new': self._x_new, 'y_new': self._y_new, 'idx': self.idx} - - def retrieve_outputs(self): - """Retrieve outputs. - - Declare the outputs of the algorithms as attributes: x_final, - y_final, metrics. - - """ - metrics = {} - for obs in self._observers['cv_metrics']: - metrics[obs.name] = obs.retrieve_metrics() - self.metrics = metrics - - class POGM(SetUp): """Proximal Optimised Gradient Method. @@ -1399,6 +829,18 @@ class POGM(SetUp): The `beta_param` can also be set using the keyword `step_size`, which will override the value of `beta_param`. + The following state variable are available for metrics measurememts at + each iterations: + + * `u_new` : new estimate of u + * `x_new` : new estimate of x + * `y_new` : new estimate of y + * `z_new` : new estimate of z + * `xi`: xi variable + * `t` : new estimate of t + * `sigma`: sigma variable + * `idx` : index of the iteration. + See Also -------- SetUp : parent class diff --git a/modopt/opt/algorithms/gradient_descent.py b/modopt/opt/algorithms/gradient_descent.py new file mode 100644 index 00000000..8d526674 --- /dev/null +++ b/modopt/opt/algorithms/gradient_descent.py @@ -0,0 +1,428 @@ +# -*- coding: utf-8 -*- +"""Gradient Descent Algorithms.""" + +import numpy as np + +from modopt.opt.algorithms.base import SetUp +from modopt.opt.cost import costObj + + +class GenericGradOpt(SetUp): + r"""Generic Gradient descent operator. + + Performs the descent algorithm in the direction m_k at speed s_k. + + + Parameters + ---------- + x: ndarray + Initial value + grad: Instance of GradBase + Gradient operator + prox: Instance of ProximalOperator + Proximal operator, + linear: Instance of OperatorBase + Linear operator (the image domain should be sparse) + cost: Instance of costObj + Cost Operator + eta: float, default 1.0 + Descent step + eta_update: callable, default None + If not None, used to update eta at each step. + epsilon: float, default 1e-6 + Numerical stability constant for the gradient. + epoch_size, int, default 1 + Size of epoch for the descent. + metric_call_period: int, default 5 + The period of iteration on which metrics will be computed. + metrics: dict, default None + If not None, specify which metrics to use. + + Notes + ----- + The Gradient descent step is defined as: + + .. math:: x_{k+1} = x_k - \frac{\eta}{\sqrt{s_k + \epsilon}} m_k + + where: + + * :math:`m_k` is the gradient direction + * :math:`\eta` is the gradient descent step + * :math:`s_k` is the gradient "speed" + + + At each Epoch, an optional Proximal step can be performed. + + The following state variable are available for metrics measurememts: + + * `x_new` : new estimate of the iterations + * `dir_grad` : direction of the gradient descent step + * `speed_grad` : speed for the gradient descent step + * `idx` : index of the iteration being reconstructed. + + See Also + -------- + modopt.opt.algorithms.base.SetUp : parent class + + """ + + def __init__( + self, + x, + grad, + prox, + cost, + eta=1.0, + eta_update=None, + epsilon=1e-6, + epoch_size=1, + metric_call_period=5, + metrics=None, + **kwargs, + ): + # Set the initial variable values + if metrics is None: + metrics = {} + # Set default algorithm properties + super().__init__( + metric_call_period=metric_call_period, + metrics=metrics, + **kwargs, + ) + self.iter = 0 + self._check_input_data(x) + self._x_old = np.copy(x) + self._x_new = np.copy(x) + self._speed_grad = np.zeros(x.shape, dtype=float) + self._dir_grad = np.zeros_like(x) + # Set the algorithm operators + for operator in (grad, prox, cost): + self._check_operator(operator) + self._grad = grad + self._prox = prox + if cost == 'auto': + self._cost_func = costObj([self._grad, self._prox]) + else: + self._cost_func = cost + # Set the algorithm parameters + for param_val in (eta, epsilon): + self._check_param(param_val) + self._eta = eta + self._eps = epsilon + + # Set the algorithm parameter update methods + self._check_param_update(eta_update) + self._eta_update = eta_update + self.idx = 0 + self.epoch_size = epoch_size + + def iterate(self, max_iter=150): + """Iterate. + + This method calls update until either convergence criteria is met or + the maximum number of iterations is reached. + + Parameters + ---------- + max_iter : int, optional + Maximum number of iterations (default is ``150``) + + """ + self._run_alg(max_iter) + + # retrieve metrics results + self.retrieve_outputs() + + self.x_final = self._x_new + + def _update(self): + self._grad.get_grad(self._x_old) + self._update_grad_dir(self._grad.grad) + self._update_grad_speed(self._grad.grad) + step = self._eta / (np.sqrt(self._speed_grad) + self._eps) + self._x_new = self._x_old - step * self._dir_grad + if self.idx % self.epoch_size == 0: + self.reset() + self._update_reg(step) + self._x_old = self._x_new.copy() + if self._eta_update is not None: + self._eta = self._eta_update(self._eta, self.idx) + # Test cost function for convergence. + if self._cost_func: + self.converge = ( + self.any_convergence_flag() + or self._cost_func.get_cost(self._x_new) + ) + + def _update_grad_dir(self, grad): + """Update the gradient descent direction. + + Parameters + ---------- + grad: ndarray + The gradien direction + """ + self._dir_grad = grad + + def _update_grad_speed(self, grad): + """Update the gradient descent speed. + + Parameters + ---------- + grad: ndarray + The gradien direction + + """ + pass + + def _update_reg(self, factor): + """Regularisation step. + + Parameters + ---------- + factor: float or array_like + extra factor for the proximal step. + """ + self._x_new = self._prox.op(self._x_new, extra_factor=factor) + + def get_notify_observers_kwargs(self): + """Notify observers. + + Return the mapping between the metrics call and the iterated + variables. + + Returns + ------- + notify_observers_kwargs : dict, + The mapping between the iterated variables + + """ + return { + 'x_new': self._x_new, + 'dir_grad': self._dir_grad, + 'speed_grad': self._speed_grad, + 'idx': self.idx, + } + + def retrieve_outputs(self): + """Retrieve outputs. + + Declare the outputs of the algorithms as attributes: x_final, + y_final, metrics. + + """ + metrics = {} + for obs in self._observers['cv_metrics']: + metrics[obs.name] = obs.retrieve_metrics() + self.metrics = metrics + + def reset(self): + """Reset internal state of the algorithm.""" + pass + + +class VanillaGenericGradOpt(GenericGradOpt): + """Vanilla Descent Algorithm. + + Fixed step size and no numerical precision threshold. + + See Also + -------- + GenericGradOpt : parent class + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # no scale factor + self._speed_grad = 1.0 + self._eps = 0 + + +class AdaGenericGradOpt(GenericGradOpt): + r"""Generic Grad descent Algorithm with ADA acceleration scheme. + + Notes + ----- + For AdaGrad (Section 4.2 of :cite:`ruder2017`) the gradient is + preconditioned using a speed update: + + .. math:: s_k = \sum_{i=0}^k g_k * g_k + + + See Also + -------- + GenericGradOpt : parent class + """ + + def _update_grad_speed(self, grad): + """Ada Acceleration Scheme. + + Parameters + ---------- + grad: ndarray + The new gradient for updating the speed. + + """ + self._speed_grad += abs(grad) ** 2 + + +class RMSpropGradOpt(GenericGradOpt): + r"""RMSprop Gradient descent algorithm. + + Parameters + ---------- + gamma: float, default 0.5 + Update weight for the speed of descent. + + Raises + ------ + ValueError + If gamma is outside ]0,1[ + + Notes + ----- + The gradient speed of RMSProp (Section 4.5 of :cite:`ruder2017`) is + defined as : + + .. math:: s_k = \gamma s_{k-1} + (1-\gamma) * |\nabla f|^2 + + See Also + -------- + GenericGradOpt : parent class + """ + + def __init__(self, *args, gamma=0.5, **kwargs): + super().__init__(*args, **kwargs) + if gamma < 0 or gamma > 1: + raise ValueError('gamma is outside of range [0,1]') + self._check_param(gamma) + self._gamma = gamma + + def _update_grad_speed(self, grad): + """Rmsprop update speed.""" + self._speed_grad = ( + self._gamma * self._speed_grad + (1 - self._gamma) * abs(grad) ** 2 + ) + + +class MomentumGradOpt(GenericGradOpt): + r"""Momentum (Heavy-ball) descent algorithm. + + Parameters + ---------- + beta: float, default 0.9 + update weight for the momentum. + + Notes + ----- + The Momentum (Section 4.1 of :cite:`ruder2017` update is defined as: + + .. math:: m_k = \beta * m_{k-1} + \nabla f(x_k) + + See Also + -------- + GenericGradOpt : parent class + """ + + def __init__(self, *args, beta=0.9, **kwargs): + super().__init__(*args, **kwargs) + self._check_param(beta) + self._beta = beta + # no scale factor + self._speed_grad = 1.0 + self._eps = 0 + + def _update_grad_dir(self, grad): + """Momentum gradient direction update.""" + self._dir_grad = self._beta * self._dir_grad + grad + + def reset(self): + """Reset the gradient direction.""" + self._dir_grad = np.zeros_like(self._x_new) + + +class ADAMGradOpt(GenericGradOpt): + r"""ADAM optimizer. + + Parameters + ---------- + gamma: float + update weight for the direction in ]0,1[ + beta: float + update weight for the speed in ]0,1[ + + Raises + ------ + ValueError + If gamma or beta is outside ]0,1[ + + Notes + ----- + The ADAM optimizer (Section 4.6 of :cite:`ruder2017` is defined as: + + .. math:: + m_{k+1} = \frac{1}{1-\beta^k}(\beta*m_{k}+(1-\beta)*|\nabla f_k|^2) + .. math:: + s_{k+1} = \frac{1}{1-\gamma^k}(\gamma*s_k+(1-\gamma)*\nabla f_k) + + See Also + -------- + GenericGradOpt : parent class + """ + + def __init__(self, *args, gamma=0.9, beta=0.9, **kwargs): + super().__init__(*args, **kwargs) + self._check_param(gamma) + self._check_param(beta) + if gamma < 0 or gamma >= 1: + raise ValueError('gamma is outside of range [0,1]') + if beta < 0 or beta >= 1: + raise ValueError('beta is outside of range [0,1]') + self._gamma = gamma + self._beta = beta + self._beta_pow = 1 + self._gamma_pow = 1 + + def _update_grad_dir(self, grad): + """ADAM Update of gradient direction.""" + self._beta_pow *= self._beta + + self._dir_grad = (1.0 / (1.0 - self._beta_pow)) * ( + self._beta * self._dir_grad + (1 - self._beta) * grad + ) + + def _update_grad_speed(self, grad): + """ADAM Updatae of gradient speed.""" + self._gamma_pow *= self._gamma + self._speed_grad = (1.0 / (1.0 - self._gamma_pow)) * ( + self._gamma * self._speed_grad + (1 - self._gamma) * abs(grad) ** 2 + ) + + +class SAGAOptGradOpt(GenericGradOpt): + """SAGA optimizer. + + Implements equation (7) of :cite:`defazio2014` + + Notes + ----- + The stochastic part is not handled here, and should be implemented by + changing the obs_data between each call to the _update function. + + See Also + -------- + GenericGradOpt : parent class + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._grad_memory = np.zeros( + (self.epoch_size, *self._x_old.shape), + dtype=self._x_old.dtype, + ) + + def _update_grad_dir(self, grad): + """SAGA Update gradient direction.""" + cycle = self.idx % self.epoch_size + self._dir_grad = self._dir_grad - self._grad_memory[cycle] + grad + self._grad_memory[cycle] = grad diff --git a/modopt/opt/algorithms/primal_dual.py b/modopt/opt/algorithms/primal_dual.py new file mode 100644 index 00000000..8ba0630f --- /dev/null +++ b/modopt/opt/algorithms/primal_dual.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +"""Primal-Dual Algorithms.""" + +from modopt.opt.algorithms.base import SetUp +from modopt.opt.cost import costObj +from modopt.opt.linear import Identity + + +class Condat(SetUp): + """Condat optimisation. + + This class implements algorithm 3.1 from :cite:`condat2013` + + Parameters + ---------- + x : numpy.ndarray + Initial guess for the primal variable + y : numpy.ndarray + Initial guess for the dual variable + grad : class instance + Gradient operator class + prox : class instance + Proximity primal operator class + prox_dual : class instance + Proximity dual operator class + linear : class instance, optional + Linear operator class (default is ``None``) + cost : class or str, optional + Cost function class (default is 'auto'); Use 'auto' to automatically + generate a costObj instance + reweight : class instance, optional + Reweighting class + rho : float, optional + Relaxation parameter (default is ``0.5``) + sigma : float, optional + Proximal dual parameter (default is ``1.0``) + tau : float, optional + Proximal primal paramater (default is ``1.0``) + rho_update : function, optional + Relaxation parameter update method (default is ``None``) + sigma_update : function, optional + Proximal dual parameter update method (default is ``None``) + tau_update : function, optional + Proximal primal parameter update method (default is ``None``) + auto_iterate : bool, optional + Option to automatically begin iterations upon initialisation (default + is ``True``) + max_iter : int, optional + Maximum number of iterations (default is ``150``) + n_rewightings : int, optional + Number of reweightings to perform (default is ``1``) + + Notes + ----- + The `tau_param` can also be set using the keyword `step_size`, which will + override the value of `tau_param`. + + See Also + -------- + SetUp : parent class + + """ + + def __init__( + self, + x, + y, + grad, + prox, + prox_dual, + linear=None, + cost='auto', + reweight=None, + rho=0.5, + sigma=1.0, + tau=1.0, + rho_update=None, + sigma_update=None, + tau_update=None, + auto_iterate=True, + max_iter=150, + n_rewightings=1, + metric_call_period=5, + metrics=None, + **kwargs, + ): + + # Set default algorithm properties + super().__init__( + metric_call_period=metric_call_period, + metrics=metrics, + **kwargs, + ) + + # Set the initial variable values + for input_data in (x, y): + self._check_input_data(input_data) + + self._x_old = self.xp.copy(x) + self._y_old = self.xp.copy(y) + + # Set the algorithm operators + for operator in (grad, prox, prox_dual, linear, cost): + self._check_operator(operator) + + self._grad = grad + self._prox = prox + self._prox_dual = prox_dual + self._reweight = reweight + if isinstance(linear, type(None)): + self._linear = Identity() + else: + self._linear = linear + if cost == 'auto': + self._cost_func = costObj([ + self._grad, + self._prox, + self._prox_dual, + ]) + else: + self._cost_func = cost + + # Set the algorithm parameters + for param_val in (rho, sigma, tau): + self._check_param(param_val) + + self._rho = rho + self._sigma = sigma + self._tau = self.step_size or tau + + # Set the algorithm parameter update methods + for param_update in (rho_update, sigma_update, tau_update): + self._check_param_update(param_update) + + self._rho_update = rho_update + self._sigma_update = sigma_update + self._tau_update = tau_update + + # Automatically run the algorithm + if auto_iterate: + self.iterate(max_iter=max_iter, n_rewightings=n_rewightings) + + def _update_param(self): + """Update parameters. + + This method updates the values of the algorthm parameters with the + methods provided + + """ + # Update relaxation parameter. + if not isinstance(self._rho_update, type(None)): + self._rho = self._rho_update(self._rho) + + # Update proximal dual parameter. + if not isinstance(self._sigma_update, type(None)): + self._sigma = self._sigma_update(self._sigma) + + # Update proximal primal parameter. + if not isinstance(self._tau_update, type(None)): + self._tau = self._tau_update(self._tau) + + def _update(self): + """Update. + + This method updates the current reconstruction + + Notes + ----- + Implements equation 9 (algorithm 3.1) from :cite:`condat2013` + + - primal proximity operator set up for positivity constraint + + """ + # Step 1 from eq.9. + self._grad.get_grad(self._x_old) + + x_prox = self._prox.op( + self._x_old - self._tau * self._grad.grad - self._tau + * self._linear.adj_op(self._y_old), + ) + + # Step 2 from eq.9. + y_temp = ( + self._y_old + self._sigma + * self._linear.op(2 * x_prox - self._x_old) + ) + + y_prox = ( + y_temp - self._sigma + * self._prox_dual.op( + y_temp / self._sigma, + extra_factor=(1.0 / self._sigma), + ) + ) + + # Step 3 from eq.9. + self._x_new = self._rho * x_prox + (1 - self._rho) * self._x_old + self._y_new = self._rho * y_prox + (1 - self._rho) * self._y_old + + del x_prox, y_prox, y_temp + + # Update old values for next iteration. + self.xp.copyto(self._x_old, self._x_new) + self.xp.copyto(self._y_old, self._y_new) + + # Update parameter values for next iteration. + self._update_param() + + # Test cost function for convergence. + if self._cost_func: + self.converge = ( + self.any_convergence_flag() + or self._cost_func.get_cost(self._x_new, self._y_new) + ) + + def iterate(self, max_iter=150, n_rewightings=1): + """Iterate. + + This method calls update until either convergence criteria is met or + the maximum number of iterations is reached + + Parameters + ---------- + max_iter : int, optional + Maximum number of iterations (default is ``150``) + n_rewightings : int, optional + Number of reweightings to perform (default is ``1``) + + """ + self._run_alg(max_iter) + + if not isinstance(self._reweight, type(None)): + for _ in range(n_rewightings): + self._reweight.reweight(self._linear.op(self._x_new)) + self._run_alg(max_iter) + + # retrieve metrics results + self.retrieve_outputs() + # rename outputs as attributes + self.x_final = self._x_new + self.y_final = self._y_new + + def get_notify_observers_kwargs(self): + """Notify observers. + + Return the mapping between the metrics call and the iterated + variables. + + Returns + ------- + notify_observers_kwargs : dict, + The mapping between the iterated variables + + """ + return {'x_new': self._x_new, 'y_new': self._y_new, 'idx': self.idx} + + def retrieve_outputs(self): + """Retrieve outputs. + + Declare the outputs of the algorithms as attributes: x_final, + y_final, metrics. + + """ + metrics = {} + for obs in self._observers['cv_metrics']: + metrics[obs.name] = obs.retrieve_metrics() + self.metrics = metrics diff --git a/modopt/tests/test_algorithms.py b/modopt/tests/test_algorithms.py new file mode 100644 index 00000000..7ff96a8b --- /dev/null +++ b/modopt/tests/test_algorithms.py @@ -0,0 +1,470 @@ +# -*- coding: utf-8 -*- + +"""UNIT TESTS FOR OPT.ALGORITHMS. + +This module contains unit tests for the modopt.opt.algorithms module. + +:Author: Samuel Farrens + +""" + +from unittest import TestCase + +import numpy as np +import numpy.testing as npt + +from modopt.opt import algorithms, cost, gradient, linear, proximity, reweight + +# Basic functions to be used as operators or as dummy functions +func_identity = lambda x_val: x_val +func_double = lambda x_val: x_val * 2 +func_sq = lambda x_val: x_val ** 2 +func_cube = lambda x_val: x_val ** 3 + + +class Dummy(object): + """Dummy class for tests.""" + + pass + + +class AlgorithmTestCase(TestCase): + """Test case for algorithms module.""" + + def setUp(self): + """Set test parameter values.""" + self.data1 = np.arange(9).reshape(3, 3).astype(float) + self.data2 = self.data1 + np.random.randn(*self.data1.shape) * 1e-6 + self.data3 = np.arange(9).reshape(3, 3).astype(float) + 1 + + grad_inst = gradient.GradBasic( + self.data1, + func_identity, + func_identity, + ) + + prox_inst = proximity.Positivity() + prox_dual_inst = proximity.IdentityProx() + linear_inst = linear.Identity() + reweight_inst = reweight.cwbReweight(self.data3) + cost_inst = cost.costObj([grad_inst, prox_inst, prox_dual_inst]) + self.setup = algorithms.SetUp() + self.max_iter = 20 + + self.fb_all_iter = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=None, + auto_iterate=False, + beta_update=func_identity, + ) + self.fb_all_iter.iterate(self.max_iter) + + self.fb1 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + beta_update=func_identity, + ) + + self.fb2 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + lambda_update=None, + ) + + self.fb3 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + beta_update=func_identity, + a_cd=3, + ) + + self.fb4 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + beta_update=func_identity, + r_lazy=3, + p_lazy=0.7, + q_lazy=0.7, + ) + + self.fb5 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + restart_strategy='adaptive', + xi_restart=0.9, + ) + + self.fb6 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + restart_strategy='greedy', + xi_restart=0.9, + min_beta=1.0, + s_greedy=1.1, + ) + + self.gfb_all_iter = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + cost=None, + auto_iterate=False, + gamma_update=func_identity, + beta_update=func_identity, + ) + self.gfb_all_iter.iterate(self.max_iter) + + self.gfb1 = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + gamma_update=func_identity, + lambda_update=func_identity, + ) + + self.gfb2 = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + cost=cost_inst, + ) + + self.gfb3 = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + cost=cost_inst, + step_size=2, + ) + + self.condat_all_iter = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + cost=None, + prox_dual=prox_dual_inst, + sigma_update=func_identity, + tau_update=func_identity, + rho_update=func_identity, + auto_iterate=False, + ) + self.condat_all_iter.iterate(self.max_iter) + + self.condat1 = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + prox_dual=prox_dual_inst, + sigma_update=func_identity, + tau_update=func_identity, + rho_update=func_identity, + ) + + self.condat2 = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + prox_dual=prox_dual_inst, + linear=linear_inst, + cost=cost_inst, + reweight=reweight_inst, + ) + + self.condat3 = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + prox_dual=prox_dual_inst, + linear=Dummy(), + cost=cost_inst, + auto_iterate=False, + ) + + self.pogm_all_iter = algorithms.POGM( + u=self.data1, + x=self.data1, + y=self.data1, + z=self.data1, + grad=grad_inst, + prox=prox_inst, + auto_iterate=False, + cost=None, + ) + self.pogm_all_iter.iterate(self.max_iter) + + self.pogm1 = algorithms.POGM( + u=self.data1, + x=self.data1, + y=self.data1, + z=self.data1, + grad=grad_inst, + prox=prox_inst, + ) + + self.vanilla_grad = algorithms.VanillaGenericGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.ada_grad = algorithms.AdaGenericGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.adam_grad = algorithms.ADAMGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.momentum_grad = algorithms.MomentumGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.rms_grad = algorithms.RMSpropGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.saga_grad = algorithms.SAGAOptGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + + self.dummy = Dummy() + self.dummy.cost = func_identity + self.setup._check_operator(self.dummy.cost) + + def tearDown(self): + """Unset test parameter values.""" + self.data1 = None + self.data2 = None + self.setup = None + self.fb_all_iter = None + self.fb1 = None + self.fb2 = None + self.gfb_all_iter = None + self.gfb1 = None + self.gfb2 = None + self.condat_all_iter = None + self.condat1 = None + self.condat2 = None + self.condat3 = None + self.pogm1 = None + self.pogm_all_iter = None + self.dummy = None + + def test_set_up(self): + """Test set_up.""" + npt.assert_raises(TypeError, self.setup._check_input_data, 1) + + npt.assert_raises(TypeError, self.setup._check_param, 1) + + npt.assert_raises(TypeError, self.setup._check_param_update, 1) + + def test_all_iter(self): + """Test if all opt run for all iterations.""" + opts = [ + self.fb_all_iter, + self.gfb_all_iter, + self.condat_all_iter, + self.pogm_all_iter, + ] + for opt in opts: + npt.assert_equal(opt.idx, self.max_iter - 1) + + def test_forward_backward(self): + """Test forward_backward.""" + npt.assert_array_equal( + self.fb1.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb2.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb3.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb4.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb5.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb6.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + def test_gen_forward_backward(self): + """Test gen_forward_backward.""" + npt.assert_array_equal( + self.gfb1.x_final, + self.data1, + err_msg='Incorrect GenForwardBackward result.', + ) + + npt.assert_array_equal( + self.gfb2.x_final, + self.data1, + err_msg='Incorrect GenForwardBackward result.', + ) + + npt.assert_array_equal( + self.gfb3.x_final, + self.data1, + err_msg='Incorrect GenForwardBackward result.', + ) + + npt.assert_equal( + self.gfb3.step_size, + 2, + err_msg='Incorrect step size.', + ) + + npt.assert_raises( + TypeError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=1, + ) + + npt.assert_raises( + ValueError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=[1], + ) + + npt.assert_raises( + ValueError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=[0.5, 0.5], + ) + + npt.assert_raises( + ValueError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=[0.5], + ) + + def test_condat(self): + """Test gen_condat.""" + npt.assert_almost_equal( + self.condat1.x_final, + self.data1, + err_msg='Incorrect Condat result.', + ) + + npt.assert_almost_equal( + self.condat2.x_final, + self.data1, + err_msg='Incorrect Condat result.', + ) + + def test_pogm(self): + """Test pogm.""" + npt.assert_almost_equal( + self.pogm1.x_final, + self.data1, + err_msg='Incorrect POGM result.', + ) + + def test_ada_grad(self): + """Test ADA Gradient Descent.""" + self.ada_grad.iterate() + npt.assert_almost_equal( + self.ada_grad.x_final, + self.data1, + err_msg='Incorrect ADAGrad results.', + ) + + def test_adam_grad(self): + """Test ADAM Gradient Descent.""" + self.adam_grad.iterate() + npt.assert_almost_equal( + self.adam_grad.x_final, + self.data1, + err_msg='Incorrect ADAMGrad results.', + ) + + def test_momemtum_grad(self): + """Test Momemtum Gradient Descent.""" + self.momentum_grad.iterate() + npt.assert_almost_equal( + self.momentum_grad.x_final, + self.data1, + err_msg='Incorrect MomentumGrad results.', + ) + + def test_rmsprop_grad(self): + """Test RMSProp Gradient Descent.""" + self.rms_grad.iterate() + npt.assert_almost_equal( + self.rms_grad.x_final, + self.data1, + err_msg='Incorrect RMSPropGrad results.', + ) + + def test_saga_grad(self): + """Test SAGA Descent.""" + self.saga_grad.iterate() + npt.assert_almost_equal( + self.saga_grad.x_final, + self.data1, + err_msg='Incorrect SAGA Grad results.', + ) + + def test_vanilla_grad(self): + """Test Vanilla Gradient Descent.""" + self.vanilla_grad.iterate() + npt.assert_almost_equal( + self.vanilla_grad.x_final, + self.data1, + err_msg='Incorrect VanillaGrad results.', + ) diff --git a/setup.cfg b/setup.cfg index 14784b51..0619a295 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,8 +36,14 @@ per-file-ignores = modopt/math/convolve.py: WPS301,WPS420 #Todo: Rethink conditional imports modopt/math/matrix.py: WPS420 - #Todo: Check need for del statement, - modopt/opt/algorithms.py: WPS111,WPS420 + #Todo: import has bad parenthesis + modopt/opt/algorithms/__init__.py: F401,F403,WPS318, WPS319, WPS412, WPS410 + #Todo: x is a too short name. + modopt/opt/algorithms/forward_backward.py: WPS111 + #Todo: Check need for del statement + modopt/opt/algorithms/primal_dual.py: WPS111, WPS420 + #multiline parameters bug with tuples + modopt/opt/algorithms/gradient_descent.py: WPS111, WPS420, WPS317 #Todo: Consider changing costObj name modopt/opt/cost.py: N801 #Todo: