From af01498f614552da2ddf600329490db34459e08e Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Thu, 22 Apr 2021 09:37:06 +0200 Subject: [PATCH 01/10] Version 1.5.1 patch release (#114) * Add support for tensorflow backend which allows for differentiability (#112) * Added support for tensorflow * Updates to get tests passing * Or --> And * Moving modopt to allow working with tensorflow * Fix issues with wos * Fix all flakes finally! * Update modopt/base/backend.py Co-authored-by: Samuel Farrens * Update modopt/base/backend.py Co-authored-by: Samuel Farrens * Minute updates to codes * Add dynamic module * Fix docu * Fix PEP Co-authored-by: chaithyagr Co-authored-by: Samuel Farrens * Fix 115 (#116) * Fix issues * Add right tests * Fix PEP Co-authored-by: chaithyagr * Minor bug fix, remove elif (#124) Co-authored-by: chaithyagr * Add tests for modopt.base.backend and fix minute bug uncovered (#126) * Minor bug fix, remove elif * Add tests for backend * Fix tests * Add tests * Remove cupy * PEP fixes * Fix PEP * Fix PEP and update * Final PEP * Update setup.cfg Co-authored-by: Samuel Farrens * Update test_base.py Co-authored-by: chaithyagr Co-authored-by: Samuel Farrens * Release cleanup (#128) * updated GPU dependencies * added logo to manifest * updated package version and release date Co-authored-by: Chaithya G R Co-authored-by: chaithyagr --- .github/workflows/ci-build.yml | 1 + MANIFEST.in | 1 + README.md | 1 + docs/source/dependencies.rst | 6 ++ docs/source/index.rst | 4 +- modopt/base/backend.py | 108 +++++++++++++++++++++++---------- modopt/math/matrix.py | 16 ++--- modopt/opt/algorithms.py | 50 ++++++--------- modopt/tests/test_base.py | 54 ++++++++++++++++- modopt/tests/test_opt.py | 64 +++++++++++++++++++ setup.cfg | 5 +- setup.py | 9 +-- 12 files changed, 237 insertions(+), 82 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 2ed4960d..3ffcb6f4 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -53,6 +53,7 @@ jobs: python -m pip install -r develop.txt python -m pip install -r docs/requirements.txt python -m pip install astropy scikit-image scikit-learn + python -m pip install tensorflow>=2.4.1 python -m pip install twine python -m pip install . diff --git a/MANIFEST.in b/MANIFEST.in index 9a2f374e..74db0634 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include develop.txt include docs/requirements.txt include README.rst include LICENSE.txt +include docs/source/modopt_logo.png diff --git a/README.md b/README.md index aa72d976..0f7501f0 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Note that none of these are required for running on a CPU. * [CuPy](https://cupy.dev/) * [Torch](https://pytorch.org/) +* [TensorFlow](https://www.tensorflow.org/) ## Citation diff --git a/docs/source/dependencies.rst b/docs/source/dependencies.rst index 782807f8..2a513158 100644 --- a/docs/source/dependencies.rst +++ b/docs/source/dependencies.rst @@ -82,6 +82,7 @@ For GPU compliance the following packages can also be installed: * |link-to-cupy| * |link-to-torch| +* |link-to-tf| .. |link-to-cupy| raw:: html @@ -93,6 +94,11 @@ For GPU compliance the following packages can also be installed: Torch +.. |link-to-tf| raw:: html + + TensorFlow + .. note:: Note that none of these are required for running on a CPU. diff --git a/docs/source/index.rst b/docs/source/index.rst index 8262d43d..238aa5b6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,8 +12,8 @@ ModOpt Documentation .. include:: toc.rst :Author: Samuel Farrens `(samuel.farrens@cea.fr) `_ -:Version: 1.5.0 -:Release Date: 31/03/2021 +:Version: 1.5.1 +:Release Date: 22/04/2021 :Repository: |link-to-repo| .. |link-to-repo| raw:: html diff --git a/modopt/base/backend.py b/modopt/base/backend.py index 1f5a7a3a..5fbe912f 100644 --- a/modopt/base/backend.py +++ b/modopt/base/backend.py @@ -8,35 +8,73 @@ """ -import warnings from importlib import util import numpy as np +from modopt.interface.errors import warn + try: import torch + from torch.utils.dlpack import from_dlpack as torch_from_dlpack + from torch.utils.dlpack import to_dlpack as torch_to_dlpack + except ImportError: # pragma: no cover import_torch = False else: import_torch = True # Handle the compatibility with variable -gpu_compatibility = { - 'cupy': False, - 'cupy-cudnn': False, +LIBRARIES = { + 'cupy': None, + 'tensorflow': None, + 'numpy': np, } if util.find_spec('cupy') is not None: try: import cupy as cp - gpu_compatibility['cupy'] = True + LIBRARIES['cupy'] = cp + except ImportError: + pass - if util.find_spec('cupy.cuda.cudnn') is not None: - gpu_compatibility['cupy-cudnn'] = True +if util.find_spec('tensorflow') is not None: + try: + from tensorflow.experimental import numpy as tnp + LIBRARIES['tensorflow'] = tnp except ImportError: pass +def get_backend(backend): + """Get backend. + + Returns the backend module for input specified by string + + Parameters + ---------- + backend: str + String holding the backend name. One of `tensorflow`, + `numpy` or `cupy`. + + Returns + ------- + tuple + Returns the module for carrying out calculations and the actual backend + that was reverted towards. If the right libraries are not installed, + the function warns and reverts to `numpy` backend + """ + if backend not in LIBRARIES.keys() or LIBRARIES[backend] is None: + msg = ( + '{0} backend not possible, please ensure that ' + + 'the optional libraries are installed.\n' + + 'Reverting to numpy' + ) + warn(msg.format(backend)) + backend = 'numpy' + return LIBRARIES[backend], backend + + def get_array_module(input_data): """Get Array Module. @@ -54,48 +92,47 @@ def get_array_module(input_data): The numpy or cupy module """ - if gpu_compatibility['cupy']: - return cp.get_array_module(input_data) - + if LIBRARIES['tensorflow'] is not None: + if isinstance(input_data, LIBRARIES['tensorflow'].ndarray): + return LIBRARIES['tensorflow'] + if LIBRARIES['cupy'] is not None: + if isinstance(input_data, LIBRARIES['cupy'].ndarray): + return LIBRARIES['cupy'] return np -def move_to_device(input_data): +def change_backend(input_data, backend='cupy'): """Move data to device. - This method moves data from CPU to GPU if we have the - compatibility to do so. It returns the same data if - it is already on GPU. + This method changes the backend of an array + This can be used to copy data to GPU or to CPU Parameters ---------- input_data : numpy.ndarray or cupy.ndarray Input data array to be moved + backend: str, optional + The backend to use, one among `tensorflow`, `cupy` and + `numpy`. Default is `cupy`. Returns ------- - cupy.ndarray - The CuPy array residing on GPU + backend.ndarray + An ndarray of specified backend """ xp = get_array_module(input_data) - - if xp == cp: + txp, target_backend = get_backend(backend) + if xp == txp: return input_data - - if gpu_compatibility['cupy']: - return cp.array(input_data) - - warnings.warn('Cupy is not installed, cannot move data to GPU') - - return input_data + return txp.array(input_data) def move_to_cpu(input_data): """Move data to CPU. - This method moves data from GPU to CPU.It returns the same data if it is - already on CPU. + This method moves data from GPU to CPU. + It returns the same data if it is already on CPU. Parameters ---------- @@ -107,13 +144,20 @@ def move_to_cpu(input_data): numpy.ndarray The NumPy array residing on CPU + Raises + ------ + ValueError + if the input does not correspond to any array """ xp = get_array_module(input_data) - if xp == np: + if xp == LIBRARIES['numpy']: return input_data - - return input_data.get() + elif xp == LIBRARIES['cupy']: + return input_data.get() + elif xp == LIBRARIES['tensorflow']: + return input_data.data.numpy() + raise ValueError('Cannot identify the array type.') def convert_to_tensor(input_data): @@ -150,7 +194,7 @@ def convert_to_tensor(input_data): if xp == np: return torch.Tensor(input_data) - return torch.utils.dlpack.from_dlpack(input_data.toDlpack()).float() + return torch_from_dlpack(input_data.toDlpack()).float() def convert_to_cupy_array(input_data): @@ -182,6 +226,6 @@ def convert_to_cupy_array(input_data): ) if input_data.is_cuda: - return cp.fromDlpack(torch.utils.dlpack.to_dlpack(input_data)) + return cp.fromDlpack(torch_to_dlpack(input_data)) return input_data.detach().numpy() diff --git a/modopt/math/matrix.py b/modopt/math/matrix.py index cb54cebc..be737f52 100644 --- a/modopt/math/matrix.py +++ b/modopt/math/matrix.py @@ -12,12 +12,7 @@ import numpy as np -from modopt.base.backend import get_array_module - -try: - import cupy as cp -except ImportError: # pragma: no cover - pass +from modopt.base.backend import get_array_module, get_backend def gram_schmidt(matrix, return_opt='orthonormal'): @@ -303,7 +298,7 @@ def __init__( data_shape, data_type=float, auto_run=True, - use_gpu=False, + compute_backend='numpy', verbose=False, ): @@ -311,10 +306,9 @@ def __init__( self._data_shape = data_shape self._data_type = data_type self._verbose = verbose - if use_gpu: - self.xp = cp - else: - self.xp = np + xp, compute_backend = get_backend(compute_backend) + self.xp = xp + self.compute_backend = compute_backend if auto_run: self.get_spec_rad() diff --git a/modopt/opt/algorithms.py b/modopt/opt/algorithms.py index ccd8fe21..125ac84c 100644 --- a/modopt/opt/algorithms.py +++ b/modopt/opt/algorithms.py @@ -54,11 +54,6 @@ from modopt.opt.cost import costObj from modopt.opt.linear import Identity -try: - import cupy as cp -except ImportError: # pragma: no cover - pass - class SetUp(Observable): r"""Algorithm Set-Up. @@ -92,7 +87,7 @@ def __init__( verbose=False, progress=True, step_size=None, - use_gpu=False, + compute_backend='numpy', **dummy_kwargs, ): @@ -123,20 +118,9 @@ def __init__( ) self.add_observer('cv_metrics', observer) - # Check for GPU - if use_gpu: - if backend.gpu_compatibility['cupy']: - self.xp = cp - else: - warn( - 'CuPy is not installed, cannot run on GPU!' - + 'Running optimization on CPU.', - ) - self.xp = np - use_gpu = False - else: - self.xp = np - self.use_gpu = use_gpu + xp, compute_backend = backend.get_backend(compute_backend) + self.xp = xp + self.compute_backend = compute_backend @property def metrics(self): @@ -148,7 +132,9 @@ def metrics(self, metrics): if isinstance(metrics, type(None)): self._metrics = {} - elif not isinstance(metrics, dict): + elif isinstance(metrics, dict): + self._metrics = metrics + else: raise TypeError( 'Metrics must be a dictionary, not {0}.'.format(type(metrics)), ) @@ -184,10 +170,10 @@ def copy_data(self, input_data): Copy of input data """ - if self.use_gpu: - return backend.move_to_device(input_data) - - return self.xp.copy(input_data) + return self.xp.copy(backend.change_backend( + input_data, + self.compute_backend, + )) def _check_input_data(self, input_data): """Check input data type. @@ -205,8 +191,10 @@ def _check_input_data(self, input_data): For invalid input type """ - if not isinstance(input_data, self.xp.ndarray): - raise TypeError('Input data must be a numpy array.') + if not (isinstance(input_data, (self.xp.ndarray, np.ndarray))): + raise TypeError( + 'Input data must be a numpy array or backend array', + ) def _check_param(self, param_val): """Check algorithm parameters. @@ -779,8 +767,8 @@ def _update(self): self._z_new = self._x_new # Update old values for next iteration. - self.xp.copyto(self._x_old, self._x_new) - self.xp.copyto(self._z_old, self._z_new) + self._x_old = self.xp.copy(self._x_new) + self._z_old = self.xp.copy(self._z_new) # Update parameter values for next iteration. self._update_param() @@ -789,7 +777,7 @@ def _update(self): if self._cost_func: self.converge = ( self.any_convergence_flag() - or self._cost_func.get_cost(self._x_new), + or self._cost_func.get_cost(self._x_new) ) def iterate(self, max_iter=150): @@ -1548,7 +1536,7 @@ def _update(self): if self._cost_func: self.converge = ( self.any_convergence_flag() - or self._cost_func.get_cost(self._x_new), + or self._cost_func.get_cost(self._x_new) ) def iterate(self, max_iter=150): diff --git a/modopt/tests/test_base.py b/modopt/tests/test_base.py index 26e1e4ea..873a4506 100644 --- a/modopt/tests/test_base.py +++ b/modopt/tests/test_base.py @@ -9,12 +9,14 @@ """ from builtins import range -from unittest import TestCase +from unittest import TestCase, skipIf import numpy as np import numpy.testing as npt from modopt.base import np_adjust, transform, types +from modopt.base.backend import (LIBRARIES, change_backend, get_array_module, + get_backend) class NPAdjustTestCase(TestCase): @@ -275,3 +277,53 @@ def test_check_npndarray(self): self.data3, dtype=np.integer, ) + + +class TestBackend(TestCase): + """Test the backend codes.""" + + def setUp(self): + """Set test parameter values.""" + self.input = np.array([10, 10]) + + @skipIf(LIBRARIES['tensorflow'] is None, 'tensorflow library not installed') + def test_tf_backend(self): + """Test tensorflow backend.""" + xp, backend = get_backend('tensorflow') + if backend != 'tensorflow' or xp != LIBRARIES['tensorflow']: + raise AssertionError('tensorflow get_backend fails!') + tf_input = change_backend(self.input, 'tensorflow') + if ( + get_array_module(LIBRARIES['tensorflow'].ones(1)) != LIBRARIES['tensorflow'] + or get_array_module(tf_input) != LIBRARIES['tensorflow'] + ): + raise AssertionError('tensorflow backend fails!') + + @skipIf(LIBRARIES['cupy'] is None, 'cupy library not installed') + def test_cp_backend(self): + """Test cupy backend.""" + xp, backend = get_backend('cupy') + if backend != 'cupy' or xp != LIBRARIES['cupy']: + raise AssertionError('cupy get_backend fails!') + cp_input = change_backend(self.input, 'cupy') + if ( + get_array_module(LIBRARIES['cupy'].ones(1)) != LIBRARIES['cupy'] + or get_array_module(cp_input) != LIBRARIES['cupy'] + ): + raise AssertionError('cupy backend fails!') + + def test_np_backend(self): + """Test numpy backend.""" + xp, backend = get_backend('numpy') + if backend != 'numpy' or xp != LIBRARIES['numpy']: + raise AssertionError('numpy get_backend fails!') + np_input = change_backend(self.input, 'numpy') + if ( + get_array_module(LIBRARIES['numpy'].ones(1)) != LIBRARIES['numpy'] + or get_array_module(np_input) != LIBRARIES['numpy'] + ): + raise AssertionError('numpy backend fails!') + + def tearDown(self): + """Tear Down of objects.""" + self.input = None diff --git a/modopt/tests/test_opt.py b/modopt/tests/test_opt.py index b6805006..3c33c948 100644 --- a/modopt/tests/test_opt.py +++ b/modopt/tests/test_opt.py @@ -58,6 +58,17 @@ def setUp(self): reweight_inst = reweight.cwbReweight(self.data3) cost_inst = cost.costObj([grad_inst, prox_inst, prox_dual_inst]) self.setup = algorithms.SetUp() + self.max_iter = 20 + + self.fb_all_iter = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=None, + auto_iterate=False, + beta_update=func_identity, + ) + self.fb_all_iter.iterate(self.max_iter) self.fb1 = algorithms.ForwardBackward( self.data1, @@ -110,6 +121,17 @@ def setUp(self): s_greedy=1.1, ) + self.gfb_all_iter = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + cost=None, + auto_iterate=False, + gamma_update=func_identity, + beta_update=func_identity, + ) + self.gfb_all_iter.iterate(self.max_iter) + self.gfb1 = algorithms.GenForwardBackward( self.data1, grad=grad_inst, @@ -133,6 +155,20 @@ def setUp(self): step_size=2, ) + self.condat_all_iter = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + cost=None, + prox_dual=prox_dual_inst, + sigma_update=func_identity, + tau_update=func_identity, + rho_update=func_identity, + auto_iterate=False, + ) + self.condat_all_iter.iterate(self.max_iter) + self.condat1 = algorithms.Condat( self.data1, self.data2, @@ -166,6 +202,18 @@ def setUp(self): auto_iterate=False, ) + self.pogm_all_iter = algorithms.POGM( + u=self.data1, + x=self.data1, + y=self.data1, + z=self.data1, + grad=grad_inst, + prox=prox_inst, + auto_iterate=False, + cost=None, + ) + self.pogm_all_iter.iterate(self.max_iter) + self.pogm1 = algorithms.POGM( u=self.data1, x=self.data1, @@ -184,13 +232,18 @@ def tearDown(self): self.data1 = None self.data2 = None self.setup = None + self.fb_all_iter = None self.fb1 = None self.fb2 = None + self.gfb_all_iter = None self.gfb1 = None self.gfb2 = None + self.condat_all_iter = None self.condat1 = None self.condat2 = None self.condat3 = None + self.pogm1 = None + self.pogm_all_iter = None self.dummy = None def test_set_up(self): @@ -201,6 +254,17 @@ def test_set_up(self): npt.assert_raises(TypeError, self.setup._check_param_update, 1) + def test_all_iter(self): + """Test if all opt run for all iterations.""" + opts = [ + self.fb_all_iter, + self.gfb_all_iter, + self.condat_all_iter, + self.pogm_all_iter, + ] + for opt in opts: + npt.assert_equal(opt.idx, self.max_iter - 1) + def test_forward_backward(self): """Test forward_backward.""" npt.assert_array_equal( diff --git a/setup.cfg b/setup.cfg index 56a40d12..d2f544f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,8 @@ per-file-ignores = #Justification: Needed for keeping package version and current API *__init__.py*: F401,F403,WPS347,WPS410,WPS412 #Todo: Rethink conditional imports - modopt/base/backend.py: WPS229, WPS420 + #Todo: How can we bypass mutable constants? + modopt/base/backend.py: WPS229, WPS420, WPS407 #Todo: Rethink conditional imports modopt/base/observable.py: WPS420,WPS604 #Todo: Check string for log formatting @@ -54,6 +55,8 @@ per-file-ignores = modopt/signal/wavelet.py: S404,S603 #Todo: Clean up tests modopt/tests/*.py: E731,F401,WPS301,WPS420,WPS425,WPS437,WPS604 + #Todo: Import has bad parenthesis + modopt/tests/test_base.py: WPS318,WPS319,E501,WPS301 #WPS Settings max-arguments = 25 max-attributes = 40 diff --git a/setup.py b/setup.py index 92040f6d..841ca1b1 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ # Set the package release version major = 1 minor = 5 -patch = 0 +patch = 1 # Set the package details name = 'modopt' @@ -29,9 +29,10 @@ os_str = 'Operating System :: {0}' classifiers = ( - [lc_str.format(license)] + [ln_str] + - [py_str.format(ver) for ver in python_versions_supported] + - [os_str.format(ops) for ops in os_platforms_supported] + [lc_str.format(license)] + + [ln_str] + + [py_str.format(ver) for ver in python_versions_supported] + + [os_str.format(ops) for ops in os_platforms_supported] ) # Source package description from README.md From 6dfae5f0015991164c7d36f726b532a770a96635 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Wed, 1 Dec 2021 16:56:45 +0100 Subject: [PATCH 02/10] make algorithms a module. --- modopt/opt/algorithms/__init__.py | 48 ++ modopt/opt/algorithms/base.py | 288 +++++++++ .../forward_backward.py} | 587 +----------------- modopt/opt/algorithms/primal_dual.py | 268 ++++++++ 4 files changed, 606 insertions(+), 585 deletions(-) create mode 100644 modopt/opt/algorithms/__init__.py create mode 100644 modopt/opt/algorithms/base.py rename modopt/opt/{algorithms.py => algorithms/forward_backward.py} (64%) create mode 100644 modopt/opt/algorithms/primal_dual.py diff --git a/modopt/opt/algorithms/__init__.py b/modopt/opt/algorithms/__init__.py new file mode 100644 index 00000000..e774f0d5 --- /dev/null +++ b/modopt/opt/algorithms/__init__.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +r"""OPTIMISATION ALGOTITHMS. + +This module contains class implementations of various optimisation algoritms. + +:Authors: Samuel Farrens , + Zaccharie Ramzi + +:Notes: + +Input classes must have the following properties: + + * **Gradient Operators** + + Must have the following methods: + + * ``get_grad()`` - calculate the gradient + + Must have the following variables: + + * ``grad`` - the gradient + + * **Linear Operators** + + Must have the following methods: + + * ``op()`` - operator + * ``adj_op()`` - adjoint operator + + * **Proximity Operators** + + Must have the following methods: + + * ``op()`` - operator + +The following notation is used to implement the algorithms: + + * x_old is used in place of :math:`x_{n}`. + * x_new is used in place of :math:`x_{n+1}`. + * x_prox is used in place of :math:`\tilde{x}_{n+1}`. + * x_temp is used for intermediate operations. + +""" + +from .base import SetUp +from .forward_backward import * +from .primal_dual import * diff --git a/modopt/opt/algorithms/base.py b/modopt/opt/algorithms/base.py new file mode 100644 index 00000000..ef2f1452 --- /dev/null +++ b/modopt/opt/algorithms/base.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- +"""Base SetUp for optimisation algorithms.""" + +from inspect import getmro + +import numpy as np +from progressbar import ProgressBar + +from modopt.base import backend +from modopt.base.observable import MetricObserver, Observable +from modopt.interface.errors import warn + + +class SetUp(Observable): + r"""Algorithm Set-Up. + + This class contains methods for checking the set-up of an optimisation + algotithm and produces warnings if they do not comply. + + Parameters + ---------- + metric_call_period : int, optional + Metric call period (default is ``5``) + metrics : dict, optional + Metrics to be used (default is ``\{\}``) + verbose : bool, optional + Option for verbose output (default is ``False``) + progress : bool, optional + Option to display progress bar (default is ``True``) + step_size : int, optional + Generic step size parameter to override default algorithm + parameter name (`e.g.` `step_size` will override the value set for + `beta_param` in `ForwardBackward`) + use_gpu : bool, optional + Option to use available GPU + + """ + + def __init__( + self, + metric_call_period=5, + metrics=None, + verbose=False, + progress=True, + step_size=None, + compute_backend='numpy', + **dummy_kwargs, + ): + + self.converge = False + self.verbose = verbose + self.progress = progress + self.metrics = metrics + self.step_size = step_size + self._op_parents = ( + 'GradParent', + 'ProximityParent', + 'LinearParent', + 'costObj', + ) + + self.metric_call_period = metric_call_period + + # Declaration of observers for metrics + super().__init__(['cv_metrics']) + + for name, dic in self.metrics.items(): + observer = MetricObserver( + name, + dic['metric'], + dic['mapping'], + dic['cst_kwargs'], + dic['early_stopping'], + ) + self.add_observer('cv_metrics', observer) + + xp, compute_backend = backend.get_backend(compute_backend) + self.xp = xp + self.compute_backend = compute_backend + + @property + def metrics(self): + """Metrics.""" + return self._metrics + + @metrics.setter + def metrics(self, metrics): + + if isinstance(metrics, type(None)): + self._metrics = {} + elif isinstance(metrics, dict): + self._metrics = metrics + else: + raise TypeError( + 'Metrics must be a dictionary, not {0}.'.format(type(metrics)), + ) + + def any_convergence_flag(self): + """Check convergence flag. + + Return if any matrices values matched the convergence criteria. + + Returns + ------- + bool + True if any convergence criteria met + + """ + return any( + obs.converge_flag for obs in self._observers['cv_metrics'] + ) + + def copy_data(self, input_data): + """Copy Data. + + Set directive for copying data. + + Parameters + ---------- + input_data : numpy.ndarray + Input data + + Returns + ------- + numpy.ndarray + Copy of input data + + """ + return self.xp.copy(backend.change_backend( + input_data, + self.compute_backend, + )) + + def _check_input_data(self, input_data): + """Check input data type. + + This method checks if the input data is a numpy array + + Parameters + ---------- + input_data : numpy.ndarray + Input data array + + Raises + ------ + TypeError + For invalid input type + + """ + if not (isinstance(input_data, (self.xp.ndarray, np.ndarray))): + raise TypeError( + 'Input data must be a numpy array or backend array', + ) + + def _check_param(self, param_val): + """Check algorithm parameters. + + This method checks if the specified algorithm parameters are floats + + Parameters + ---------- + param_val : float + Parameter value + + Raises + ------ + TypeError + For invalid input type + + """ + if not isinstance(param_val, float): + raise TypeError('Algorithm parameter must be a float value.') + + def _check_param_update(self, param_update): + """Check algorithm parameter update methods. + + This method checks if the specified algorithm parameters are floats + + Parameters + ---------- + param_update : function + Callable function + + Raises + ------ + TypeError + For invalid input type + + """ + param_conditions = ( + not isinstance(param_update, type(None)) + and not callable(param_update) + ) + + if param_conditions: + raise TypeError( + 'Algorithm parameter update must be a callabale function.', + ) + + def _check_operator(self, operator): + """Check set-Up. + + This method checks algorithm operator against the expected parent + classes + + Parameters + ---------- + operator : str + Algorithm operator to check + + """ + if not isinstance(operator, type(None)): + tree = [op_obj.__name__ for op_obj in getmro(operator.__class__)] + + if not any(parent in tree for parent in self._op_parents): + message = '{0} does not inherit an operator parent.' + warn(message.format(str(operator.__class__))) + + def _compute_metrics(self): + """Compute metrics during iteration. + + This method create the args necessary for metrics computation, then + call the observers to compute metrics + + """ + kwargs = self.get_notify_observers_kwargs() + self.notify_observers('cv_metrics', **kwargs) + + def _iterations(self, max_iter, progbar=None): + """Iterate method. + + Iterate the update step of the given algorithm. + + Parameters + ---------- + max_iter : int + Maximum number of iterations + progbar : progressbar.ProgressBar + Progress bar (default is ``None``) + + """ + for idx in range(max_iter): + self.idx = idx + self._update() + + # Calling metrics every metric_call_period cycle + # Also calculate at the end (max_iter or at convergence) + # We do not call metrics if metrics is empty or metric call + # period is None + if self.metrics and self.metric_call_period is not None: + + metric_conditions = ( + self.idx % self.metric_call_period == 0 + or self.idx == (max_iter - 1) + or self.converge, + ) + + if metric_conditions: + self._compute_metrics() + + if self.converge: + if self.verbose: + print(' - Converged!') + break + + if not isinstance(progbar, type(None)): + progbar.update(idx) + + def _run_alg(self, max_iter): + """Run algorithm. + + Run the update step of a given algorithm up to the maximum number of + iterations. + + Parameters + ---------- + max_iter : int + Maximum number of iterations + + """ + if self.progress: + with ProgressBar( + redirect_stdout=True, + max_value=max_iter, + ) as progbar: + self._iterations(max_iter, progbar=progbar) + else: + self._iterations(max_iter) diff --git a/modopt/opt/algorithms.py b/modopt/opt/algorithms/forward_backward.py similarity index 64% rename from modopt/opt/algorithms.py rename to modopt/opt/algorithms/forward_backward.py index 125ac84c..11a93e8e 100644 --- a/modopt/opt/algorithms.py +++ b/modopt/opt/algorithms/forward_backward.py @@ -1,335 +1,13 @@ # -*- coding: utf-8 -*- - -r"""OPTIMISATION ALGOTITHMS. - -This module contains class implementations of various optimisation algoritms. - -:Authors: Samuel Farrens , - Zaccharie Ramzi - -:Notes: - -Input classes must have the following properties: - - * **Gradient Operators** - - Must have the following methods: - - * ``get_grad()`` - calculate the gradient - - Must have the following variables: - - * ``grad`` - the gradient - - * **Linear Operators** - - Must have the following methods: - - * ``op()`` - operator - * ``adj_op()`` - adjoint operator - - * **Proximity Operators** - - Must have the following methods: - - * ``op()`` - operator - -The following notation is used to implement the algorithms: - - * x_old is used in place of :math:`x_{n}`. - * x_new is used in place of :math:`x_{n+1}`. - * x_prox is used in place of :math:`\tilde{x}_{n+1}`. - * x_temp is used for intermediate operations. - -""" - -from inspect import getmro +"""Forward-Backward Algorithms.""" import numpy as np -from progressbar import ProgressBar from modopt.base import backend -from modopt.base.observable import MetricObserver, Observable -from modopt.interface.errors import warn from modopt.opt.cost import costObj from modopt.opt.linear import Identity - -class SetUp(Observable): - r"""Algorithm Set-Up. - - This class contains methods for checking the set-up of an optimisation - algotithm and produces warnings if they do not comply. - - Parameters - ---------- - metric_call_period : int, optional - Metric call period (default is ``5``) - metrics : dict, optional - Metrics to be used (default is ``\{\}``) - verbose : bool, optional - Option for verbose output (default is ``False``) - progress : bool, optional - Option to display progress bar (default is ``True``) - step_size : int, optional - Generic step size parameter to override default algorithm - parameter name (`e.g.` `step_size` will override the value set for - `beta_param` in `ForwardBackward`) - use_gpu : bool, optional - Option to use available GPU - - """ - - def __init__( - self, - metric_call_period=5, - metrics=None, - verbose=False, - progress=True, - step_size=None, - compute_backend='numpy', - **dummy_kwargs, - ): - - self.converge = False - self.verbose = verbose - self.progress = progress - self.metrics = metrics - self.step_size = step_size - self._op_parents = ( - 'GradParent', - 'ProximityParent', - 'LinearParent', - 'costObj', - ) - - self.metric_call_period = metric_call_period - - # Declaration of observers for metrics - super().__init__(['cv_metrics']) - - for name, dic in self.metrics.items(): - observer = MetricObserver( - name, - dic['metric'], - dic['mapping'], - dic['cst_kwargs'], - dic['early_stopping'], - ) - self.add_observer('cv_metrics', observer) - - xp, compute_backend = backend.get_backend(compute_backend) - self.xp = xp - self.compute_backend = compute_backend - - @property - def metrics(self): - """Metrics.""" - return self._metrics - - @metrics.setter - def metrics(self, metrics): - - if isinstance(metrics, type(None)): - self._metrics = {} - elif isinstance(metrics, dict): - self._metrics = metrics - else: - raise TypeError( - 'Metrics must be a dictionary, not {0}.'.format(type(metrics)), - ) - - def any_convergence_flag(self): - """Check convergence flag. - - Return if any matrices values matched the convergence criteria. - - Returns - ------- - bool - True if any convergence criteria met - - """ - return any( - obs.converge_flag for obs in self._observers['cv_metrics'] - ) - - def copy_data(self, input_data): - """Copy Data. - - Set directive for copying data. - - Parameters - ---------- - input_data : numpy.ndarray - Input data - - Returns - ------- - numpy.ndarray - Copy of input data - - """ - return self.xp.copy(backend.change_backend( - input_data, - self.compute_backend, - )) - - def _check_input_data(self, input_data): - """Check input data type. - - This method checks if the input data is a numpy array - - Parameters - ---------- - input_data : numpy.ndarray - Input data array - - Raises - ------ - TypeError - For invalid input type - - """ - if not (isinstance(input_data, (self.xp.ndarray, np.ndarray))): - raise TypeError( - 'Input data must be a numpy array or backend array', - ) - - def _check_param(self, param_val): - """Check algorithm parameters. - - This method checks if the specified algorithm parameters are floats - - Parameters - ---------- - param_val : float - Parameter value - - Raises - ------ - TypeError - For invalid input type - - """ - if not isinstance(param_val, float): - raise TypeError('Algorithm parameter must be a float value.') - - def _check_param_update(self, param_update): - """Check algorithm parameter update methods. - - This method checks if the specified algorithm parameters are floats - - Parameters - ---------- - param_update : function - Callable function - - Raises - ------ - TypeError - For invalid input type - - """ - param_conditions = ( - not isinstance(param_update, type(None)) - and not callable(param_update) - ) - - if param_conditions: - raise TypeError( - 'Algorithm parameter update must be a callabale function.', - ) - - def _check_operator(self, operator): - """Check set-Up. - - This method checks algorithm operator against the expected parent - classes - - Parameters - ---------- - operator : str - Algorithm operator to check - - """ - if not isinstance(operator, type(None)): - tree = [op_obj.__name__ for op_obj in getmro(operator.__class__)] - - if not any(parent in tree for parent in self._op_parents): - message = '{0} does not inherit an operator parent.' - warn(message.format(str(operator.__class__))) - - def _compute_metrics(self): - """Compute metrics during iteration. - - This method create the args necessary for metrics computation, then - call the observers to compute metrics - - """ - kwargs = self.get_notify_observers_kwargs() - self.notify_observers('cv_metrics', **kwargs) - - def _iterations(self, max_iter, progbar=None): - """Iterate method. - - Iterate the update step of the given algorithm. - - Parameters - ---------- - max_iter : int - Maximum number of iterations - progbar : progressbar.ProgressBar - Progress bar (default is ``None``) - - """ - for idx in range(max_iter): - self.idx = idx - self._update() - - # Calling metrics every metric_call_period cycle - # Also calculate at the end (max_iter or at convergence) - # We do not call metrics if metrics is empty or metric call - # period is None - if self.metrics and self.metric_call_period is not None: - - metric_conditions = ( - self.idx % self.metric_call_period == 0 - or self.idx == (max_iter - 1) - or self.converge, - ) - - if metric_conditions: - self._compute_metrics() - - if self.converge: - if self.verbose: - print(' - Converged!') - break - - if not isinstance(progbar, type(None)): - progbar.update(idx) - - def _run_alg(self, max_iter): - """Run algorithm. - - Run the update step of a given algorithm up to the maximum number of - iterations. - - Parameters - ---------- - max_iter : int - Maximum number of iterations - - """ - if self.progress: - with ProgressBar( - redirect_stdout=True, - max_value=max_iter, - ) as progbar: - self._iterations(max_iter, progbar=progbar) - else: - self._iterations(max_iter) +from .base import SetUp class FISTA(object): @@ -1100,267 +778,6 @@ def retrieve_outputs(self): self.metrics = metrics -class Condat(SetUp): - """Condat optimisation. - - This class implements algorithm 3.1 from :cite:`condat2013` - - Parameters - ---------- - x : numpy.ndarray - Initial guess for the primal variable - y : numpy.ndarray - Initial guess for the dual variable - grad : class instance - Gradient operator class - prox : class instance - Proximity primal operator class - prox_dual : class instance - Proximity dual operator class - linear : class instance, optional - Linear operator class (default is ``None``) - cost : class or str, optional - Cost function class (default is 'auto'); Use 'auto' to automatically - generate a costObj instance - reweight : class instance, optional - Reweighting class - rho : float, optional - Relaxation parameter (default is ``0.5``) - sigma : float, optional - Proximal dual parameter (default is ``1.0``) - tau : float, optional - Proximal primal paramater (default is ``1.0``) - rho_update : function, optional - Relaxation parameter update method (default is ``None``) - sigma_update : function, optional - Proximal dual parameter update method (default is ``None``) - tau_update : function, optional - Proximal primal parameter update method (default is ``None``) - auto_iterate : bool, optional - Option to automatically begin iterations upon initialisation (default - is ``True``) - max_iter : int, optional - Maximum number of iterations (default is ``150``) - n_rewightings : int, optional - Number of reweightings to perform (default is ``1``) - - Notes - ----- - The `tau_param` can also be set using the keyword `step_size`, which will - override the value of `tau_param`. - - See Also - -------- - SetUp : parent class - - """ - - def __init__( - self, - x, - y, - grad, - prox, - prox_dual, - linear=None, - cost='auto', - reweight=None, - rho=0.5, - sigma=1.0, - tau=1.0, - rho_update=None, - sigma_update=None, - tau_update=None, - auto_iterate=True, - max_iter=150, - n_rewightings=1, - metric_call_period=5, - metrics=None, - **kwargs, - ): - - # Set default algorithm properties - super().__init__( - metric_call_period=metric_call_period, - metrics=metrics, - **kwargs, - ) - - # Set the initial variable values - for input_data in (x, y): - self._check_input_data(input_data) - - self._x_old = self.xp.copy(x) - self._y_old = self.xp.copy(y) - - # Set the algorithm operators - for operator in (grad, prox, prox_dual, linear, cost): - self._check_operator(operator) - - self._grad = grad - self._prox = prox - self._prox_dual = prox_dual - self._reweight = reweight - if isinstance(linear, type(None)): - self._linear = Identity() - else: - self._linear = linear - if cost == 'auto': - self._cost_func = costObj([ - self._grad, - self._prox, - self._prox_dual, - ]) - else: - self._cost_func = cost - - # Set the algorithm parameters - for param_val in (rho, sigma, tau): - self._check_param(param_val) - - self._rho = rho - self._sigma = sigma - self._tau = self.step_size or tau - - # Set the algorithm parameter update methods - for param_update in (rho_update, sigma_update, tau_update): - self._check_param_update(param_update) - - self._rho_update = rho_update - self._sigma_update = sigma_update - self._tau_update = tau_update - - # Automatically run the algorithm - if auto_iterate: - self.iterate(max_iter=max_iter, n_rewightings=n_rewightings) - - def _update_param(self): - """Update parameters. - - This method updates the values of the algorthm parameters with the - methods provided - - """ - # Update relaxation parameter. - if not isinstance(self._rho_update, type(None)): - self._rho = self._rho_update(self._rho) - - # Update proximal dual parameter. - if not isinstance(self._sigma_update, type(None)): - self._sigma = self._sigma_update(self._sigma) - - # Update proximal primal parameter. - if not isinstance(self._tau_update, type(None)): - self._tau = self._tau_update(self._tau) - - def _update(self): - """Update. - - This method updates the current reconstruction - - Notes - ----- - Implements equation 9 (algorithm 3.1) from :cite:`condat2013` - - - primal proximity operator set up for positivity constraint - - """ - # Step 1 from eq.9. - self._grad.get_grad(self._x_old) - - x_prox = self._prox.op( - self._x_old - self._tau * self._grad.grad - self._tau - * self._linear.adj_op(self._y_old), - ) - - # Step 2 from eq.9. - y_temp = ( - self._y_old + self._sigma - * self._linear.op(2 * x_prox - self._x_old) - ) - - y_prox = ( - y_temp - self._sigma - * self._prox_dual.op( - y_temp / self._sigma, - extra_factor=(1.0 / self._sigma), - ) - ) - - # Step 3 from eq.9. - self._x_new = self._rho * x_prox + (1 - self._rho) * self._x_old - self._y_new = self._rho * y_prox + (1 - self._rho) * self._y_old - - del x_prox, y_prox, y_temp - - # Update old values for next iteration. - self.xp.copyto(self._x_old, self._x_new) - self.xp.copyto(self._y_old, self._y_new) - - # Update parameter values for next iteration. - self._update_param() - - # Test cost function for convergence. - if self._cost_func: - self.converge = ( - self.any_convergence_flag() - or self._cost_func.get_cost(self._x_new, self._y_new) - ) - - def iterate(self, max_iter=150, n_rewightings=1): - """Iterate. - - This method calls update until either convergence criteria is met or - the maximum number of iterations is reached - - Parameters - ---------- - max_iter : int, optional - Maximum number of iterations (default is ``150``) - n_rewightings : int, optional - Number of reweightings to perform (default is ``1``) - - """ - self._run_alg(max_iter) - - if not isinstance(self._reweight, type(None)): - for _ in range(n_rewightings): - self._reweight.reweight(self._linear.op(self._x_new)) - self._run_alg(max_iter) - - # retrieve metrics results - self.retrieve_outputs() - # rename outputs as attributes - self.x_final = self._x_new - self.y_final = self._y_new - - def get_notify_observers_kwargs(self): - """Notify observers. - - Return the mapping between the metrics call and the iterated - variables. - - Returns - ------- - notify_observers_kwargs : dict, - The mapping between the iterated variables - - """ - return {'x_new': self._x_new, 'y_new': self._y_new, 'idx': self.idx} - - def retrieve_outputs(self): - """Retrieve outputs. - - Declare the outputs of the algorithms as attributes: x_final, - y_final, metrics. - - """ - metrics = {} - for obs in self._observers['cv_metrics']: - metrics[obs.name] = obs.retrieve_metrics() - self.metrics = metrics - - class POGM(SetUp): """Proximal Optimised Gradient Method. diff --git a/modopt/opt/algorithms/primal_dual.py b/modopt/opt/algorithms/primal_dual.py new file mode 100644 index 00000000..dd10a389 --- /dev/null +++ b/modopt/opt/algorithms/primal_dual.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +"""Primal-Dual Algorithms.""" + +from modopt.opt.cost import costObj +from modopt.opt.linear import Identity + +from .base import SetUp + + +class Condat(SetUp): + """Condat optimisation. + + This class implements algorithm 3.1 from :cite:`condat2013` + + Parameters + ---------- + x : numpy.ndarray + Initial guess for the primal variable + y : numpy.ndarray + Initial guess for the dual variable + grad : class instance + Gradient operator class + prox : class instance + Proximity primal operator class + prox_dual : class instance + Proximity dual operator class + linear : class instance, optional + Linear operator class (default is ``None``) + cost : class or str, optional + Cost function class (default is 'auto'); Use 'auto' to automatically + generate a costObj instance + reweight : class instance, optional + Reweighting class + rho : float, optional + Relaxation parameter (default is ``0.5``) + sigma : float, optional + Proximal dual parameter (default is ``1.0``) + tau : float, optional + Proximal primal paramater (default is ``1.0``) + rho_update : function, optional + Relaxation parameter update method (default is ``None``) + sigma_update : function, optional + Proximal dual parameter update method (default is ``None``) + tau_update : function, optional + Proximal primal parameter update method (default is ``None``) + auto_iterate : bool, optional + Option to automatically begin iterations upon initialisation (default + is ``True``) + max_iter : int, optional + Maximum number of iterations (default is ``150``) + n_rewightings : int, optional + Number of reweightings to perform (default is ``1``) + + Notes + ----- + The `tau_param` can also be set using the keyword `step_size`, which will + override the value of `tau_param`. + + See Also + -------- + SetUp : parent class + + """ + + def __init__( + self, + x, + y, + grad, + prox, + prox_dual, + linear=None, + cost='auto', + reweight=None, + rho=0.5, + sigma=1.0, + tau=1.0, + rho_update=None, + sigma_update=None, + tau_update=None, + auto_iterate=True, + max_iter=150, + n_rewightings=1, + metric_call_period=5, + metrics=None, + **kwargs, + ): + + # Set default algorithm properties + super().__init__( + metric_call_period=metric_call_period, + metrics=metrics, + **kwargs, + ) + + # Set the initial variable values + for input_data in (x, y): + self._check_input_data(input_data) + + self._x_old = self.xp.copy(x) + self._y_old = self.xp.copy(y) + + # Set the algorithm operators + for operator in (grad, prox, prox_dual, linear, cost): + self._check_operator(operator) + + self._grad = grad + self._prox = prox + self._prox_dual = prox_dual + self._reweight = reweight + if isinstance(linear, type(None)): + self._linear = Identity() + else: + self._linear = linear + if cost == 'auto': + self._cost_func = costObj([ + self._grad, + self._prox, + self._prox_dual, + ]) + else: + self._cost_func = cost + + # Set the algorithm parameters + for param_val in (rho, sigma, tau): + self._check_param(param_val) + + self._rho = rho + self._sigma = sigma + self._tau = self.step_size or tau + + # Set the algorithm parameter update methods + for param_update in (rho_update, sigma_update, tau_update): + self._check_param_update(param_update) + + self._rho_update = rho_update + self._sigma_update = sigma_update + self._tau_update = tau_update + + # Automatically run the algorithm + if auto_iterate: + self.iterate(max_iter=max_iter, n_rewightings=n_rewightings) + + def _update_param(self): + """Update parameters. + + This method updates the values of the algorthm parameters with the + methods provided + + """ + # Update relaxation parameter. + if not isinstance(self._rho_update, type(None)): + self._rho = self._rho_update(self._rho) + + # Update proximal dual parameter. + if not isinstance(self._sigma_update, type(None)): + self._sigma = self._sigma_update(self._sigma) + + # Update proximal primal parameter. + if not isinstance(self._tau_update, type(None)): + self._tau = self._tau_update(self._tau) + + def _update(self): + """Update. + + This method updates the current reconstruction + + Notes + ----- + Implements equation 9 (algorithm 3.1) from :cite:`condat2013` + + - primal proximity operator set up for positivity constraint + + """ + # Step 1 from eq.9. + self._grad.get_grad(self._x_old) + + x_prox = self._prox.op( + self._x_old - self._tau * self._grad.grad - self._tau + * self._linear.adj_op(self._y_old), + ) + + # Step 2 from eq.9. + y_temp = ( + self._y_old + self._sigma + * self._linear.op(2 * x_prox - self._x_old) + ) + + y_prox = ( + y_temp - self._sigma + * self._prox_dual.op( + y_temp / self._sigma, + extra_factor=(1.0 / self._sigma), + ) + ) + + # Step 3 from eq.9. + self._x_new = self._rho * x_prox + (1 - self._rho) * self._x_old + self._y_new = self._rho * y_prox + (1 - self._rho) * self._y_old + + del x_prox, y_prox, y_temp + + # Update old values for next iteration. + self.xp.copyto(self._x_old, self._x_new) + self.xp.copyto(self._y_old, self._y_new) + + # Update parameter values for next iteration. + self._update_param() + + # Test cost function for convergence. + if self._cost_func: + self.converge = ( + self.any_convergence_flag() + or self._cost_func.get_cost(self._x_new, self._y_new) + ) + + def iterate(self, max_iter=150, n_rewightings=1): + """Iterate. + + This method calls update until either convergence criteria is met or + the maximum number of iterations is reached + + Parameters + ---------- + max_iter : int, optional + Maximum number of iterations (default is ``150``) + n_rewightings : int, optional + Number of reweightings to perform (default is ``1``) + + """ + self._run_alg(max_iter) + + if not isinstance(self._reweight, type(None)): + for _ in range(n_rewightings): + self._reweight.reweight(self._linear.op(self._x_new)) + self._run_alg(max_iter) + + # retrieve metrics results + self.retrieve_outputs() + # rename outputs as attributes + self.x_final = self._x_new + self.y_final = self._y_new + + def get_notify_observers_kwargs(self): + """Notify observers. + + Return the mapping between the metrics call and the iterated + variables. + + Returns + ------- + notify_observers_kwargs : dict, + The mapping between the iterated variables + + """ + return {'x_new': self._x_new, 'y_new': self._y_new, 'idx': self.idx} + + def retrieve_outputs(self): + """Retrieve outputs. + + Declare the outputs of the algorithms as attributes: x_final, + y_final, metrics. + + """ + metrics = {} + for obs in self._observers['cv_metrics']: + metrics[obs.name] = obs.retrieve_metrics() + self.metrics = metrics From b7fe44a3d6ad57f5dc6f1d539d63388905343870 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Wed, 1 Dec 2021 17:34:44 +0100 Subject: [PATCH 03/10] add Gradient Descent Algorithms --- modopt/opt/algorithms/gradient_descent.py | 319 ++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 modopt/opt/algorithms/gradient_descent.py diff --git a/modopt/opt/algorithms/gradient_descent.py b/modopt/opt/algorithms/gradient_descent.py new file mode 100644 index 00000000..a289fbd5 --- /dev/null +++ b/modopt/opt/algorithms/gradient_descent.py @@ -0,0 +1,319 @@ +# -*- coding: utf-8 -*- +"""Gradient Descent Algorithms.""" + +import numpy as np + +from ..opt.cost import costObj +from .base import SetUp + + +class GenericGradOpt(SetUp): + r"""Generic Gradient descent operator. + + Performs the descent algorithm in the direction m_k at speed s_k. + + + Parameters + ---------- + x: ndarray + Initial value + grad: Instance of GradBase + Gradient operator + prox: Instance of ProximalOperator + Proximal operator, + linear: Instance of OperatorBase + Linear operator (the image domain should be sparse) + cost: + Cost Operator + eta: float, default 1.0 + Descent step + eta_update: callable, default None + If not None, used to update eta at each step. + epsilon: float, default 1e-6 + Numerical stability constant for the gradient. + epoch_size, int, default 1 + Size of epoch for the descent. + metric_call_period: int, default 5 + The period of iteration on which metrics will be computed. + metrics: dict, default None + If not None, specify which metrics to use. + + Notes + ----- + The Gradient descent step is defined as: + + ..math:: x_{k+1} = x_k - \frac{\eta}{\sqrt{s_k + \epsilon}} m_k + + At each Epoch, an optional Proximal step can be performed. + + See Also + -------- + metric api + modopt.opt.algorithms.base.SetUp + """ + + def __init__(self, x, grad, prox, cost, + eta=1.0, eta_update=None, epsilon=1e-6, epoch_size=1, + metric_call_period=5, metrics=None, **kwargs): + # Set the initial variable values + if metrics is None: + metrics = {} + # Set default algorithm properties + super().__init__(metric_call_period=metric_call_period, + metrics=metrics, + **kwargs) + self.iter = 0 + self._check_input_data(x) + self._x_old = np.copy(x) + self._x_new = np.copy(x) + self._speed_grad = np.zeros(x.shape, dtype=float) + self._dir_grad = np.zeros_like(x) + # Set the algorithm operators + for operator in (grad, prox, cost): + self._check_operator(operator) + self._grad = grad + self._prox = prox + if cost == 'auto': + self._cost_func = costObj([self._grad, self._prox]) + else: + self._cost_func = cost + # Set the algorithm parameters + for param in (eta, epsilon): + self._check_param(param) + self._eta = eta + self._eps = epsilon + + # Set the algorithm parameter update methods + self._check_param_update(eta_update) + self._eta_update = eta_update + self.idx = 0 + self.epoch_size = epoch_size + + def _update(self): + self._grad.get_grad(self._x_old) + self.update_grad_dir(self._grad.grad) + self.update_grad_speed(self._grad.grad) + step = self._eta / (np.sqrt(self._speed_grad) + self._eps) + self._x_new = self._x_old - step * self._dir_grad + if self.idx % self.epoch_size == 0: + self.reset() + self.update_reg(step) + self._x_old = self._x_new.copy() + if self._eta_update is not None: + self._eta = self._eta_update(self._eta, self.idx) + # Test cost function for convergence. + if self._cost_func: + self.converge = (self.any_convergence_flag() + or self._cost_func.get_cost(self._x_new)) + + def update_grad_dir(self, grad): + """Update the gradient descent direction.""" + self._dir_grad = grad + + def update_grad_speed(self, grad): + """Update the gradient descent speed.""" + pass + + def update_reg(self, factor): + """Regularisation step.""" + self._x_new = self._prox.op(self._x_new, extra_factor=factor) + + def get_notify_observers_kwargs(self): + """Notify observers. + + Return the mapping between the metrics call and the iterated + variables. + + Returns + ------- + notify_observers_kwargs : dict, + The mapping between the iterated variables + + """ + return { + 'x_new': self._x_new, + 'dir_grad': self._dir_grad, + 'speed_grad': self._speed_grad, + 'idx': self.idx, + } + + def retrieve_outputs(self): + """Retrieve outputs. + + Declare the outputs of the algorithms as attributes: x_final, + y_final, metrics. + + """ + metrics = {} + for obs in self._observers['cv_metrics']: + metrics[obs.name] = obs.retrieve_metrics() + self.metrics = metrics + + def reset(self): + """Reset internal state of the algorithm.""" + pass + + +class VanillaGenericGradOpt(GenericGradOpt): + """Vanilla Descent Algorithm. + + Fixed step size and no numerical precision threshold. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # no scale factor + self._speed_grad = 1.0 + self._eps = 0 + + +class AdaGenericGradOpt(GenericGradOpt): + """Generic Grad descent Algorithm with ADA acceleration scheme.""" + + def update_grad_speed(self, grad): + """Ada Acceleration Scheme.""" + self._speed_grad += abs(grad) ** 2 + + +class RMSpropGradOpt(GenericGradOpt): + r"""RMSprop Gradient descent algorithm. + + Parameters + ---------- + gamma: float, default 0.5 + Update weight for the speed of descent. + + Raises + ------ + ValueError + If gamma is outside ]0,1[ + + Notes + ----- + The gradient speed is defined as : + + .. math:: s_k = \gamma s_{k-1} + (1-\gamma) * |\nabla f|^2 + + """ + + def __init__(self, *args, gamma=0.5, **kwargs): + super().__init__(*args, **kwargs) + if gamma < 0 or gamma > 1: + raise ValueError('gamma is outside of range [0,1]') + self._check_param(gamma) + self._gamma = gamma + + def update_grad_speed(self, grad): + """Rmsprop update speed.""" + self._speed_grad = ( + self._gamma * self._speed_grad + (1 - self._gamma) * abs(grad) ** 2 + ) + + +class MomentumGradOpt(GenericGradOpt): + r"""Momentum (Heavy-ball) descent algorithm. + + Parameters + ---------- + beta: float, default 0.9 + update weight for the momentum. + + Notes + ----- + The momentum update is defined as: + + .. math:: m_k = \beta * m_{k-1} + \nabla f(x_k) + """ + + def __init__(self, *args, beta=0.9, **kwargs): + super().__init__(*args, **kwargs) + self._check_param(beta) + self._beta = beta + # no scale factor + self._speed_grad = 1.0 + self._eps = 0.0 + + def update_grad_dir(self, grad): + """Momentum gradient direction update.""" + self._dir_grad = self._beta * self._dir_grad + grad + + def reset(self): + """Reset the gradient direction.""" + self._dir_grad = np.zeros_like(self._x_new) + + +class ADAMGradOpt(GenericGradOpt): + r"""ADAM optimizer. + + Parameters + ---------- + gamma: float + update weight for the direction in ]0,1[ + beta: float + update weight for the speed in ]0,1[ + + Raises + ------ + ValueError + If gamma or beta is outside ]0,1[ + + Notes + ----- + The ADAM optimizer is defined as: + + .. math:: + m_{k+1} = \frac{1}{1-\beta^k}(\beta*m_{k}+(1-\beta)*|\nabla f_k|^2) + .. math:: + s_{k+1} = \frac{1}{1-\gamma^k}(\gamma*s_k+(1-\gamma)*\nabla f_k) + """ + + def __init__(self, *args, **kwargs): + gamma = kwargs.pop('gamma') + beta = kwargs.pop('beta') + super().__init__(*args, **kwargs) + self._check_param(gamma) + self._check_param(beta) + if gamma < 0 or gamma >= 1: + raise ValueError('gamma is outside of range [0,1]') + if beta < 0 or beta >= 1: + raise ValueError('beta is outside of range [0,1]') + self._gamma = gamma + self._beta = beta + self._beta_pow = 1 + self._gamma_pow = 1 + + def update_grad_dir(self, grad): + """ADAM Update of gradient direction.""" + self._beta_pow *= self._beta + + self._dir_grad = (1.0 / (1.0 - self._beta_pow)) * ( + self._beta * self._dir_grad + (1 - self._beta) * grad + ) + + def update_grad_speed(self, grad): + """ADAM Updatae of gradient speed.""" + self._gamma_pow *= self._gamma + self._speed_grad = (1.0 / (1.0 - self._gamma_pow)) * ( + self._gamma * self._speed_grad + (1 - self._gamma) * abs(grad) ** 2 + ) + + +class SAGAOptGradOpt(GenericGradOpt): + """SAGA optimizer. + + Notes + ----- + The stochastic part is not handled here, and should be implemented by + changing the obs_data between each call to the _update function. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._grad_memory = np.zeros((self.epoch_size, *self._x_old.size), + dtype=self._x_old.dtype) + + def update_grad_dir(self, grad): + """SAGA Update gradient direction.""" + cycle = self.idx % self.epoch_size + self._dir_grad = self._dir_grad - self._grad_memory[cycle] + grad + self._grad_memory[cycle] = grad From 6cc1b5b8ef81e5ddef4ce38ddb9f92b1a907cf62 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Thu, 2 Dec 2021 15:39:39 +0100 Subject: [PATCH 04/10] enforce WPS compliance. --- modopt/opt/algorithms/__init__.py | 16 ++++++-- modopt/opt/algorithms/forward_backward.py | 3 +- modopt/opt/algorithms/gradient_descent.py | 47 +++++++++++++++-------- modopt/opt/algorithms/primal_dual.py | 3 +- setup.cfg | 10 ++++- 5 files changed, 54 insertions(+), 25 deletions(-) diff --git a/modopt/opt/algorithms/__init__.py b/modopt/opt/algorithms/__init__.py index e774f0d5..aa645e47 100644 --- a/modopt/opt/algorithms/__init__.py +++ b/modopt/opt/algorithms/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - r"""OPTIMISATION ALGOTITHMS. This module contains class implementations of various optimisation algoritms. @@ -43,6 +42,15 @@ """ -from .base import SetUp -from .forward_backward import * -from .primal_dual import * +from modopt.opt.algorithms.base import SetUp +from modopt.opt.algorithms.forward_backward import (FISTA, POGM, + ForwardBackward, + GenForwardBackward) +from modopt.opt.algorithms.gradient_descent import (AdaGenericGradOpt, + ADAMGradOpt, + GenericGradOpt, + MomentumGradOpt, + RMSpropGradOpt, + SAGAOptGradOpt, + VanillaGenericGradOpt) +from modopt.opt.algorithms.primal_dual import Condat diff --git a/modopt/opt/algorithms/forward_backward.py b/modopt/opt/algorithms/forward_backward.py index 11a93e8e..94287fd3 100644 --- a/modopt/opt/algorithms/forward_backward.py +++ b/modopt/opt/algorithms/forward_backward.py @@ -4,11 +4,10 @@ import numpy as np from modopt.base import backend +from modopt.opt.algorithms.base import SetUp from modopt.opt.cost import costObj from modopt.opt.linear import Identity -from .base import SetUp - class FISTA(object): """FISTA. diff --git a/modopt/opt/algorithms/gradient_descent.py b/modopt/opt/algorithms/gradient_descent.py index a289fbd5..cc150547 100644 --- a/modopt/opt/algorithms/gradient_descent.py +++ b/modopt/opt/algorithms/gradient_descent.py @@ -3,8 +3,8 @@ import numpy as np -from ..opt.cost import costObj -from .base import SetUp +from modopt.opt.algorithms.base import SetUp +from modopt.opt.cost import costObj class GenericGradOpt(SetUp): @@ -52,16 +52,29 @@ class GenericGradOpt(SetUp): modopt.opt.algorithms.base.SetUp """ - def __init__(self, x, grad, prox, cost, - eta=1.0, eta_update=None, epsilon=1e-6, epoch_size=1, - metric_call_period=5, metrics=None, **kwargs): + def __init__( + self, + x, + grad, + prox, + cost, + eta=1.0, + eta_update=None, + epsilon=1e-6, + epoch_size=1, + metric_call_period=5, + metrics=None, + **kwargs, + ): # Set the initial variable values if metrics is None: metrics = {} # Set default algorithm properties - super().__init__(metric_call_period=metric_call_period, - metrics=metrics, - **kwargs) + super().__init__( + metric_call_period=metric_call_period, + metrics=metrics, + **kwargs, + ) self.iter = 0 self._check_input_data(x) self._x_old = np.copy(x) @@ -78,8 +91,8 @@ def __init__(self, x, grad, prox, cost, else: self._cost_func = cost # Set the algorithm parameters - for param in (eta, epsilon): - self._check_param(param) + for param_val in (eta, epsilon): + self._check_param(param_val) self._eta = eta self._eps = epsilon @@ -103,8 +116,10 @@ def _update(self): self._eta = self._eta_update(self._eta, self.idx) # Test cost function for convergence. if self._cost_func: - self.converge = (self.any_convergence_flag() - or self._cost_func.get_cost(self._x_new)) + self.converge = ( + self.any_convergence_flag() + or self._cost_func.get_cost(self._x_new) + ) def update_grad_dir(self, grad): """Update the gradient descent direction.""" @@ -231,7 +246,7 @@ def __init__(self, *args, beta=0.9, **kwargs): self._beta = beta # no scale factor self._speed_grad = 1.0 - self._eps = 0.0 + self._eps = 0 def update_grad_dir(self, grad): """Momentum gradient direction update.""" @@ -309,8 +324,10 @@ class SAGAOptGradOpt(GenericGradOpt): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._grad_memory = np.zeros((self.epoch_size, *self._x_old.size), - dtype=self._x_old.dtype) + self._grad_memory = np.zeros( + (self.epoch_size, *self._x_old.size), + dtype=self._x_old.dtype, + ) def update_grad_dir(self, grad): """SAGA Update gradient direction.""" diff --git a/modopt/opt/algorithms/primal_dual.py b/modopt/opt/algorithms/primal_dual.py index dd10a389..8ba0630f 100644 --- a/modopt/opt/algorithms/primal_dual.py +++ b/modopt/opt/algorithms/primal_dual.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- """Primal-Dual Algorithms.""" +from modopt.opt.algorithms.base import SetUp from modopt.opt.cost import costObj from modopt.opt.linear import Identity -from .base import SetUp - class Condat(SetUp): """Condat optimisation. diff --git a/setup.cfg b/setup.cfg index d2f544f0..18f161a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,8 +36,14 @@ per-file-ignores = modopt/math/convolve.py: WPS301,WPS420 #Todo: Rethink conditional imports modopt/math/matrix.py: WPS420 - #Todo: Check need for del statement, - modopt/opt/algorithms.py: WPS111,WPS420 + #Todo: import has bad parenthesis + modopt/opt/algorithms/__init__.py: F401,F403,WPS318, WPS319, WPS412, WPS410 + #Todo: x is a too short name. + modopt/opt/algorithms/forward_backward.py: WPS111 + #Todo: Check need for del statement + modopt/opt/algorithms/primal_dual.py: WPS111, WPS420 + #multiline parameters bug with tuples + modopt/opt/algorithms/gradient_descent.py: WPS111, WPS420, WPS317 #Todo: Consider changing costObj name modopt/opt/cost.py: N801 #Todo: From bad50bf45b208424a18cd5f80be40df33dba2afa Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Fri, 3 Dec 2021 18:38:03 +0100 Subject: [PATCH 05/10] add test for gradient descent --- modopt/opt/algorithms/gradient_descent.py | 26 +- modopt/tests/test_algorithms.py | 470 ++++++++++++++++++++++ modopt/tests/test_opt.py | 350 ---------------- 3 files changed, 492 insertions(+), 354 deletions(-) create mode 100644 modopt/tests/test_algorithms.py diff --git a/modopt/opt/algorithms/gradient_descent.py b/modopt/opt/algorithms/gradient_descent.py index cc150547..2488c1fb 100644 --- a/modopt/opt/algorithms/gradient_descent.py +++ b/modopt/opt/algorithms/gradient_descent.py @@ -102,6 +102,26 @@ def __init__( self.idx = 0 self.epoch_size = epoch_size + + def iterate(self, max_iter=150): + """Iterate. + + This method calls update until either convergence criteria is met or + the maximum number of iterations is reached. + + Parameters + ---------- + max_iter : int, optional + Maximum number of iterations (default is ``150``) + + """ + self._run_alg(max_iter) + + # retrieve metrics results + self.retrieve_outputs() + + self.x_final = self._x_new + def _update(self): self._grad.get_grad(self._x_old) self.update_grad_dir(self._grad.grad) @@ -282,9 +302,7 @@ class ADAMGradOpt(GenericGradOpt): s_{k+1} = \frac{1}{1-\gamma^k}(\gamma*s_k+(1-\gamma)*\nabla f_k) """ - def __init__(self, *args, **kwargs): - gamma = kwargs.pop('gamma') - beta = kwargs.pop('beta') + def __init__(self, *args, gamma=0.9, beta=0.9, **kwargs): super().__init__(*args, **kwargs) self._check_param(gamma) self._check_param(beta) @@ -325,7 +343,7 @@ class SAGAOptGradOpt(GenericGradOpt): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._grad_memory = np.zeros( - (self.epoch_size, *self._x_old.size), + (self.epoch_size, *self._x_old.shape), dtype=self._x_old.dtype, ) diff --git a/modopt/tests/test_algorithms.py b/modopt/tests/test_algorithms.py new file mode 100644 index 00000000..7ff96a8b --- /dev/null +++ b/modopt/tests/test_algorithms.py @@ -0,0 +1,470 @@ +# -*- coding: utf-8 -*- + +"""UNIT TESTS FOR OPT.ALGORITHMS. + +This module contains unit tests for the modopt.opt.algorithms module. + +:Author: Samuel Farrens + +""" + +from unittest import TestCase + +import numpy as np +import numpy.testing as npt + +from modopt.opt import algorithms, cost, gradient, linear, proximity, reweight + +# Basic functions to be used as operators or as dummy functions +func_identity = lambda x_val: x_val +func_double = lambda x_val: x_val * 2 +func_sq = lambda x_val: x_val ** 2 +func_cube = lambda x_val: x_val ** 3 + + +class Dummy(object): + """Dummy class for tests.""" + + pass + + +class AlgorithmTestCase(TestCase): + """Test case for algorithms module.""" + + def setUp(self): + """Set test parameter values.""" + self.data1 = np.arange(9).reshape(3, 3).astype(float) + self.data2 = self.data1 + np.random.randn(*self.data1.shape) * 1e-6 + self.data3 = np.arange(9).reshape(3, 3).astype(float) + 1 + + grad_inst = gradient.GradBasic( + self.data1, + func_identity, + func_identity, + ) + + prox_inst = proximity.Positivity() + prox_dual_inst = proximity.IdentityProx() + linear_inst = linear.Identity() + reweight_inst = reweight.cwbReweight(self.data3) + cost_inst = cost.costObj([grad_inst, prox_inst, prox_dual_inst]) + self.setup = algorithms.SetUp() + self.max_iter = 20 + + self.fb_all_iter = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=None, + auto_iterate=False, + beta_update=func_identity, + ) + self.fb_all_iter.iterate(self.max_iter) + + self.fb1 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + beta_update=func_identity, + ) + + self.fb2 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + lambda_update=None, + ) + + self.fb3 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + beta_update=func_identity, + a_cd=3, + ) + + self.fb4 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + beta_update=func_identity, + r_lazy=3, + p_lazy=0.7, + q_lazy=0.7, + ) + + self.fb5 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + restart_strategy='adaptive', + xi_restart=0.9, + ) + + self.fb6 = algorithms.ForwardBackward( + self.data1, + grad=grad_inst, + prox=prox_inst, + restart_strategy='greedy', + xi_restart=0.9, + min_beta=1.0, + s_greedy=1.1, + ) + + self.gfb_all_iter = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + cost=None, + auto_iterate=False, + gamma_update=func_identity, + beta_update=func_identity, + ) + self.gfb_all_iter.iterate(self.max_iter) + + self.gfb1 = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + gamma_update=func_identity, + lambda_update=func_identity, + ) + + self.gfb2 = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + cost=cost_inst, + ) + + self.gfb3 = algorithms.GenForwardBackward( + self.data1, + grad=grad_inst, + prox_list=[prox_inst, prox_dual_inst], + cost=cost_inst, + step_size=2, + ) + + self.condat_all_iter = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + cost=None, + prox_dual=prox_dual_inst, + sigma_update=func_identity, + tau_update=func_identity, + rho_update=func_identity, + auto_iterate=False, + ) + self.condat_all_iter.iterate(self.max_iter) + + self.condat1 = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + prox_dual=prox_dual_inst, + sigma_update=func_identity, + tau_update=func_identity, + rho_update=func_identity, + ) + + self.condat2 = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + prox_dual=prox_dual_inst, + linear=linear_inst, + cost=cost_inst, + reweight=reweight_inst, + ) + + self.condat3 = algorithms.Condat( + self.data1, + self.data2, + grad=grad_inst, + prox=prox_inst, + prox_dual=prox_dual_inst, + linear=Dummy(), + cost=cost_inst, + auto_iterate=False, + ) + + self.pogm_all_iter = algorithms.POGM( + u=self.data1, + x=self.data1, + y=self.data1, + z=self.data1, + grad=grad_inst, + prox=prox_inst, + auto_iterate=False, + cost=None, + ) + self.pogm_all_iter.iterate(self.max_iter) + + self.pogm1 = algorithms.POGM( + u=self.data1, + x=self.data1, + y=self.data1, + z=self.data1, + grad=grad_inst, + prox=prox_inst, + ) + + self.vanilla_grad = algorithms.VanillaGenericGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.ada_grad = algorithms.AdaGenericGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.adam_grad = algorithms.ADAMGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.momentum_grad = algorithms.MomentumGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.rms_grad = algorithms.RMSpropGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + self.saga_grad = algorithms.SAGAOptGradOpt( + self.data1, + grad=grad_inst, + prox=prox_inst, + cost=cost_inst, + ) + + self.dummy = Dummy() + self.dummy.cost = func_identity + self.setup._check_operator(self.dummy.cost) + + def tearDown(self): + """Unset test parameter values.""" + self.data1 = None + self.data2 = None + self.setup = None + self.fb_all_iter = None + self.fb1 = None + self.fb2 = None + self.gfb_all_iter = None + self.gfb1 = None + self.gfb2 = None + self.condat_all_iter = None + self.condat1 = None + self.condat2 = None + self.condat3 = None + self.pogm1 = None + self.pogm_all_iter = None + self.dummy = None + + def test_set_up(self): + """Test set_up.""" + npt.assert_raises(TypeError, self.setup._check_input_data, 1) + + npt.assert_raises(TypeError, self.setup._check_param, 1) + + npt.assert_raises(TypeError, self.setup._check_param_update, 1) + + def test_all_iter(self): + """Test if all opt run for all iterations.""" + opts = [ + self.fb_all_iter, + self.gfb_all_iter, + self.condat_all_iter, + self.pogm_all_iter, + ] + for opt in opts: + npt.assert_equal(opt.idx, self.max_iter - 1) + + def test_forward_backward(self): + """Test forward_backward.""" + npt.assert_array_equal( + self.fb1.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb2.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb3.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb4.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb5.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + npt.assert_array_equal( + self.fb6.x_final, + self.data1, + err_msg='Incorrect ForwardBackward result.', + ) + + def test_gen_forward_backward(self): + """Test gen_forward_backward.""" + npt.assert_array_equal( + self.gfb1.x_final, + self.data1, + err_msg='Incorrect GenForwardBackward result.', + ) + + npt.assert_array_equal( + self.gfb2.x_final, + self.data1, + err_msg='Incorrect GenForwardBackward result.', + ) + + npt.assert_array_equal( + self.gfb3.x_final, + self.data1, + err_msg='Incorrect GenForwardBackward result.', + ) + + npt.assert_equal( + self.gfb3.step_size, + 2, + err_msg='Incorrect step size.', + ) + + npt.assert_raises( + TypeError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=1, + ) + + npt.assert_raises( + ValueError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=[1], + ) + + npt.assert_raises( + ValueError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=[0.5, 0.5], + ) + + npt.assert_raises( + ValueError, + algorithms.GenForwardBackward, + self.data1, + self.dummy, + [self.dummy], + weights=[0.5], + ) + + def test_condat(self): + """Test gen_condat.""" + npt.assert_almost_equal( + self.condat1.x_final, + self.data1, + err_msg='Incorrect Condat result.', + ) + + npt.assert_almost_equal( + self.condat2.x_final, + self.data1, + err_msg='Incorrect Condat result.', + ) + + def test_pogm(self): + """Test pogm.""" + npt.assert_almost_equal( + self.pogm1.x_final, + self.data1, + err_msg='Incorrect POGM result.', + ) + + def test_ada_grad(self): + """Test ADA Gradient Descent.""" + self.ada_grad.iterate() + npt.assert_almost_equal( + self.ada_grad.x_final, + self.data1, + err_msg='Incorrect ADAGrad results.', + ) + + def test_adam_grad(self): + """Test ADAM Gradient Descent.""" + self.adam_grad.iterate() + npt.assert_almost_equal( + self.adam_grad.x_final, + self.data1, + err_msg='Incorrect ADAMGrad results.', + ) + + def test_momemtum_grad(self): + """Test Momemtum Gradient Descent.""" + self.momentum_grad.iterate() + npt.assert_almost_equal( + self.momentum_grad.x_final, + self.data1, + err_msg='Incorrect MomentumGrad results.', + ) + + def test_rmsprop_grad(self): + """Test RMSProp Gradient Descent.""" + self.rms_grad.iterate() + npt.assert_almost_equal( + self.rms_grad.x_final, + self.data1, + err_msg='Incorrect RMSPropGrad results.', + ) + + def test_saga_grad(self): + """Test SAGA Descent.""" + self.saga_grad.iterate() + npt.assert_almost_equal( + self.saga_grad.x_final, + self.data1, + err_msg='Incorrect SAGA Grad results.', + ) + + def test_vanilla_grad(self): + """Test Vanilla Gradient Descent.""" + self.vanilla_grad.iterate() + npt.assert_almost_equal( + self.vanilla_grad.x_final, + self.data1, + err_msg='Incorrect VanillaGrad results.', + ) diff --git a/modopt/tests/test_opt.py b/modopt/tests/test_opt.py index 3c33c948..713e802d 100644 --- a/modopt/tests/test_opt.py +++ b/modopt/tests/test_opt.py @@ -37,356 +37,6 @@ class Dummy(object): pass -class AlgorithmTestCase(TestCase): - """Test case for algorithms module.""" - - def setUp(self): - """Set test parameter values.""" - self.data1 = np.arange(9).reshape(3, 3).astype(float) - self.data2 = self.data1 + np.random.randn(*self.data1.shape) * 1e-6 - self.data3 = np.arange(9).reshape(3, 3).astype(float) + 1 - - grad_inst = gradient.GradBasic( - self.data1, - func_identity, - func_identity, - ) - - prox_inst = proximity.Positivity() - prox_dual_inst = proximity.IdentityProx() - linear_inst = linear.Identity() - reweight_inst = reweight.cwbReweight(self.data3) - cost_inst = cost.costObj([grad_inst, prox_inst, prox_dual_inst]) - self.setup = algorithms.SetUp() - self.max_iter = 20 - - self.fb_all_iter = algorithms.ForwardBackward( - self.data1, - grad=grad_inst, - prox=prox_inst, - cost=None, - auto_iterate=False, - beta_update=func_identity, - ) - self.fb_all_iter.iterate(self.max_iter) - - self.fb1 = algorithms.ForwardBackward( - self.data1, - grad=grad_inst, - prox=prox_inst, - beta_update=func_identity, - ) - - self.fb2 = algorithms.ForwardBackward( - self.data1, - grad=grad_inst, - prox=prox_inst, - cost=cost_inst, - lambda_update=None, - ) - - self.fb3 = algorithms.ForwardBackward( - self.data1, - grad=grad_inst, - prox=prox_inst, - beta_update=func_identity, - a_cd=3, - ) - - self.fb4 = algorithms.ForwardBackward( - self.data1, - grad=grad_inst, - prox=prox_inst, - beta_update=func_identity, - r_lazy=3, - p_lazy=0.7, - q_lazy=0.7, - ) - - self.fb5 = algorithms.ForwardBackward( - self.data1, - grad=grad_inst, - prox=prox_inst, - restart_strategy='adaptive', - xi_restart=0.9, - ) - - self.fb6 = algorithms.ForwardBackward( - self.data1, - grad=grad_inst, - prox=prox_inst, - restart_strategy='greedy', - xi_restart=0.9, - min_beta=1.0, - s_greedy=1.1, - ) - - self.gfb_all_iter = algorithms.GenForwardBackward( - self.data1, - grad=grad_inst, - prox_list=[prox_inst, prox_dual_inst], - cost=None, - auto_iterate=False, - gamma_update=func_identity, - beta_update=func_identity, - ) - self.gfb_all_iter.iterate(self.max_iter) - - self.gfb1 = algorithms.GenForwardBackward( - self.data1, - grad=grad_inst, - prox_list=[prox_inst, prox_dual_inst], - gamma_update=func_identity, - lambda_update=func_identity, - ) - - self.gfb2 = algorithms.GenForwardBackward( - self.data1, - grad=grad_inst, - prox_list=[prox_inst, prox_dual_inst], - cost=cost_inst, - ) - - self.gfb3 = algorithms.GenForwardBackward( - self.data1, - grad=grad_inst, - prox_list=[prox_inst, prox_dual_inst], - cost=cost_inst, - step_size=2, - ) - - self.condat_all_iter = algorithms.Condat( - self.data1, - self.data2, - grad=grad_inst, - prox=prox_inst, - cost=None, - prox_dual=prox_dual_inst, - sigma_update=func_identity, - tau_update=func_identity, - rho_update=func_identity, - auto_iterate=False, - ) - self.condat_all_iter.iterate(self.max_iter) - - self.condat1 = algorithms.Condat( - self.data1, - self.data2, - grad=grad_inst, - prox=prox_inst, - prox_dual=prox_dual_inst, - sigma_update=func_identity, - tau_update=func_identity, - rho_update=func_identity, - ) - - self.condat2 = algorithms.Condat( - self.data1, - self.data2, - grad=grad_inst, - prox=prox_inst, - prox_dual=prox_dual_inst, - linear=linear_inst, - cost=cost_inst, - reweight=reweight_inst, - ) - - self.condat3 = algorithms.Condat( - self.data1, - self.data2, - grad=grad_inst, - prox=prox_inst, - prox_dual=prox_dual_inst, - linear=Dummy(), - cost=cost_inst, - auto_iterate=False, - ) - - self.pogm_all_iter = algorithms.POGM( - u=self.data1, - x=self.data1, - y=self.data1, - z=self.data1, - grad=grad_inst, - prox=prox_inst, - auto_iterate=False, - cost=None, - ) - self.pogm_all_iter.iterate(self.max_iter) - - self.pogm1 = algorithms.POGM( - u=self.data1, - x=self.data1, - y=self.data1, - z=self.data1, - grad=grad_inst, - prox=prox_inst, - ) - - self.dummy = Dummy() - self.dummy.cost = func_identity - self.setup._check_operator(self.dummy.cost) - - def tearDown(self): - """Unset test parameter values.""" - self.data1 = None - self.data2 = None - self.setup = None - self.fb_all_iter = None - self.fb1 = None - self.fb2 = None - self.gfb_all_iter = None - self.gfb1 = None - self.gfb2 = None - self.condat_all_iter = None - self.condat1 = None - self.condat2 = None - self.condat3 = None - self.pogm1 = None - self.pogm_all_iter = None - self.dummy = None - - def test_set_up(self): - """Test set_up.""" - npt.assert_raises(TypeError, self.setup._check_input_data, 1) - - npt.assert_raises(TypeError, self.setup._check_param, 1) - - npt.assert_raises(TypeError, self.setup._check_param_update, 1) - - def test_all_iter(self): - """Test if all opt run for all iterations.""" - opts = [ - self.fb_all_iter, - self.gfb_all_iter, - self.condat_all_iter, - self.pogm_all_iter, - ] - for opt in opts: - npt.assert_equal(opt.idx, self.max_iter - 1) - - def test_forward_backward(self): - """Test forward_backward.""" - npt.assert_array_equal( - self.fb1.x_final, - self.data1, - err_msg='Incorrect ForwardBackward result.', - ) - - npt.assert_array_equal( - self.fb2.x_final, - self.data1, - err_msg='Incorrect ForwardBackward result.', - ) - - npt.assert_array_equal( - self.fb3.x_final, - self.data1, - err_msg='Incorrect ForwardBackward result.', - ) - - npt.assert_array_equal( - self.fb4.x_final, - self.data1, - err_msg='Incorrect ForwardBackward result.', - ) - - npt.assert_array_equal( - self.fb5.x_final, - self.data1, - err_msg='Incorrect ForwardBackward result.', - ) - - npt.assert_array_equal( - self.fb6.x_final, - self.data1, - err_msg='Incorrect ForwardBackward result.', - ) - - def test_gen_forward_backward(self): - """Test gen_forward_backward.""" - npt.assert_array_equal( - self.gfb1.x_final, - self.data1, - err_msg='Incorrect GenForwardBackward result.', - ) - - npt.assert_array_equal( - self.gfb2.x_final, - self.data1, - err_msg='Incorrect GenForwardBackward result.', - ) - - npt.assert_array_equal( - self.gfb3.x_final, - self.data1, - err_msg='Incorrect GenForwardBackward result.', - ) - - npt.assert_equal( - self.gfb3.step_size, - 2, - err_msg='Incorrect step size.', - ) - - npt.assert_raises( - TypeError, - algorithms.GenForwardBackward, - self.data1, - self.dummy, - [self.dummy], - weights=1, - ) - - npt.assert_raises( - ValueError, - algorithms.GenForwardBackward, - self.data1, - self.dummy, - [self.dummy], - weights=[1], - ) - - npt.assert_raises( - ValueError, - algorithms.GenForwardBackward, - self.data1, - self.dummy, - [self.dummy], - weights=[0.5, 0.5], - ) - - npt.assert_raises( - ValueError, - algorithms.GenForwardBackward, - self.data1, - self.dummy, - [self.dummy], - weights=[0.5], - ) - - def test_condat(self): - """Test gen_condat.""" - npt.assert_almost_equal( - self.condat1.x_final, - self.data1, - err_msg='Incorrect Condat result.', - ) - - npt.assert_almost_equal( - self.condat2.x_final, - self.data1, - err_msg='Incorrect Condat result.', - ) - - def test_pogm(self): - """Test pogm.""" - npt.assert_almost_equal( - self.pogm1.x_final, - self.data1, - err_msg='Incorrect POGM result.', - ) - class CostTestCase(TestCase): """Test case for cost module.""" From 011413b5289ad71ef3bf334fe1bb6484637ae8f1 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Thu, 9 Dec 2021 10:06:27 +0100 Subject: [PATCH 06/10] Docstrings improvements --- docs/source/refs.bib | 19 +++++ modopt/opt/algorithms/base.py | 4 ++ modopt/opt/algorithms/forward_backward.py | 26 +++++++ modopt/opt/algorithms/gradient_descent.py | 86 ++++++++++++++++++----- 4 files changed, 118 insertions(+), 17 deletions(-) diff --git a/docs/source/refs.bib b/docs/source/refs.bib index cac8159e..62ef87f0 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -175,3 +175,22 @@ @article{zou2005 journal = {Journal of the Royal Statistical Society Series B}, doi = {10.1111/j.1467-9868.2005.00527.x} } + +@article{ruder2017, + title = {An overview of gradient descent optimization algorithms}, + url = {http://arxiv.org/abs/1609.04747}, + journaltitle = {{arXiv}:1609.04747 [cs]}, + author = {Ruder, Sebastian}, + urldate = {2021-12-09}, + date = {2017-06-15}, + langid = {english}, + eprinttype = {arxiv}, + eprint = {1609.04747}, +} + +@article{defazio2014, + title = {{SAGA}: A Fast Incremental Gradient Method With Support for Non-Strongly Convex Composite Objectives}, + pages = {15}, + author = {Defazio, Aaron and Bach, Francis and Lacoste-Julien, Simon}, + langid = {english}, +} diff --git a/modopt/opt/algorithms/base.py b/modopt/opt/algorithms/base.py index ef2f1452..faa7fddb 100644 --- a/modopt/opt/algorithms/base.py +++ b/modopt/opt/algorithms/base.py @@ -34,6 +34,10 @@ class SetUp(Observable): use_gpu : bool, optional Option to use available GPU + See Also + -------- + modopt.base.observable.MetricObserver : + Definition of Metrics. """ def __init__( diff --git a/modopt/opt/algorithms/forward_backward.py b/modopt/opt/algorithms/forward_backward.py index 94287fd3..5c6fb404 100644 --- a/modopt/opt/algorithms/forward_backward.py +++ b/modopt/opt/algorithms/forward_backward.py @@ -322,6 +322,13 @@ class ForwardBackward(SetUp): The `beta_param` can also be set using the keyword `step_size`, which will override the value of `beta_param`. + The following state variable are available for metrics measurememts at + each iteration : + + * `x_new` : new estimate of x + * `z_new` : new estimate of z (adjoint representation of x). + * `idx` : index of the iteration. + See Also -------- FISTA : complementary class @@ -542,6 +549,13 @@ class GenForwardBackward(SetUp): The `gamma_param` can also be set using the keyword `step_size`, which will override the value of `gamma_param`. + The following state variable are available for metrics measurememts at + each iteration : + + * `x_new` : new estimate of x + * `z_new` : new estimate of z (adjoint representation of x). + * `idx` : index of the iteration. + See Also -------- SetUp : parent class @@ -815,6 +829,18 @@ class POGM(SetUp): The `beta_param` can also be set using the keyword `step_size`, which will override the value of `beta_param`. + The following state variable are available for metrics measurememts at + each iterations: + + * `u_new` : new estimate of u + * `x_new` : new estimate of x + * `y_new` : new estimate of y + * `z_new` : new estimate of z + * `xi`: xi variable + * `t` : new estimate of t + * `sigma`: sigma variable + * `idx` : index of the iteration. + See Also -------- SetUp : parent class diff --git a/modopt/opt/algorithms/gradient_descent.py b/modopt/opt/algorithms/gradient_descent.py index 2488c1fb..9ad33291 100644 --- a/modopt/opt/algorithms/gradient_descent.py +++ b/modopt/opt/algorithms/gradient_descent.py @@ -42,14 +42,28 @@ class GenericGradOpt(SetUp): ----- The Gradient descent step is defined as: - ..math:: x_{k+1} = x_k - \frac{\eta}{\sqrt{s_k + \epsilon}} m_k + .. math:: x_{k+1} = x_k - \frac{\eta}{\sqrt{s_k + \epsilon}} m_k + + where: + + * :math:`m_k` is the gradient direction + * :math:`\eta` is the gradient descent step + * :math: `s_k` is the gradient "speed" + At each Epoch, an optional Proximal step can be performed. + The following state variable are available for metrics measurememts: + + * `x_new` : new estimate of the iterations + * `dir_grad` : direction of the gradient descent step + * `speed_grad` : speed for the gradient descent step + * `idx` : index of the iteration being reconstructed. + See Also -------- - metric api - modopt.opt.algorithms.base.SetUp + modopt.opt.algorithms.base.SetUp : parent class + """ def __init__( @@ -124,8 +138,8 @@ def iterate(self, max_iter=150): def _update(self): self._grad.get_grad(self._x_old) - self.update_grad_dir(self._grad.grad) - self.update_grad_speed(self._grad.grad) + self._update_grad_dir(self._grad.grad) + self._update_grad_speed(self._grad.grad) step = self._eta / (np.sqrt(self._speed_grad) + self._eps) self._x_new = self._x_old - step * self._dir_grad if self.idx % self.epoch_size == 0: @@ -141,16 +155,35 @@ def _update(self): or self._cost_func.get_cost(self._x_new) ) - def update_grad_dir(self, grad): - """Update the gradient descent direction.""" + def _update_grad_dir(self, grad): + """Update the gradient descent direction. + + Parameters + ---------- + grad: ndarray + The gradien direction + """ self._dir_grad = grad - def update_grad_speed(self, grad): - """Update the gradient descent speed.""" + def _update_grad_speed(self, grad): + """Update the gradient descent speed. + + Parameters + ---------- + grad: ndarray + The gradien direction + + """ pass - def update_reg(self, factor): - """Regularisation step.""" + def _update_reg(self, factor): + """Regularisation step. + + Parameters + ---------- + factor: float or array_like + extra factor for the proximal step. + """ self._x_new = self._prox.op(self._x_new, extra_factor=factor) def get_notify_observers_kwargs(self): @@ -203,10 +236,26 @@ def __init__(self, *args, **kwargs): class AdaGenericGradOpt(GenericGradOpt): - """Generic Grad descent Algorithm with ADA acceleration scheme.""" + r"""Generic Grad descent Algorithm with ADA acceleration scheme. - def update_grad_speed(self, grad): - """Ada Acceleration Scheme.""" + + Notes + ----- + For AdaGrad [1]_ the gradient is preconditioned using a speed update: + .. math: s_k = \sum_{i=0}^k g_k * g_k + + .. [1] Section 4.2 https://arxiv.org/pdf/1609.04747.pdf + """ + + def _update_grad_speed(self, grad): + """Ada Acceleration Scheme. + + Parameters + ---------- + grad: ndarray + The new gradient for updating the speed. + + """ self._speed_grad += abs(grad) ** 2 @@ -225,7 +274,7 @@ class RMSpropGradOpt(GenericGradOpt): Notes ----- - The gradient speed is defined as : + The gradient speed of RMSProp (Section 4.5 of :cite:`ruder2017`) is defined as : .. math:: s_k = \gamma s_{k-1} + (1-\gamma) * |\nabla f|^2 @@ -255,7 +304,7 @@ class MomentumGradOpt(GenericGradOpt): Notes ----- - The momentum update is defined as: + The Momentum (Section 4.1 of :cite:`ruder2017` update is defined as: .. math:: m_k = \beta * m_{k-1} + \nabla f(x_k) """ @@ -294,12 +343,13 @@ class ADAMGradOpt(GenericGradOpt): Notes ----- - The ADAM optimizer is defined as: + The ADAM optimizer (Section 4.6 of :cite:`ruder2017` is defined as: .. math:: m_{k+1} = \frac{1}{1-\beta^k}(\beta*m_{k}+(1-\beta)*|\nabla f_k|^2) .. math:: s_{k+1} = \frac{1}{1-\gamma^k}(\gamma*s_k+(1-\gamma)*\nabla f_k) + """ def __init__(self, *args, gamma=0.9, beta=0.9, **kwargs): @@ -336,6 +386,8 @@ class SAGAOptGradOpt(GenericGradOpt): Notes ----- + Implements equation (7) of :cite:`defazio2014` + The stochastic part is not handled here, and should be implemented by changing the obs_data between each call to the _update function. """ From bd5b4d20f4e0e15a30b75ed79c3a2f09f0202e50 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Thu, 9 Dec 2021 10:20:07 +0100 Subject: [PATCH 07/10] Add See Also and minor corrections --- modopt/opt/algorithms/gradient_descent.py | 52 ++++++++++++++++------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/modopt/opt/algorithms/gradient_descent.py b/modopt/opt/algorithms/gradient_descent.py index 9ad33291..6b8b2c31 100644 --- a/modopt/opt/algorithms/gradient_descent.py +++ b/modopt/opt/algorithms/gradient_descent.py @@ -23,7 +23,7 @@ class GenericGradOpt(SetUp): Proximal operator, linear: Instance of OperatorBase Linear operator (the image domain should be sparse) - cost: + cost: Instance of costObj Cost Operator eta: float, default 1.0 Descent step @@ -48,7 +48,7 @@ class GenericGradOpt(SetUp): * :math:`m_k` is the gradient direction * :math:`\eta` is the gradient descent step - * :math: `s_k` is the gradient "speed" + * :math:`s_k` is the gradient "speed" At each Epoch, an optional Proximal step can be performed. @@ -116,7 +116,6 @@ def __init__( self.idx = 0 self.epoch_size = epoch_size - def iterate(self, max_iter=150): """Iterate. @@ -226,6 +225,10 @@ class VanillaGenericGradOpt(GenericGradOpt): """Vanilla Descent Algorithm. Fixed step size and no numerical precision threshold. + + See Also + -------- + GenericGradOpt : parent class """ def __init__(self, *args, **kwargs): @@ -238,13 +241,17 @@ def __init__(self, *args, **kwargs): class AdaGenericGradOpt(GenericGradOpt): r"""Generic Grad descent Algorithm with ADA acceleration scheme. - Notes ----- - For AdaGrad [1]_ the gradient is preconditioned using a speed update: - .. math: s_k = \sum_{i=0}^k g_k * g_k + For AdaGrad (Section 4.2 of :cite:`ruder2017`) the gradient is + preconditioned using a speed update: + + .. math:: s_k = \sum_{i=0}^k g_k * g_k + - .. [1] Section 4.2 https://arxiv.org/pdf/1609.04747.pdf + See Also + -------- + GenericGradOpt : parent class """ def _update_grad_speed(self, grad): @@ -274,10 +281,14 @@ class RMSpropGradOpt(GenericGradOpt): Notes ----- - The gradient speed of RMSProp (Section 4.5 of :cite:`ruder2017`) is defined as : + The gradient speed of RMSProp (Section 4.5 of :cite:`ruder2017`) is + defined as : .. math:: s_k = \gamma s_{k-1} + (1-\gamma) * |\nabla f|^2 + See Also + -------- + GenericGradOpt : parent class """ def __init__(self, *args, gamma=0.5, **kwargs): @@ -287,7 +298,7 @@ def __init__(self, *args, gamma=0.5, **kwargs): self._check_param(gamma) self._gamma = gamma - def update_grad_speed(self, grad): + def _update_grad_speed(self, grad): """Rmsprop update speed.""" self._speed_grad = ( self._gamma * self._speed_grad + (1 - self._gamma) * abs(grad) ** 2 @@ -307,6 +318,10 @@ class MomentumGradOpt(GenericGradOpt): The Momentum (Section 4.1 of :cite:`ruder2017` update is defined as: .. math:: m_k = \beta * m_{k-1} + \nabla f(x_k) + + See Also + -------- + GenericGradOpt : parent class """ def __init__(self, *args, beta=0.9, **kwargs): @@ -317,7 +332,7 @@ def __init__(self, *args, beta=0.9, **kwargs): self._speed_grad = 1.0 self._eps = 0 - def update_grad_dir(self, grad): + def _update_grad_dir(self, grad): """Momentum gradient direction update.""" self._dir_grad = self._beta * self._dir_grad + grad @@ -350,6 +365,9 @@ class ADAMGradOpt(GenericGradOpt): .. math:: s_{k+1} = \frac{1}{1-\gamma^k}(\gamma*s_k+(1-\gamma)*\nabla f_k) + See Also + -------- + GenericGradOpt : parent class """ def __init__(self, *args, gamma=0.9, beta=0.9, **kwargs): @@ -365,7 +383,7 @@ def __init__(self, *args, gamma=0.9, beta=0.9, **kwargs): self._beta_pow = 1 self._gamma_pow = 1 - def update_grad_dir(self, grad): + def _update_grad_dir(self, grad): """ADAM Update of gradient direction.""" self._beta_pow *= self._beta @@ -373,7 +391,7 @@ def update_grad_dir(self, grad): self._beta * self._dir_grad + (1 - self._beta) * grad ) - def update_grad_speed(self, grad): + def _update_grad_speed(self, grad): """ADAM Updatae of gradient speed.""" self._gamma_pow *= self._gamma self._speed_grad = (1.0 / (1.0 - self._gamma_pow)) * ( @@ -384,12 +402,16 @@ def update_grad_speed(self, grad): class SAGAOptGradOpt(GenericGradOpt): """SAGA optimizer. - Notes - ----- Implements equation (7) of :cite:`defazio2014` + Notes + ----- The stochastic part is not handled here, and should be implemented by changing the obs_data between each call to the _update function. + + See Also + -------- + GenericGradOpt : parent class """ def __init__(self, *args, **kwargs): @@ -399,7 +421,7 @@ def __init__(self, *args, **kwargs): dtype=self._x_old.dtype, ) - def update_grad_dir(self, grad): + def _update_grad_dir(self, grad): """SAGA Update gradient direction.""" cycle = self.idx % self.epoch_size self._dir_grad = self._dir_grad - self._grad_memory[cycle] + grad From 77495f318e604fd3dd549285967798d1ec7d576f Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Thu, 9 Dec 2021 14:57:37 +0100 Subject: [PATCH 08/10] add idx initialisation for all algorithms. --- modopt/opt/algorithms/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modopt/opt/algorithms/base.py b/modopt/opt/algorithms/base.py index faa7fddb..9191642e 100644 --- a/modopt/opt/algorithms/base.py +++ b/modopt/opt/algorithms/base.py @@ -50,7 +50,7 @@ def __init__( compute_backend='numpy', **dummy_kwargs, ): - + self.idx = 0 self.converge = False self.verbose = verbose self.progress = progress From f612ecbd0be7f08cd5c4e9c1b945676bf6ddf528 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Fri, 10 Dec 2021 10:25:13 +0100 Subject: [PATCH 09/10] fix merge error --- modopt/opt/algorithms/forward_backward.py | 277 ---------------------- 1 file changed, 277 deletions(-) diff --git a/modopt/opt/algorithms/forward_backward.py b/modopt/opt/algorithms/forward_backward.py index 1a75409f..5c6fb404 100644 --- a/modopt/opt/algorithms/forward_backward.py +++ b/modopt/opt/algorithms/forward_backward.py @@ -9,283 +9,6 @@ from modopt.opt.linear import Identity -class SetUp(Observable): - r"""Algorithm Set-Up. - - This class contains methods for checking the set-up of an optimisation - algotithm and produces warnings if they do not comply. - - Parameters - ---------- - metric_call_period : int, optional - Metric call period (default is ``5``) - metrics : dict, optional - Metrics to be used (default is ``\{\}``) - verbose : bool, optional - Option for verbose output (default is ``False``) - progress : bool, optional - Option to display progress bar (default is ``True``) - step_size : int, optional - Generic step size parameter to override default algorithm - parameter name (`e.g.` `step_size` will override the value set for - `beta_param` in `ForwardBackward`) - use_gpu : bool, optional - Option to use available GPU - - """ - - def __init__( - self, - metric_call_period=5, - metrics=None, - verbose=False, - progress=True, - step_size=None, - compute_backend='numpy', - **dummy_kwargs, - ): - - self.converge = False - self.verbose = verbose - self.progress = progress - self.metrics = metrics - self.step_size = step_size - self._op_parents = ( - 'GradParent', - 'ProximityParent', - 'LinearParent', - 'costObj', - ) - - self.metric_call_period = metric_call_period - - # Declaration of observers for metrics - super().__init__(['cv_metrics']) - - for name, dic in self.metrics.items(): - observer = MetricObserver( - name, - dic['metric'], - dic['mapping'], - dic['cst_kwargs'], - dic['early_stopping'], - ) - self.add_observer('cv_metrics', observer) - - xp, compute_backend = backend.get_backend(compute_backend) - self.xp = xp - self.compute_backend = compute_backend - - @property - def metrics(self): - """Metrics.""" - return self._metrics - - @metrics.setter - def metrics(self, metrics): - - if isinstance(metrics, type(None)): - self._metrics = {} - elif isinstance(metrics, dict): - self._metrics = metrics - else: - raise TypeError( - 'Metrics must be a dictionary, not {0}.'.format(type(metrics)), - ) - - def any_convergence_flag(self): - """Check convergence flag. - - Return if any matrices values matched the convergence criteria. - - Returns - ------- - bool - True if any convergence criteria met - - """ - return any( - obs.converge_flag for obs in self._observers['cv_metrics'] - ) - - def copy_data(self, input_data): - """Copy Data. - - Set directive for copying data. - - Parameters - ---------- - input_data : numpy.ndarray - Input data - - Returns - ------- - numpy.ndarray - Copy of input data - - """ - return self.xp.copy(backend.change_backend( - input_data, - self.compute_backend, - )) - - def _check_input_data(self, input_data): - """Check input data type. - - This method checks if the input data is a numpy array - - Parameters - ---------- - input_data : numpy.ndarray - Input data array - - Raises - ------ - TypeError - For invalid input type - - """ - if not (isinstance(input_data, (self.xp.ndarray, np.ndarray))): - raise TypeError( - 'Input data must be a numpy array or backend array', - ) - - def _check_param(self, param_val): - """Check algorithm parameters. - - This method checks if the specified algorithm parameters are floats - - Parameters - ---------- - param_val : float - Parameter value - - Raises - ------ - TypeError - For invalid input type - - """ - if not isinstance(param_val, float): - raise TypeError('Algorithm parameter must be a float value.') - - def _check_param_update(self, param_update): - """Check algorithm parameter update methods. - - This method checks if the specified algorithm parameters are floats - - Parameters - ---------- - param_update : function - Callable function - - Raises - ------ - TypeError - For invalid input type - - """ - param_conditions = ( - not isinstance(param_update, type(None)) - and not callable(param_update) - ) - - if param_conditions: - raise TypeError( - 'Algorithm parameter update must be a callabale function.', - ) - - def _check_operator(self, operator): - """Check set-Up. - - This method checks algorithm operator against the expected parent - classes - - Parameters - ---------- - operator : str - Algorithm operator to check - - """ - if not isinstance(operator, type(None)): - tree = [op_obj.__name__ for op_obj in getmro(operator.__class__)] - - if not any(parent in tree for parent in self._op_parents): - message = '{0} does not inherit an operator parent.' - warn(message.format(str(operator.__class__))) - - def _compute_metrics(self): - """Compute metrics during iteration. - - This method create the args necessary for metrics computation, then - call the observers to compute metrics - - """ - kwargs = self.get_notify_observers_kwargs() - self.notify_observers('cv_metrics', **kwargs) - - def _iterations(self, max_iter, progbar=None): - """Iterate method. - - Iterate the update step of the given algorithm. - - Parameters - ---------- - max_iter : int - Maximum number of iterations - progbar : progressbar.ProgressBar - Progress bar (default is ``None``) - - """ - for idx in range(max_iter): - self.idx = idx - self._update() - - # Calling metrics every metric_call_period cycle - # Also calculate at the end (max_iter or at convergence) - # We do not call metrics if metrics is empty or metric call - # period is None - if self.metrics and self.metric_call_period is not None: - - metric_conditions = ( - self.idx % self.metric_call_period == 0 - or self.idx == (max_iter - 1) - or self.converge, - ) - - if metric_conditions: - self._compute_metrics() - - if self.converge: - if self.verbose: - print(' - Converged!') - break - - if not isinstance(progbar, type(None)): - progbar.update(idx) - - def _run_alg(self, max_iter): - """Run algorithm. - - Run the update step of a given algorithm up to the maximum number of - iterations. - - Parameters - ---------- - max_iter : int - Maximum number of iterations - - """ - if self.progress: - with ProgressBar( - redirect_stdout=True, - max_value=max_iter, - ) as progbar: - self._iterations(max_iter, progbar=progbar) - else: - self._iterations(max_iter) - - class FISTA(object): """FISTA. From d3eb4a0ed98417d655ca401b0b2714e234def56f Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Fri, 10 Dec 2021 10:28:31 +0100 Subject: [PATCH 10/10] fix typo --- modopt/opt/algorithms/gradient_descent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modopt/opt/algorithms/gradient_descent.py b/modopt/opt/algorithms/gradient_descent.py index 6b8b2c31..8d526674 100644 --- a/modopt/opt/algorithms/gradient_descent.py +++ b/modopt/opt/algorithms/gradient_descent.py @@ -143,7 +143,7 @@ def _update(self): self._x_new = self._x_old - step * self._dir_grad if self.idx % self.epoch_size == 0: self.reset() - self.update_reg(step) + self._update_reg(step) self._x_old = self._x_new.copy() if self._eta_update is not None: self._eta = self._eta_update(self._eta, self.idx)