From cc67e3ad43e6dd49b6b720d75256cb0d5e5f8248 Mon Sep 17 00:00:00 2001 From: Zaccharie Ramzi Date: Wed, 16 Dec 2020 16:26:09 +0100 Subject: [PATCH 1/5] added fista with restarting strategies to the solvers --- solvers/modopt_fista.py | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 solvers/modopt_fista.py diff --git a/solvers/modopt_fista.py b/solvers/modopt_fista.py new file mode 100644 index 00000000..b09af86d --- /dev/null +++ b/solvers/modopt_fista.py @@ -0,0 +1,68 @@ +import numpy as np + +from benchopt import BaseSolver +from benchopt import safe_import_context + + +with safe_import_context() as import_ctx: + from modopt.opt.algorithms import ForwardBackward + from modopt.opt.proximity import SparseThreshold + from modopt.opt.linear import Identity + from modopt.opt.gradient import GradBasic + +class Solver(BaseSolver): + name = 'ModOpt-FISTA' + stop_strategy = 'iteration' + + install_cmd = 'conda' + requirements = [ + 'pip:git+https://github.com/CEA-COSMIC/ModOpt.git', + ] + + parameters = { + 'restart_strategy': ['greedy', 'adaptive-1'], + + } + + def set_objective(self, X, y, lmbd): + self.X, self.y, self.lmbd = X, y, lmbd + n_features = self.X.shape[1] + L = np.linalg.norm(self.X, ord=2) ** 2 + if self.restart_strategy == 'greedy': + beta_param = 1.3 * (1/L) + min_beta = 1.0 + s_greedy = 1.1 + p_lazy = 1.0 + q_lazy = 1.0 + else: + beta_param = 1 / L + min_beta = None + s_greedy = None + p_lazy = 1 / 30 + q_lazy = 1 / 10 + self.fb = ForwardBackward( + x=np.zeros(n_features), # this is the coefficient w + grad=GradBasic( + op=lambda w: self.X@w, + trans_op=lambda res: self.X.T@res, + data=y, + ), + prox=SparseThreshold(Identity(), lmbd), + beta_param=beta_param, + min_beta=min_beta, + metric_call_period=None, + restart_strategy=self.restart_strategy, + xi_restart=0.96, + s_greedy=s_greedy, + p_lazy=p_lazy, + q_lazy=q_lazy, + auto_iterate=False, + progress=False, + cost=None, + ) + + def run(self, n_iter): + self.fb.iterate(max_iter=n_iter) + + def get_result(self): + return self.fb.x_final From 91757b1d3c716bf4aa41fd459b6629e04d04f7cc Mon Sep 17 00:00:00 2001 From: Zaccharie Ramzi Date: Wed, 16 Dec 2020 16:32:14 +0100 Subject: [PATCH 2/5] removed unnecessary whiteline --- solvers/modopt_fista.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solvers/modopt_fista.py b/solvers/modopt_fista.py index b09af86d..d6aba9a4 100644 --- a/solvers/modopt_fista.py +++ b/solvers/modopt_fista.py @@ -21,7 +21,6 @@ class Solver(BaseSolver): parameters = { 'restart_strategy': ['greedy', 'adaptive-1'], - } def set_objective(self, X, y, lmbd): From 3033dcb3da9a1b22cc373d2e96a925c2af1619cf Mon Sep 17 00:00:00 2001 From: Zaccharie Ramzi Date: Wed, 16 Dec 2020 16:32:28 +0100 Subject: [PATCH 3/5] added pogm prime to the solvers --- solvers/modopt_pogm.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 solvers/modopt_pogm.py diff --git a/solvers/modopt_pogm.py b/solvers/modopt_pogm.py new file mode 100644 index 00000000..413d3920 --- /dev/null +++ b/solvers/modopt_pogm.py @@ -0,0 +1,52 @@ +import numpy as np + +from benchopt import BaseSolver +from benchopt import safe_import_context + + +with safe_import_context() as import_ctx: + from modopt.opt.algorithms import POGM + from modopt.opt.proximity import SparseThreshold + from modopt.opt.linear import Identity + from modopt.opt.gradient import GradBasic + +class Solver(BaseSolver): + name = 'ModOpt-POGM' + stop_strategy = 'iteration' + + install_cmd = 'conda' + requirements = [ + 'pip:git+https://github.com/CEA-COSMIC/ModOpt.git', + ] + + def set_objective(self, X, y, lmbd): + self.X, self.y, self.lmbd = X, y, lmbd + n_features = self.X.shape[1] + L = np.linalg.norm(self.X, ord=2) ** 2 + beta_param = 1 / L + sigma_bar = 0.96 + var_init = np.zeros(n_features) + self.pogm = POGM( + x=var_init, # this is the coefficient w + u=var_init, + y=var_init, + z=var_init, + grad=GradBasic( + op=lambda w: self.X@w, + trans_op=lambda res: self.X.T@res, + data=y, + ), + prox=SparseThreshold(Identity(), lmbd), + beta_param=beta_param, + metric_call_period=None, + sigma_bar=sigma_bar, + auto_iterate=False, + progress=False, + cost=None, + ) + + def run(self, n_iter): + self.pogm.iterate(max_iter=n_iter) + + def get_result(self): + return self.pogm.x_final From 74df18c93ac2e58c8536f208046fa97e63471464 Mon Sep 17 00:00:00 2001 From: Zaccharie Ramzi Date: Wed, 16 Dec 2020 16:43:57 +0100 Subject: [PATCH 4/5] corrected pep8 --- solvers/modopt_fista.py | 1 + solvers/modopt_pogm.py | 1 + 2 files changed, 2 insertions(+) diff --git a/solvers/modopt_fista.py b/solvers/modopt_fista.py index d6aba9a4..c9356085 100644 --- a/solvers/modopt_fista.py +++ b/solvers/modopt_fista.py @@ -10,6 +10,7 @@ from modopt.opt.linear import Identity from modopt.opt.gradient import GradBasic + class Solver(BaseSolver): name = 'ModOpt-FISTA' stop_strategy = 'iteration' diff --git a/solvers/modopt_pogm.py b/solvers/modopt_pogm.py index 413d3920..3b7d3dd2 100644 --- a/solvers/modopt_pogm.py +++ b/solvers/modopt_pogm.py @@ -10,6 +10,7 @@ from modopt.opt.linear import Identity from modopt.opt.gradient import GradBasic + class Solver(BaseSolver): name = 'ModOpt-POGM' stop_strategy = 'iteration' From c821c51e60a821c4e3c65c2deaac872110cfda8b Mon Sep 17 00:00:00 2001 From: Zaccharie Ramzi Date: Mon, 21 Dec 2020 11:19:40 +0100 Subject: [PATCH 5/5] made solvers compute norm in run rather than set obj --- solvers/modopt_fista.py | 12 ++++++++---- solvers/modopt_pogm.py | 7 ++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/solvers/modopt_fista.py b/solvers/modopt_fista.py index c9356085..eab63f0d 100644 --- a/solvers/modopt_fista.py +++ b/solvers/modopt_fista.py @@ -27,15 +27,12 @@ class Solver(BaseSolver): def set_objective(self, X, y, lmbd): self.X, self.y, self.lmbd = X, y, lmbd n_features = self.X.shape[1] - L = np.linalg.norm(self.X, ord=2) ** 2 if self.restart_strategy == 'greedy': - beta_param = 1.3 * (1/L) min_beta = 1.0 s_greedy = 1.1 p_lazy = 1.0 q_lazy = 1.0 else: - beta_param = 1 / L min_beta = None s_greedy = None p_lazy = 1 / 30 @@ -48,7 +45,7 @@ def set_objective(self, X, y, lmbd): data=y, ), prox=SparseThreshold(Identity(), lmbd), - beta_param=beta_param, + beta_param=1.0, min_beta=min_beta, metric_call_period=None, restart_strategy=self.restart_strategy, @@ -62,6 +59,13 @@ def set_objective(self, X, y, lmbd): ) def run(self, n_iter): + L = np.linalg.norm(self.X, ord=2) ** 2 + if self.restart_strategy == 'greedy': + beta_param = 1.3 * (1/L) + else: + beta_param = 1 / L + self.fb.beta_param = beta_param + self.fb._beta = self.fb.step_size or beta_param self.fb.iterate(max_iter=n_iter) def get_result(self): diff --git a/solvers/modopt_pogm.py b/solvers/modopt_pogm.py index 3b7d3dd2..c2ad7cba 100644 --- a/solvers/modopt_pogm.py +++ b/solvers/modopt_pogm.py @@ -23,8 +23,6 @@ class Solver(BaseSolver): def set_objective(self, X, y, lmbd): self.X, self.y, self.lmbd = X, y, lmbd n_features = self.X.shape[1] - L = np.linalg.norm(self.X, ord=2) ** 2 - beta_param = 1 / L sigma_bar = 0.96 var_init = np.zeros(n_features) self.pogm = POGM( @@ -38,7 +36,7 @@ def set_objective(self, X, y, lmbd): data=y, ), prox=SparseThreshold(Identity(), lmbd), - beta_param=beta_param, + beta_param=1.0, metric_call_period=None, sigma_bar=sigma_bar, auto_iterate=False, @@ -47,6 +45,9 @@ def set_objective(self, X, y, lmbd): ) def run(self, n_iter): + L = np.linalg.norm(self.X, ord=2) ** 2 + beta_param = 1 / L + self.pogm._beta = self.pogm.step_size or beta_param self.pogm.iterate(max_iter=n_iter) def get_result(self):