From bf1d0e3968defd6780f04bae848421df13af68ba Mon Sep 17 00:00:00 2001 From: Calogero Zarbo Date: Fri, 20 Mar 2020 11:20:49 +0100 Subject: [PATCH 1/2] removed restrictions for custom optimizer --- deepspeed/pt/deepspeed_light.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deepspeed/pt/deepspeed_light.py b/deepspeed/pt/deepspeed_light.py index abbb5cd339ea..2f5b3c18e7ef 100755 --- a/deepspeed/pt/deepspeed_light.py +++ b/deepspeed/pt/deepspeed_light.py @@ -441,7 +441,11 @@ def _configure_optimizer(self, client_optimizer, model_parameters): logging.info('DeepSpeed Basic Optimizer = {}'.format(basic_optimizer)) - if self.zero_optimization() and self.optimizer_name() == ADAM_OPTIMIZER: + if self.zero_optimization(): + if self.optimizer_name != ADAM_OPTIMIZER: + logging.warning( + "**** You are using ZeRO with an untested optimizer, proceed with caution *****" + ) self.optimizer = self._configure_zero_optimizer(basic_optimizer) elif self.fp16_enabled(): self.optimizer = self._configure_fp16_optimizer(basic_optimizer) From 2f07f4129e25b1c0c9211a2c7c2112d5732f7e77 Mon Sep 17 00:00:00 2001 From: Calogero Zarbo Date: Fri, 20 Mar 2020 11:51:07 +0100 Subject: [PATCH 2/2] fixed yapf missformatting --- deepspeed/pt/deepspeed_light.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/pt/deepspeed_light.py b/deepspeed/pt/deepspeed_light.py index 2f5b3c18e7ef..740727c837e5 100755 --- a/deepspeed/pt/deepspeed_light.py +++ b/deepspeed/pt/deepspeed_light.py @@ -445,7 +445,7 @@ def _configure_optimizer(self, client_optimizer, model_parameters): if self.optimizer_name != ADAM_OPTIMIZER: logging.warning( "**** You are using ZeRO with an untested optimizer, proceed with caution *****" - ) + ) self.optimizer = self._configure_zero_optimizer(basic_optimizer) elif self.fp16_enabled(): self.optimizer = self._configure_fp16_optimizer(basic_optimizer)