From f90b4b7f0de8b0f1814b513e1d9e85dbd9cf5f5f Mon Sep 17 00:00:00 2001 From: Cheng Li Date: Sat, 6 Mar 2021 00:56:08 +0000 Subject: [PATCH 1/4] Remove param_group that does not have 'params' in the client optimizer passed into deepspeed engine --- deepspeed/runtime/engine.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 716f73d3b469..6b81fc0d2e2b 100755 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -544,6 +544,11 @@ def _configure_distributed_model(self, model): def _configure_optimizer(self, client_optimizer, model_parameters): if client_optimizer is not None: + for i, param_group in enumerate(client_optimizer.param_groups): + if len(param_group["params"]) == 0: + client_optimizer.param_groups.pop(i) + logger.info("Removing param_group {} in the client optimizer since it has no 'params'".format(i)) + basic_optimizer = client_optimizer if self.global_rank == 0: logger.info('Using client Optimizer as basic optimizer') From 9053c9e5f987a2ada39dcd57ecdfa05a01c88088 Mon Sep 17 00:00:00 2001 From: Cheng Li Date: Sat, 6 Mar 2021 01:44:56 +0000 Subject: [PATCH 2/4] fix typo --- deepspeed/runtime/engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 6b81fc0d2e2b..8dce818cb9a5 100755 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -547,7 +547,9 @@ def _configure_optimizer(self, client_optimizer, model_parameters): for i, param_group in enumerate(client_optimizer.param_groups): if len(param_group["params"]) == 0: client_optimizer.param_groups.pop(i) - logger.info("Removing param_group {} in the client optimizer since it has no 'params'".format(i)) + logger.info( + "Removing param_group {} in the client Optimizer since it has no 'params'" + .format(i)) basic_optimizer = client_optimizer if self.global_rank == 0: From 8f7469ed6b62e766f303e6f868a5c1c1c4a8e741 Mon Sep 17 00:00:00 2001 From: Cheng Li Date: Sat, 6 Mar 2021 02:46:42 +0000 Subject: [PATCH 3/4] fix printing --- deepspeed/runtime/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 8dce818cb9a5..41a75e7ee78c 100755 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -548,8 +548,8 @@ def _configure_optimizer(self, client_optimizer, model_parameters): if len(param_group["params"]) == 0: client_optimizer.param_groups.pop(i) logger.info( - "Removing param_group {} in the client Optimizer since it has no 'params'" - .format(i)) + "Removing param_group that has no 'params'in the client Optimizer" + ) basic_optimizer = client_optimizer if self.global_rank == 0: From 7b64a9689607db634f0be823d7ae5fb1ade129f6 Mon Sep 17 00:00:00 2001 From: Cheng Li Date: Sun, 7 Mar 2021 21:24:59 +0000 Subject: [PATCH 4/4] fix bug --- deepspeed/runtime/engine.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 41a75e7ee78c..005618800eb5 100755 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -544,12 +544,11 @@ def _configure_distributed_model(self, model): def _configure_optimizer(self, client_optimizer, model_parameters): if client_optimizer is not None: - for i, param_group in enumerate(client_optimizer.param_groups): - if len(param_group["params"]) == 0: - client_optimizer.param_groups.pop(i) - logger.info( - "Removing param_group that has no 'params'in the client Optimizer" - ) + client_optimizer.param_groups[:] = [ + pg for pg in client_optimizer.param_groups if len(pg["params"]) != 0 + ] + logger.info( + "Removing param_group that has no 'params'in the client Optimizer") basic_optimizer = client_optimizer if self.global_rank == 0: