From 2fed7c33dc018433f077bb62a9a2da0da7e64c87 Mon Sep 17 00:00:00 2001 From: Abhishree Date: Tue, 10 Oct 2023 16:58:39 -0700 Subject: [PATCH 1/2] Add strategy=auto for None and dp Signed-off-by: Abhishree --- examples/nlp/dialogue/dialogue.py | 2 +- tutorials/asr/ASR_with_Subword_Tokenization.ipynb | 4 ++-- tutorials/asr/Speech_Commands.ipynb | 2 +- tutorials/asr/Voice_Activity_Detection.ipynb | 2 +- tutorials/nlp/Question_Answering.ipynb | 2 +- .../speaker_tasks/Speaker_Identification_Verification.ipynb | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/nlp/dialogue/dialogue.py b/examples/nlp/dialogue/dialogue.py index 0da5ae5717da..de91b60d1ed3 100644 --- a/examples/nlp/dialogue/dialogue.py +++ b/examples/nlp/dialogue/dialogue.py @@ -68,7 +68,7 @@ def main(cfg: DictConfig) -> None: try: strategy = NLPDDPStrategy(no_ddp_communication_hook=True, find_unused_parameters=True,) except (ImportError, ModuleNotFoundError): - strategy = None + strategy = 'auto' trainer = pl.Trainer(**cfg.trainer, strategy=strategy) diff --git a/tutorials/asr/ASR_with_Subword_Tokenization.ipynb b/tutorials/asr/ASR_with_Subword_Tokenization.ipynb index b8f72a47292b..8b74cd2fc721 100644 --- a/tutorials/asr/ASR_with_Subword_Tokenization.ipynb +++ b/tutorials/asr/ASR_with_Subword_Tokenization.ipynb @@ -1374,7 +1374,7 @@ "trainer = pl.Trainer(amp_level='O1', precision=16)\r\n", "\r\n", "# Trainer with a distributed backend:\r\n", - "trainer = pl.Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='dp')\r\n", + "trainer = pl.Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='auto')\r\n", "\r\n", "# Of course, you can combine these flags as well.\r\n", "```\r\n", @@ -1443,4 +1443,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/tutorials/asr/Speech_Commands.ipynb b/tutorials/asr/Speech_Commands.ipynb index 905ed1987410..5295af81e3ee 100644 --- a/tutorials/asr/Speech_Commands.ipynb +++ b/tutorials/asr/Speech_Commands.ipynb @@ -646,7 +646,7 @@ "\n", "```python\n", "# Trainer with a distributed backend:\n", - "trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='dp')\n", + "trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='auto')\n", "\n", "# Mixed precision:\n", "trainer = Trainer(amp_level='O1', precision=16)\n", diff --git a/tutorials/asr/Voice_Activity_Detection.ipynb b/tutorials/asr/Voice_Activity_Detection.ipynb index d56965f0c0ac..06d3cde671c4 100644 --- a/tutorials/asr/Voice_Activity_Detection.ipynb +++ b/tutorials/asr/Voice_Activity_Detection.ipynb @@ -678,7 +678,7 @@ "\n", "```python\n", "# Trainer with a distributed backend:\n", - "trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='dp')\n", + "trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='auto')\n", "\n", "# Mixed precision:\n", "trainer = Trainer(amp_level='O1', precision=16)\n", diff --git a/tutorials/nlp/Question_Answering.ipynb b/tutorials/nlp/Question_Answering.ipynb index 5d353dd76074..9ebbbd819790 100644 --- a/tutorials/nlp/Question_Answering.ipynb +++ b/tutorials/nlp/Question_Answering.ipynb @@ -357,7 +357,7 @@ "config.trainer.precision = 16\n", "config.trainer.devices = [0] # 0 for CPU, or list of the GPUs to use [0] this tutorial does not support multiple GPUs. If needed please use NeMo/examples/nlp/question_answering/question_answering.py\n", "config.trainer.accelerator = \"gpu\"\n", - "config.trainer.strategy=\"dp\"" + "config.trainer.strategy=\"auto\"" ] }, { diff --git a/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb b/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb index 54b66ec16371..2188714e92f0 100644 --- a/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb +++ b/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb @@ -637,7 +637,7 @@ "We can dramatically improve the time taken to train this model by using Multi GPU training along with Mixed Precision.\n", "\n", "### Trainer with a distributed backend:\n", - "
trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='dp')\n",
+    "
trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='auto')\n",
     "
\n",
     "\n",
     "### Mixed precision:\n",

From befe147bf8c016fa149cac7d99ddf1222ae73a9b Mon Sep 17 00:00:00 2001
From: Abhishree 
Date: Wed, 11 Oct 2023 10:20:08 -0700
Subject: [PATCH 2/2] Change strategy from None to auto

Signed-off-by: Abhishree 
---
 .../nlp/text_classification/text_classification_with_bert.py    | 2 +-
 examples/nlp/token_classification/token_classification_train.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/nlp/text_classification/text_classification_with_bert.py b/examples/nlp/text_classification/text_classification_with_bert.py
index 2deb3b34c444..01e8fae9bba5 100644
--- a/examples/nlp/text_classification/text_classification_with_bert.py
+++ b/examples/nlp/text_classification/text_classification_with_bert.py
@@ -111,7 +111,7 @@ def main(cfg: DictConfig) -> None:
     try:
         strategy = NLPDDPStrategy(find_unused_parameters=True)
     except (ImportError, ModuleNotFoundError):
-        strategy = None
+        strategy = 'auto'
 
     trainer = pl.Trainer(strategy=strategy, **cfg.trainer)
     exp_manager(trainer, cfg.get("exp_manager", None))
diff --git a/examples/nlp/token_classification/token_classification_train.py b/examples/nlp/token_classification/token_classification_train.py
index 51983a1af98b..56c1487cf9c5 100644
--- a/examples/nlp/token_classification/token_classification_train.py
+++ b/examples/nlp/token_classification/token_classification_train.py
@@ -105,7 +105,7 @@ def main(cfg: DictConfig) -> None:
     try:
         strategy = NLPDDPStrategy(find_unused_parameters=True)
     except (ImportError, ModuleNotFoundError):
-        strategy = None
+        strategy = 'auto'
 
     trainer = pl.Trainer(strategy=strategy, **cfg.trainer)
     exp_manager(trainer, cfg.get("exp_manager", None))