Merge r1.13.0 main

README.rst

@@ -224,7 +224,7 @@ Install it manually if not using the NVIDIA PyTorch container.
 
     git clone https://github.com/ericharper/apex.git
     cd apex
-    git checkout nm_v1.11.0
+    git checkout nm_v1.13.0
     pip install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" --global-option="--fast_layer_norm" --global-option="--distributed_adam" --global-option="--deprecated_fused_adam" ./
 
 Transformer Engine

examples/nlp/language_modeling/conf/megatron_gpt_config.yaml

@@ -106,7 +106,6 @@ model:
   apex_transformer_log_level: 30 # Python logging level displays logs with severity greater than or equal to this
   gradient_as_bucket_view: True # PyTorch DDP argument. Allocate gradients in a contiguous bucket to save memory (less fragmentation and buffer memory)
   sync_batch_comm: False # Enable stream synchronization after each p2p communication between pipeline stages
-  use_unified_checkpoint: True # Use model parallel independent checkpointing
 
   ## Activation Checkpointing
   # NeMo Megatron supports 'selective' activation checkpointing where only the memory intensive part of attention is checkpointed.

examples/nlp/language_modeling/megatron_gpt_prompt_learning.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import torch.multiprocessing as mp
+from lightning_lite.plugins.environments import TorchElasticEnvironment
 from omegaconf.omegaconf import OmegaConf, open_dict
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks.timer import Timer

examples/nlp/language_modeling/megatron_t5_prompt_learning.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import torch.multiprocessing as mp
+from lightning_lite.plugins.environments import TorchElasticEnvironment
 from omegaconf.omegaconf import OmegaConf, open_dict
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks.timer import Timer

tests/collections/tts/test_tts_exportables.py

@@ -15,7 +15,6 @@
 import tempfile
 
 import pytest
-import torch
 from omegaconf import OmegaConf
 
 from nemo.collections.tts.models import FastPitchModel, HifiGanModel, RadTTSModel
@@ -81,5 +80,4 @@ def test_RadTTSModel_export_to_torchscript(self, radtts_model):
         model = radtts_model.cuda()
         with tempfile.TemporaryDirectory() as tmpdir:
             filename = os.path.join(tmpdir, 'rad.ts')
-            with torch.cuda.amp.autocast(enabled=True):
-                model.export(output=filename, verbose=True, check_trace=True)
+            model.export(output=filename, verbose=True, check_trace=True)