Magpietts 2508 merge main

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+blank_issues_enabled: false
+

.github/workflows/cherry-pick-release-commit.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   cherry-pick:
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cherry_pick.yml@v0.22.7
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cherry_pick.yml@v0.63.0
     secrets:
       PAT: ${{ secrets.PAT }}
       SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}

.github/workflows/cicd-main-nemo2.yml

@@ -143,6 +143,8 @@ jobs:
             runner: self-hosted-azure
           - script: L2_NeMo_2_GPT_LoRA_TP1PP1_MBS1_Chat
             runner: self-hosted-azure
+          - script: L2_NeMo_2_GPT_LoRA_TP1PP1_MBS1_TE_op_fuser
+            runner: self-hosted-azure
           - script: L2_NeMo_2_Mixtral_LoRA_EP2PP1_MBS2_exclude
             runner: self-hosted-azure
           - script: L2_NeMo_2_Mixtral_LoRA_EP2PP1_MBS2
@@ -281,6 +283,7 @@ jobs:
             script: L2_NeMo_2_Flux_ControlNet_Training_DDP_Test
           - runner: self-hosted-azure
             script: L2_NeMo_2_Flux_ControlNet_Training_FSDP_Test
+            is-optional: true
 
 
     needs: [build]

.github/workflows/cicd-main.yml

@@ -62,7 +62,7 @@ jobs:
           if [[ "$EVENT_NAME" == "pull_request" ]]; then
             python .github/scripts/components_to_run.py --source-sha ${{ github.event.pull_request.head.sha }} --target-sha ${{ github.event.pull_request.base.sha }}
           else
-            echo '["nemo2", "automodel", "export-deploy", "speech"]' | tee -a test_modules.json
+            echo '["nemo2", "export-deploy", "speech"]' | tee -a test_modules.json
           fi
 
           components_to_run=$(cat test_modules.json)
@@ -272,27 +272,6 @@ jobs:
     with:
       test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
 
-  cicd-main-automodel:
-    needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
-    uses: ./.github/workflows/cicd-main-automodel.yml
-    if: |
-      (
-        needs.pre-flight.outputs.test_to_run != '[]'
-        && (
-          contains(fromJson(needs.pre-flight.outputs.components_to_run), 'automodel')
-        )
-      )
-      && (
-        success()
-        || (
-          needs.cicd-wait-in-queue.result == 'skipped'
-          && needs.pre-flight.outputs.is_ci_workload == 'true'
-        )
-      )
-      && !cancelled()
-    with:
-      test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
-
   # cicd-main-nemo2:  disabled for magpie dev banch - no L0s and skipping L2s
   #   needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
   #   uses: ./.github/workflows/cicd-main-nemo2.yml
@@ -322,8 +301,7 @@ jobs:
       - cicd-import-tests
       - L0_Setup_Test_Data_And_Models
       - cicd-main-unit-tests
-      # - cicd-main-nemo2. not needed for magpie dev branch
-      - cicd-main-automodel
+      # - cicd-main-nemo2  not needed for magpie dev branch
       - cicd-main-speech
     if: always()
     runs-on: ubuntu-latest

.github/workflows/community-bot.yml

@@ -0,0 +1,15 @@
+name: Community Bot
+
+on:
+  issues:
+    types: [opened, edited, reopened, closed, deleted]
+  issue_comment:
+    types: [created, edited, deleted]
+
+jobs:
+  community-bot:
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_community_bot.yml@v0.62.0
+    with:
+      community_project_id: ${{ vars.COMMUNITY_PROJECT_ID }}
+    secrets:
+      GH_TOKEN: ${{ secrets.PAT }}

.github/workflows/install-test.yml

@@ -23,6 +23,22 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v2
 
+      - name: Check disk space before cleanup
+        run: df -h
+
+      - name: Free up disk space
+        run: |
+          # Remove unnecessary files on macOS
+          sudo rm -rf /usr/local/lib/android || true
+          sudo rm -rf /usr/local/.ghcup || true
+          sudo rm -rf /usr/local/lib/node_modules || true
+          brew cleanup || true
+          # Clear pip cache
+          pip cache purge || true
+
+      - name: Check disk space after cleanup
+        run: df -h
+
       - uses: actions/setup-python@v5
         with:
           python-version: "${{ matrix.python }}"
@@ -41,10 +57,13 @@ jobs:
             export NEMO_REPO
             export INSTALL_DIR=$(pwd)
 
-            bash docker/common/install_dep.sh --library all --mode install
+            bash docker/common/install_dep.sh --library "te,mcore,extra" --mode install
             pip install --no-cache-dir ".[all]"
           fi
 
+      - name: Check disk space after installation
+        run: df -h
+
       - name: Run import checks
         run: |
           # Run import checks
@@ -64,6 +83,25 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v2
 
+      - name: Check disk space before cleanup
+        run: df -h
+
+      - name: Free up disk space
+        run: |
+          # Remove unnecessary packages and files on Ubuntu
+          sudo apt-get clean
+          sudo rm -rf /usr/local/lib/android || true
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf /usr/local/.ghcup || true
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /opt/az || true
+          # Clear pip and npm caches
+          pip cache purge || true
+          sudo npm cache clean --force || true
+
+      - name: Check disk space after cleanup
+        run: df -h
+
       - name: Install Python
         uses: actions/setup-python@v5
         with:
@@ -74,14 +112,17 @@ jobs:
           INSTALLER: ${{ matrix.installer }}
         run: |
           if [ "$INSTALLER" = "pip-install" ]; then
-            pip install --upgrade pip
-            pip install ".[all]"
+            pip install --no-cache-dir --upgrade pip
+            pip install --no-cache-dir ".[all]"
           else
             export INSTALL_DIR=$(pwd)
-            bash docker/common/install_dep.sh --library all --mode install
+            bash docker/common/install_dep.sh --library "te,mcore,extra" --mode install
             pip install --no-cache-dir ".[all]"
           fi
 
+      - name: Check disk space after installation
+        run: df -h
+
       - name: Run import checks
         run: |
           # Run import checks
@@ -101,6 +142,25 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v2
 
+      - name: Check disk space before cleanup
+        run: df -h
+
+      - name: Free up disk space
+        run: |
+          # Remove unnecessary packages and files on Ubuntu ARM
+          sudo apt-get clean
+          sudo rm -rf /usr/local/lib/android || true
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf /usr/local/.ghcup || true
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /opt/az || true
+          # Clear pip and npm caches
+          pip cache purge || true
+          sudo npm cache clean --force || true
+
+      - name: Check disk space after cleanup
+        run: df -h
+
       - name: Install Python
         uses: actions/setup-python@v5
         with:
@@ -111,14 +171,17 @@ jobs:
           INSTALLER: ${{ matrix.installer }}
         run: |
           if [ "$INSTALLER" = "pip-install" ]; then
-            pip install --upgrade pip
-            pip install -vvv ".[all]"
+            pip install --no-cache-dir --upgrade pip
+            pip install --no-cache-dir ".[all]"
           else
             export INSTALL_DIR=$(pwd)
-            bash docker/common/install_dep.sh --library all --mode install
+            bash docker/common/install_dep.sh --library "te,mcore,extra" --mode install
             pip install --no-cache-dir ".[all]"
           fi
 
+      - name: Check disk space after installation
+        run: df -h
+
       - name: Run import checks
         run: |
           # Run import checks

.gitignore

@@ -181,3 +181,8 @@ examples/neural_graphs/*.yml
 nemo_experiments/
 
 slurm*.out
+
+# voice agent
+node_modules/
+.vite/
+bot_server.*