diff --git a/.github/workflows/hpu-gaudi2.yml b/.github/workflows/hpu-gaudi2.yml
index 4e9ceb32b6b1..46d714b8f902 100644
--- a/.github/workflows/hpu-gaudi2.yml
+++ b/.github/workflows/hpu-gaudi2.yml
@@ -111,6 +111,7 @@ jobs:
         run: |
           git clone https://github.com/huggingface/transformers
           cd transformers
+          git checkout v4.44.2 # Pin to specific version before we can update torch version to 2.4+ for nn.RMSNorm
           git rev-parse --short HEAD
           pip install .