diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index b5c07bebb86b..abfac77418c2 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -2357,7 +2357,7 @@ def _inner_training_loop(
                     model = self.accelerator.prepare(self.model)
                 else:
                     if delay_optimizer_creation:
-                        self.optimizer = self.accelerator.prepare(self.optimizer)
+                        model = self.accelerator.prepare(self.model)
                     else:
                         model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
             else: