diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index 07c459d61cf056..f661513cd66558 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -336,6 +336,7 @@ public void replayEndLoadJob(LoadJobFinalOperation operation) {
         LOG.info(new LogBuilder(LogKey.LOAD_JOB, operation.getId()).add("operation", operation)
                 .add("msg", "replay end load job").build());
 
+        Env.getCurrentGlobalTransactionMgr().getCallbackFactory().removeCallback(operation.getId());
         // When idToLoadJob size increase 10000 roughly, we run removeOldLoadJob to reduce mem used
         if ((idToLoadJob.size() > 0) && (idToLoadJob.size() % 10000 == 0)) {
             removeOldLoadJob();