diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
index b3d3b1b06b7cb7..8b8ff0d2085ef1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
@@ -659,6 +659,8 @@ private void executeFinish() {
         if (MetricRepo.isInit) {
             MetricRepo.COUNTER_LOAD_FINISHED.increase(1L);
         }
+        // when load job finished, there is no need to hold the tasks which are the biggest memory consumers.
+        idToTasks.clear();
     }
 
     protected boolean checkDataQuality() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
index f324b76c9bc3df..ad981d7456f340 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
@@ -40,13 +40,13 @@
 import com.google.common.collect.Maps;
 
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.util.Map;
 
 import mockit.Expectations;
+import mockit.Injectable;
 import mockit.Mocked;
 
 public class LoadJobTest {
@@ -187,18 +187,22 @@ public void testUpdateStateToLoading() {
 
     @Test
     public void testUpdateStateToFinished(@Mocked MetricRepo metricRepo,
+                                          @Injectable LoadTask loadTask1,
             @Mocked LongCounterMetric longCounterMetric) {
         
         MetricRepo.COUNTER_LOAD_FINISHED = longCounterMetric;
         LoadJob loadJob = new BrokerLoadJob();
+        loadJob.idToTasks.put(1L, loadTask1);
         
         // TxnStateCallbackFactory factory = Catalog.getCurrentCatalog().getGlobalTransactionMgr().getCallbackFactory();
         Catalog catalog = Catalog.getCurrentCatalog();
         GlobalTransactionMgr mgr = new GlobalTransactionMgr(catalog);
         Deencapsulation.setField(catalog, "globalTransactionMgr", mgr);
+        Assert.assertEquals(1, loadJob.idToTasks.size());
         loadJob.updateState(JobState.FINISHED);
         Assert.assertEquals(JobState.FINISHED, loadJob.getState());
         Assert.assertNotEquals(-1, (long) Deencapsulation.getField(loadJob, "finishTimestamp"));
         Assert.assertEquals(100, (int)Deencapsulation.getField(loadJob, "progress"));
+        Assert.assertEquals(0, loadJob.idToTasks.size());
     }
 }