From d9dac684b2d9a3412d71f7f378ad1dc32c1550b6 Mon Sep 17 00:00:00 2001 From: qijianliang Date: Tue, 1 Dec 2020 11:28:33 +0800 Subject: [PATCH 1/2] fix finished load jobs cost too much heap bug. --- .../src/main/java/org/apache/doris/load/loadv2/LoadJob.java | 2 ++ .../test/java/org/apache/doris/load/loadv2/LoadJobTest.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index b3d3b1b06b7cb7..8b8ff0d2085ef1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -659,6 +659,8 @@ private void executeFinish() { if (MetricRepo.isInit) { MetricRepo.COUNTER_LOAD_FINISHED.increase(1L); } + // when load job finished, there is no need to hold the tasks which are the biggest memory consumers. + idToTasks.clear(); } protected boolean checkDataQuality() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java index f324b76c9bc3df..0ee8c2d873741d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java @@ -19,6 +19,7 @@ package org.apache.doris.load.loadv2; import com.google.common.collect.Lists; +import mockit.Injectable; import org.apache.doris.analysis.LoadStmt; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; @@ -187,18 +188,22 @@ public void testUpdateStateToLoading() { @Test public void testUpdateStateToFinished(@Mocked MetricRepo metricRepo, + @Injectable LoadTask loadTask1, @Mocked LongCounterMetric longCounterMetric) { MetricRepo.COUNTER_LOAD_FINISHED = longCounterMetric; LoadJob loadJob = new BrokerLoadJob(); + loadJob.idToTasks.put(1L, loadTask1); // TxnStateCallbackFactory factory = Catalog.getCurrentCatalog().getGlobalTransactionMgr().getCallbackFactory(); Catalog catalog = Catalog.getCurrentCatalog(); GlobalTransactionMgr mgr = new GlobalTransactionMgr(catalog); Deencapsulation.setField(catalog, "globalTransactionMgr", mgr); + Assert.assertEquals(1, loadJob.idToTasks.size()); loadJob.updateState(JobState.FINISHED); Assert.assertEquals(JobState.FINISHED, loadJob.getState()); Assert.assertNotEquals(-1, (long) Deencapsulation.getField(loadJob, "finishTimestamp")); Assert.assertEquals(100, (int)Deencapsulation.getField(loadJob, "progress")); + Assert.assertEquals(0, loadJob.idToTasks.size()); } } From 594715b75e14ea5da1b18780dd81f4d0a3c19292 Mon Sep 17 00:00:00 2001 From: qijianliang Date: Tue, 1 Dec 2020 16:52:18 +0800 Subject: [PATCH 2/2] optimize import --- .../test/java/org/apache/doris/load/loadv2/LoadJobTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java index 0ee8c2d873741d..ad981d7456f340 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java @@ -19,7 +19,6 @@ package org.apache.doris.load.loadv2; import com.google.common.collect.Lists; -import mockit.Injectable; import org.apache.doris.analysis.LoadStmt; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; @@ -41,13 +40,13 @@ import com.google.common.collect.Maps; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import java.util.Map; import mockit.Expectations; +import mockit.Injectable; import mockit.Mocked; public class LoadJobTest {