From d7a437fe3349897d8ba9216677a22d55f5cb94fe Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 9 Oct 2024 11:25:44 +0800 Subject: [PATCH 1/2] 1 --- .../InsertOverwriteManager.java | 3 ++ .../InsertOverwriteManagerTest.java | 34 ++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java index a00107c76a74a0..fa5aa82fe69d21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java @@ -289,6 +289,9 @@ private boolean rollback(long taskId) { * @param table Run the table for insert overwrite */ public void recordRunningTableOrException(DatabaseIf db, TableIf table) { + if (!(table instanceof OlapTable)) { + return; + } long dbId = db.getId(); long tableId = table.getId(); runningLock.writeLock().lock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java index 4bf6c9f12d564b..026f821352246e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java @@ -18,23 +18,27 @@ package org.apache.doris.insertoverwrite; import org.apache.doris.catalog.DatabaseIf; -import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.datasource.hive.HMSExternalTable; import mockit.Expectations; import mockit.Mocked; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; public class InsertOverwriteManagerTest { @Mocked private DatabaseIf db; @Mocked - private TableIf table; + private OlapTable table; + + @Mocked + private HMSExternalTable hmsExternalTable; @Before public void setUp() @@ -57,6 +61,14 @@ public void setUp() table.getName(); minTimes = 0; result = "table1"; + + hmsExternalTable.getId(); + minTimes = 0; + result = 3L; + + hmsExternalTable.getName(); + minTimes = 0; + result = "hmsTable"; } }; } @@ -65,13 +77,17 @@ public void setUp() public void testParallel() { InsertOverwriteManager manager = new InsertOverwriteManager(); manager.recordRunningTableOrException(db, table); - try { - manager.recordRunningTableOrException(db, table); - } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("Not allowed")); - } + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> manager.recordRunningTableOrException(db, table)); manager.dropRunningRecord(db.getId(), table.getId()); - manager.recordRunningTableOrException(db, table); + Assertions.assertDoesNotThrow(() -> manager.recordRunningTableOrException(db, table)); } + @Test + public void testHmsTableParallel() { + InsertOverwriteManager manager = new InsertOverwriteManager(); + manager.recordRunningTableOrException(db, hmsExternalTable); + Assertions.assertDoesNotThrow(() -> manager.recordRunningTableOrException(db, hmsExternalTable)); + manager.dropRunningRecord(db.getId(), hmsExternalTable.getId()); + } } From a303fa158fef91c3caae43ec6c97beaa1baf9a1c Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 9 Oct 2024 11:38:46 +0800 Subject: [PATCH 2/2] 1 --- .../apache/doris/insertoverwrite/InsertOverwriteManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java index fa5aa82fe69d21..df16b8f1be205c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java @@ -289,6 +289,11 @@ private boolean rollback(long taskId) { * @param table Run the table for insert overwrite */ public void recordRunningTableOrException(DatabaseIf db, TableIf table) { + // The logic of OlapTable executing insert overwrite is to create temporary partitions, + // replace partitions, etc. + // If executed in parallel, it may cause problems such as not being able to find temporary partitions. + // But in terms of external table, we don't care the internal logic of execution, + // so there's no need to keep records if (!(table instanceof OlapTable)) { return; }