From 54babc0f65ef51b941bb65d6fbf810dce85bb469 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Fri, 8 Jan 2021 13:37:35 +0800 Subject: [PATCH 1/3] [Feature] Support Create Dynamic Partition Immediately FirstTime Without Wating Schedule. issue:#5208 --- .../doris/clone/DynamicPartitionScheduler.java | 13 ++++++++++--- .../doris/common/util/DynamicPartitionUtil.java | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 49d65ed0df2470..e5b4e0530e5c35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -43,6 +43,7 @@ import org.apache.doris.common.util.RangeUtils; import org.apache.doris.common.util.TimeUtils; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; @@ -52,6 +53,7 @@ import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -89,6 +91,11 @@ public DynamicPartitionScheduler(String name, long intervalMs) { this.initialize = false; } + public void executeDynamicPartitionFirstTime(Long dbId, Long tableId) { + List> tempDynamicPartitionTableInfo = Lists.newArrayList(new Pair<>(dbId, tableId)); + executeDynamicPartition(tempDynamicPartitionTableInfo); + } + public void registerDynamicPartitionTable(Long dbId, Long tableId) { dynamicPartitionTableInfo.add(new Pair<>(dbId, tableId)); } @@ -248,8 +255,8 @@ private ArrayList getDropPartitionClause(Database db, OlapT return dropPartitionClauses; } - private void executeDynamicPartition() { - Iterator> iterator = dynamicPartitionTableInfo.iterator(); + private void executeDynamicPartition(Collection> dynamicPartitionTableInfoCol) { + Iterator> iterator = dynamicPartitionTableInfoCol.iterator(); while (iterator.hasNext()) { Pair tableInfo = iterator.next(); Long dbId = tableInfo.first; @@ -388,7 +395,7 @@ protected void runAfterCatalogReady() { } setInterval(Config.dynamic_partition_check_interval_seconds * 1000L); if (Config.dynamic_partition_enable) { - executeDynamicPartition(); + executeDynamicPartition(dynamicPartitionTableInfo); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 56bd051cb5c98a..8b388c3c43c74d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -240,6 +240,7 @@ public static void registerOrRemoveDynamicPartitionTable(long dbId, OlapTable ol if (olapTable.getTableProperty() != null && olapTable.getTableProperty().getDynamicPartitionProperty() != null) { if (olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()) { + Catalog.getCurrentCatalog().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(dbId, olapTable.getId()); Catalog.getCurrentCatalog().getDynamicPartitionScheduler().registerDynamicPartitionTable(dbId, olapTable.getId()); } else { Catalog.getCurrentCatalog().getDynamicPartitionScheduler().removeDynamicPartitionTable(dbId, olapTable.getId()); From 09202d2e02a63c08a0b825f15b608a1dfef73742 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Mon, 11 Jan 2021 13:14:51 +0800 Subject: [PATCH 2/3] Add isReplay flags --- .../src/main/java/org/apache/doris/catalog/Catalog.java | 6 +++--- .../apache/doris/common/util/DynamicPartitionUtil.java | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 885c86fa04f70e..b35234cf206ca5 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -3864,7 +3864,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws DdlExcept } LOG.info("successfully create table[{};{}]", tableName, tableId); // register or remove table from DynamicPartition after table created - DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTable); + DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTable, false); dynamicPartitionScheduler.createOrUpdateRuntimeInfo( tableName, DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime()); } catch (DdlException e) { @@ -4296,7 +4296,7 @@ public void replayCreateTable(String dbName, Table table) { } } } // end for partitions - DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(dbId, olapTable); + DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(dbId, olapTable, true); } } } @@ -5370,7 +5370,7 @@ public void modifyTableDynamicPartition(Database db, OlapTable table, Map p return true; } - public static void registerOrRemoveDynamicPartitionTable(long dbId, OlapTable olapTable) { + public static void registerOrRemoveDynamicPartitionTable(long dbId, OlapTable olapTable, boolean isReplay) { if (olapTable.getTableProperty() != null && olapTable.getTableProperty().getDynamicPartitionProperty() != null) { if (olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()) { - Catalog.getCurrentCatalog().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(dbId, olapTable.getId()); + if (!isReplay) { + // execute create partition first time only in master of FE, So no need execute + // when it's replay + Catalog.getCurrentCatalog().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(dbId, olapTable.getId()); + } Catalog.getCurrentCatalog().getDynamicPartitionScheduler().registerDynamicPartitionTable(dbId, olapTable.getId()); } else { Catalog.getCurrentCatalog().getDynamicPartitionScheduler().removeDynamicPartitionTable(dbId, olapTable.getId()); From 7fddf8b728c542711c54baa387a080b4aac9fa31 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Mon, 11 Jan 2021 14:54:54 +0800 Subject: [PATCH 3/3] Fix ut problem --- .../catalog/DynamicPartitionTableTest.java | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index dd1f18775a1db6..d7dc04cec3d166 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateTableStmt; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; @@ -32,6 +31,11 @@ import org.junit.Test; import org.junit.rules.ExpectedException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Iterator; import java.util.UUID; public class DynamicPartitionTableTest { @@ -441,7 +445,7 @@ public void testSetDynamicPartitionReplicationNum() throws Exception { } @Test - public void testEmptyDynamicPartition() throws Exception { + public void testCreateDynamicPartitionImmediately() throws Exception { String createOlapTblStmt = "CREATE TABLE test.`empty_dynamic_partition` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + @@ -463,11 +467,28 @@ public void testEmptyDynamicPartition() throws Exception { "\"dynamic_partition.prefix\" = \"p\",\n" + "\"dynamic_partition.buckets\" = \"1\"\n" + ");"; - String insertStmt = "insert into test.`empty_dynamic_partition` values ('2020-09-10', 1000, 100, 'test', '2020-09-10 23:59:59');"; createTable(createOlapTblStmt); - expectedException.expect(AnalysisException.class); - expectedException.expectMessage("errCode = 2, detailMessage = data cannot be inserted into table with empty partition. " + - "Use `SHOW PARTITIONS FROM empty_dynamic_partition` to see the currently partitions of this table. "); - UtFrameUtils.parseAndAnalyzeStmt("explain " + insertStmt, connectContext); + OlapTable emptyDynamicTable = (OlapTable)Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("empty_dynamic_partition"); + Assert.assertTrue(emptyDynamicTable.getAllPartitions().size() == 4); + + int partitionCount = 0; + Iterator partitionIterator = emptyDynamicTable.getAllPartitions().iterator(); + while (partitionCount < 4) { + String partitionName = partitionIterator.next().getName(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + Date partitionDate = sdf.parse(partitionName.substring(1)); + + Date date = new Date(); + Calendar calendar = new GregorianCalendar(); + calendar.setTime(date); + calendar.add(calendar.DATE, partitionCount); + date = calendar.getTime(); + + Assert.assertEquals(partitionDate.getYear(), date.getYear()); + Assert.assertEquals(partitionDate.getMonth(), date.getMonth()); + Assert.assertEquals(partitionDate.getDay(), date.getDay()); + + partitionCount++; + } } }