Skip to content

[Feature] Support Create Dynamic Partition Immediately FirstTime Without Wating Schedule. #5209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 3 commits into from
Jan 15, 2021
Merged

[Feature] Support Create Dynamic Partition Immediately FirstTime Without Wating Schedule. #5209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
54babc0
[Feature] Support Create Dynamic Partition Immediately FirstTime With…
HappenLee Jan 8, 2021
09202d2
Add isReplay flags
HappenLee Jan 11, 2021
7fddf8b
Fix ut problem
HappenLee Jan 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3864,7 +3864,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws DdlExcept
}
LOG.info("successfully create table[{};{}]", tableName, tableId);
// register or remove table from DynamicPartition after table created
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTable);
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTable, false);
dynamicPartitionScheduler.createOrUpdateRuntimeInfo(
tableName, DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime());
} catch (DdlException e) {
Expand Down Expand Up @@ -4296,7 +4296,7 @@ public void replayCreateTable(String dbName, Table table) {
}
}
} // end for partitions
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(dbId, olapTable);
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(dbId, olapTable, true);
}
}
}
Expand Down Expand Up @@ -5370,7 +5370,7 @@ public void modifyTableDynamicPartition(Database db, OlapTable table, Map<String
tableProperty.buildDynamicProperty();
}

DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), table);
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), table, false);
dynamicPartitionScheduler.createOrUpdateRuntimeInfo(
table.getName(), DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime());
ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), logProperties);
Expand Down
13 changes: 10 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.apache.doris.common.util.RangeUtils;
import org.apache.doris.common.util.TimeUtils;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
Expand All @@ -52,6 +53,7 @@

import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
Expand Down Expand Up @@ -89,6 +91,11 @@ public DynamicPartitionScheduler(String name, long intervalMs) {
this.initialize = false;
}

public void executeDynamicPartitionFirstTime(Long dbId, Long tableId) {
List<Pair<Long, Long>> tempDynamicPartitionTableInfo = Lists.newArrayList(new Pair<>(dbId, tableId));
executeDynamicPartition(tempDynamicPartitionTableInfo);
}

public void registerDynamicPartitionTable(Long dbId, Long tableId) {
dynamicPartitionTableInfo.add(new Pair<>(dbId, tableId));
}
Expand Down Expand Up @@ -248,8 +255,8 @@ private ArrayList<DropPartitionClause> getDropPartitionClause(Database db, OlapT
return dropPartitionClauses;
}

private void executeDynamicPartition() {
Iterator<Pair<Long, Long>> iterator = dynamicPartitionTableInfo.iterator();
private void executeDynamicPartition(Collection<Pair<Long, Long>> dynamicPartitionTableInfoCol) {
Iterator<Pair<Long, Long>> iterator = dynamicPartitionTableInfoCol.iterator();
while (iterator.hasNext()) {
Pair<Long, Long> tableInfo = iterator.next();
Long dbId = tableInfo.first;
Expand Down Expand Up @@ -388,7 +395,7 @@ protected void runAfterCatalogReady() {
}
setInterval(Config.dynamic_partition_check_interval_seconds * 1000L);
if (Config.dynamic_partition_enable) {
executeDynamicPartition();
executeDynamicPartition(dynamicPartitionTableInfo);
}
}
}
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,10 +236,15 @@ public static boolean checkInputDynamicPartitionProperties(Map<String, String> p
return true;
}

public static void registerOrRemoveDynamicPartitionTable(long dbId, OlapTable olapTable) {
public static void registerOrRemoveDynamicPartitionTable(long dbId, OlapTable olapTable, boolean isReplay) {
if (olapTable.getTableProperty() != null
&& olapTable.getTableProperty().getDynamicPartitionProperty() != null) {
if (olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()) {
if (!isReplay) {
// execute create partition first time only in master of FE, So no need execute
// when it's replay
Catalog.getCurrentCatalog().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(dbId, olapTable.getId());
}
Catalog.getCurrentCatalog().getDynamicPartitionScheduler().registerDynamicPartitionTable(dbId, olapTable.getId());
} else {
Catalog.getCurrentCatalog().getDynamicPartitionScheduler().removeDynamicPartitionTable(dbId, olapTable.getId());
Expand Down
35 changes: 28 additions & 7 deletions fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
Expand All @@ -32,6 +31,11 @@
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Iterator;
import java.util.UUID;

public class DynamicPartitionTableTest {
Expand Down Expand Up @@ -441,7 +445,7 @@ public void testSetDynamicPartitionReplicationNum() throws Exception {
}

@Test
public void testEmptyDynamicPartition() throws Exception {
public void testCreateDynamicPartitionImmediately() throws Exception {
String createOlapTblStmt = "CREATE TABLE test.`empty_dynamic_partition` (\n" +
" `k1` date NULL COMMENT \"\",\n" +
" `k2` int NULL COMMENT \"\",\n" +
Expand All @@ -463,11 +467,28 @@ public void testEmptyDynamicPartition() throws Exception {
"\"dynamic_partition.prefix\" = \"p\",\n" +
"\"dynamic_partition.buckets\" = \"1\"\n" +
");";
String insertStmt = "insert into test.`empty_dynamic_partition` values ('2020-09-10', 1000, 100, 'test', '2020-09-10 23:59:59');";
createTable(createOlapTblStmt);
expectedException.expect(AnalysisException.class);
expectedException.expectMessage("errCode = 2, detailMessage = data cannot be inserted into table with empty partition. " +
"Use `SHOW PARTITIONS FROM empty_dynamic_partition` to see the currently partitions of this table. ");
UtFrameUtils.parseAndAnalyzeStmt("explain " + insertStmt, connectContext);
OlapTable emptyDynamicTable = (OlapTable)Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("empty_dynamic_partition");
Assert.assertTrue(emptyDynamicTable.getAllPartitions().size() == 4);

int partitionCount = 0;
Iterator<Partition> partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
while (partitionCount < 4) {
String partitionName = partitionIterator.next().getName();
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
Date partitionDate = sdf.parse(partitionName.substring(1));

Date date = new Date();
Calendar calendar = new GregorianCalendar();
calendar.setTime(date);
calendar.add(calendar.DATE, partitionCount);
date = calendar.getTime();

Assert.assertEquals(partitionDate.getYear(), date.getYear());
Assert.assertEquals(partitionDate.getMonth(), date.getMonth());
Assert.assertEquals(partitionDate.getDay(), date.getDay());

partitionCount++;
}
}
}