diff --git a/be/test/exec/tablet_sink_test.cpp b/be/test/exec/tablet_sink_test.cpp
index 7b4e1120e6954e..f973b048a4cb2e 100644
--- a/be/test/exec/tablet_sink_test.cpp
+++ b/be/test/exec/tablet_sink_test.cpp
@@ -453,7 +453,7 @@ TEST_F(OlapTableSinkTest, normal) {
ASSERT_TRUE(st.ok());
// close
st = sink.close(&state, Status::OK());
- ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: already stopped, skip waiting for close. cancelled/!eos: : 1/1") << st.to_string();
+ ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: wait close failed. ") << st.to_string();
// each node has a eof
ASSERT_EQ(2, service->_eof_counters);
@@ -586,7 +586,7 @@ TEST_F(OlapTableSinkTest, convert) {
ASSERT_TRUE(st.ok());
// close
st = sink.close(&state, Status::OK());
- ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: already stopped, skip waiting for close. cancelled/!eos: : 1/1") << st.to_string();
+ ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: wait close failed. ") << st.to_string();
// each node has a eof
ASSERT_EQ(2, service->_eof_counters);
@@ -966,7 +966,7 @@ TEST_F(OlapTableSinkTest, decimal) {
ASSERT_TRUE(st.ok());
// close
st = sink.close(&state, Status::OK());
- ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: already stopped, skip waiting for close. cancelled/!eos: : 1/1") << st.to_string();
+ ASSERT_TRUE(st.ok() || st.to_string() == "Internal error: wait close failed. ") << st.to_string();
ASSERT_EQ(2, output_set.size());
ASSERT_TRUE(output_set.count("[(12 12.3)]") > 0);
diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md
index f976f5ca8d4c32..2139fe49e43d9a 100644
--- a/docs/en/administrator-guide/config/fe_config.md
+++ b/docs/en/administrator-guide/config/fe_config.md
@@ -789,3 +789,8 @@ And the new UI will also try to get this base path first to assemble the URL.
Only valid when `enable_http_server_v2` is true.
The default is empty, that is, not set.
+
+### `max_dynamic_partition_num`
+
+When creating a dynamic partition table, the maximum number of partitions allowed to be automatically created. To prevent creating too many partitions at once.
+The default is 500.
diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md
index 09f3a27af8c7c4..a0afe4e68ed18e 100644
--- a/docs/en/administrator-guide/dynamic-partition.md
+++ b/docs/en/administrator-guide/dynamic-partition.md
@@ -125,6 +125,12 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`:
When `time_unit` is` MONTH`, this parameter is used to specify the start date of each month. The value ranges from 1 to 28. 1 means the 1st of every month, and 28 means the 28th of every month. The default is 1, which means that every month starts at 1st. The 29, 30 and 31 are not supported at the moment to avoid ambiguity caused by lunar years or months.
+* `dynamic_partition.create_history_partition`
+
+ The default is false. When set to true, Doris will automatically create all partitions from start to end. At the same time, the parameter `max_dynamic_partition_num` of FE will limit the total number of partitions to avoid creating too many partitions at once. If (end - start) is larger than `max_dynamic_partition_num`, the operation will fail.
+
+ When the `start` attribute is not specified, this parameter has no effect.
+
### Notice
If some partitions between `dynamic_partition.start` and `dynamic_partition.end` are lost due to some unexpected circumstances when using dynamic partition, the lost partitions between the current time and `dynamic_partition.end` will be recreated, but the lost partitions between `dynamic_partition.start` and the current time will not be recreated.
diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md
index 64836ce6c892dd..d7665b58fd0439 100644
--- a/docs/zh-CN/administrator-guide/config/fe_config.md
+++ b/docs/zh-CN/administrator-guide/config/fe_config.md
@@ -794,3 +794,8 @@ thrift_client_timeout_ms 的值被设置为大于0来避免线程卡在java.net.
仅在 `enable_http_server_v2` 为 true 的情况下才有效。
默认为空，即不设置。
+
+### `max_dynamic_partition_num`
+
+在创建动态分区表时，允许自动创建的最大分区数量。以防止一次性创建过多的分区。
+默认为 500。
diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md
index dfbc085ed35835..ddbd55f6964ec5 100644
--- a/docs/zh-CN/administrator-guide/dynamic-partition.md
+++ b/docs/zh-CN/administrator-guide/dynamic-partition.md
@@ -110,7 +110,7 @@ under the License.
* `dynamic_partition.buckets`
动态创建的分区所对应的分桶数量。
-
+
* `dynamic_partition.replication_num`
动态创建的分区所对应的副本数量，如果不填写，则默认为该表创建时指定的副本数量。
@@ -122,6 +122,12 @@ under the License.
* `dynamic_partition.start_day_of_month`
当 `time_unit` 为 `MONTH` 时，该参数用于指定每月的起始日期。取值为 1 到 28。其中 1 表示每月1号，28 表示每月28号。默认为 1，即表示每月以1号位起始点。暂不支持以29、30、31号为起始日，以避免因闰年或闰月带来的歧义。
+
+* `dynamic_partition.create_history_partition`
+
+ 默认为 false。当置为 true 时，Doris 会自动创建由 start 到 end 的所有分区。同时，FE 的参数 `max_dynamic_partition_num` 会限制总分区数量，以避免一次性创建过多分区。当 end - start 的值大于 `max_dynamic_partition_num` 值时，操作将被禁止。
+
+ 当不指定 `start` 属性时，该参数不生效。
### 注意事项
@@ -346,6 +352,6 @@ mysql> SHOW DYNAMIC PARTITION TABLES;
1. 创建动态分区表后提示 ```Could not create table with dynamic partition when fe config dynamic_partition_enable is false```
- 由于动态分区的总开关，也就是 FE 的配置 ```dynamic_partition_enable``` 为 false，导致无法创建动态分区表。
+ 由于动态分区的总开关，也就是 FE 的配置 ```dynamic_partition_enable``` 为 false，导致无法创建动态分区表。
- 这时候请修改 FE 的配置文件，增加一行 ```dynamic_partition_enable=true```，并重启 FE。或者执行命令 ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true") 将动态分区开关打开即可。
+ 这时候请修改 FE 的配置文件，增加一行 ```dynamic_partition_enable=true```，并重启 FE。或者执行命令 ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true") 将动态分区开关打开即可。
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 0711f55c99bee7..7c21c9c7b55296 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -737,6 +737,7 @@ under the License.
${basedir}/target/generated-sources/build/
+ ${basedir}/target/generated-sources/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
index 3092da65f80bb0..4fa23197685c75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
@@ -36,8 +36,10 @@ public class DynamicPartitionProperty {
public static final String START_DAY_OF_MONTH = "dynamic_partition.start_day_of_month";
public static final String TIME_ZONE = "dynamic_partition.time_zone";
public static final String REPLICATION_NUM = "dynamic_partition.replication_num";
+ public static final String CREATE_HISTORY_PARTITION = "dynamic_partition.create_history_partition";
public static final int MIN_START_OFFSET = Integer.MIN_VALUE;
+ public static final int MAX_END_OFFSET = Integer.MAX_VALUE;
public static final int NOT_SET_REPLICATION_NUM = -1;
private boolean exist;
@@ -52,6 +54,7 @@ public class DynamicPartitionProperty {
private StartOfDate startOfMonth;
private TimeZone tz = TimeUtils.getSystemTimeZone();
private int replicationNum;
+ private boolean createHistoryPartition = false;
public DynamicPartitionProperty(Map properties) {
if (properties != null && !properties.isEmpty()) {
@@ -65,6 +68,7 @@ public DynamicPartitionProperty(Map properties) {
this.prefix = properties.get(PREFIX);
this.buckets = Integer.parseInt(properties.get(BUCKETS));
this.replicationNum = Integer.parseInt(properties.getOrDefault(REPLICATION_NUM, String.valueOf(NOT_SET_REPLICATION_NUM)));
+ this.createHistoryPartition = Boolean.parseBoolean(properties.get(CREATE_HISTORY_PARTITION));
createStartOfs(properties);
} else {
this.exist = false;
@@ -123,6 +127,10 @@ public StartOfDate getStartOfMonth() {
return startOfMonth;
}
+ public boolean isCreateHistoryPartition() {
+ return createHistoryPartition;
+ }
+
public String getStartOfInfo() {
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
return startOfWeek.toDisplayInfo();
@@ -156,7 +164,8 @@ public String getProperties(int tableReplicationNum) {
",\n\"" + END + "\" = \"" + end + "\"" +
",\n\"" + PREFIX + "\" = \"" + prefix + "\"" +
",\n\"" + REPLICATION_NUM + "\" = \"" + useReplicationNum + "\"" +
- ",\n\"" + BUCKETS + "\" = \"" + buckets + "\"";
+ ",\n\"" + BUCKETS + "\" = \"" + buckets + "\"" +
+ ",\n\"" + CREATE_HISTORY_PARTITION + "\" = \"" + createHistoryPartition + "\"";
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\"";
} else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 4855554e58304e..78e02cf6208f09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -45,14 +45,14 @@
import org.apache.doris.common.util.RangeUtils;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
@@ -142,9 +142,13 @@ private ArrayList getAddPartitionClause(Database db, OlapTab
DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty();
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo();
ZonedDateTime now = ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId());
- for (int i = 0; i <= dynamicPartitionProperty.getEnd(); i++) {
- String prevBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, i, partitionFormat);
- String nextBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, i + 1, partitionFormat);
+
+ boolean createHistoryPartition = dynamicPartitionProperty.isCreateHistoryPartition();
+ int idx = createHistoryPartition ? dynamicPartitionProperty.getStart() : 0;
+
+ for (; idx <= dynamicPartitionProperty.getEnd(); idx++) {
+ String prevBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, idx, partitionFormat);
+ String nextBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, idx + 1, partitionFormat);
PartitionValue lowerValue = new PartitionValue(prevBorder);
PartitionValue upperValue = new PartitionValue(nextBorder);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index 9d177e76c37147..f535c9cfec78f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -1379,4 +1379,12 @@ public class Config extends ConfigBase {
*/
@ConfField
public static int grpc_max_message_size_bytes = 1 * 1024 * 1024 * 1024; // 1GB
+
+ /**
+ * Used to limit the maximum number of partitions that can be created when creating a dynamic partition table,
+ * to avoid creating too many partitions at one time.
+ * The number is determined by "start" and "end" in the dynamic partition parameters.
+ */
+ @ConfField(mutable = true, masterOnly = true)
+ public static int max_dynamic_partition_num = 500;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
index 18d8ec16c48a8b..71447c6db7ca92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
@@ -239,7 +239,9 @@ public enum ErrorCode {
ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_FORMAT(5072, new byte[] {'4', '2', '0', '0', '0'},
"Invalid dynamic partition replication num: %s."),
ERROR_CREATE_TABLE_LIKE_EMPTY(5073, new byte[] {'4', '2', '0', '0', '0'},
- "Origin create table stmt is empty");
+ "Origin create table stmt is empty"),
+ ERROR_DYNAMIC_PARTITION_CREATE_HISTORY_PARTITION(5074, new byte[]{'4', '2', '0', '0', '0'},
+ "Invalid dynamic partition create_history_partition: %s. Expected true or false");
ErrorCode(int code, byte[] sqlState, String errorMsg) {
this.code = code;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index 81906d11e808aa..4ae5225e90a05a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -30,18 +30,19 @@
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableProperty;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.DayOfWeek;
@@ -77,6 +78,12 @@ public static void checkTimeUnit(String timeUnit, PartitionInfo partitionInfo) t
ErrorReport.reportDdlException(DynamicPartitionProperty.TIME_UNIT + " could not be "
+ TimeUnit.HOUR.toString() + " when type of partition column "
+ partitionColumn.getDisplayName() + " is " + PrimitiveType.DATE.toString());
+ } else if (PrimitiveType.getIntegerTypes().contains(partitionColumn.getDataType())
+ && timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
+ // The partition column's type is INT, not support HOUR
+ ErrorReport.reportDdlException(DynamicPartitionProperty.TIME_UNIT + " could not be "
+ + TimeUnit.HOUR.toString() + " when type of partition column "
+ + partitionColumn.getDisplayName() + " is Integer");
}
}
@@ -88,27 +95,33 @@ private static void checkPrefix(String prefix) throws DdlException {
}
}
- private static void checkStart(String start) throws DdlException {
+ private static int checkStart(String start) throws DdlException {
try {
- if (Integer.parseInt(start) >= 0) {
+ int startInt = Integer.parseInt(start);
+ if (startInt >= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_START_ZERO, start);
}
+ return startInt;
} catch (NumberFormatException e) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_START_FORMAT, start);
}
+ return DynamicPartitionProperty.MIN_START_OFFSET;
}
- private static void checkEnd(String end) throws DdlException {
+ private static int checkEnd(String end) throws DdlException {
if (Strings.isNullOrEmpty(end)) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_EMPTY);
}
try {
- if (Integer.parseInt(end) <= 0) {
+ int endInt = Integer.parseInt(end);
+ if (endInt <= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_ZERO, end);
}
+ return endInt;
} catch (NumberFormatException e) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_FORMAT, end);
}
+ return DynamicPartitionProperty.MAX_END_OFFSET;
}
private static void checkBuckets(String buckets) throws DdlException {
@@ -131,6 +144,14 @@ private static void checkEnable(String enable) throws DdlException {
}
}
+ private static boolean checkCreateHistoryPartition(String create) throws DdlException {
+ if (Strings.isNullOrEmpty(create)
+ || (!Boolean.TRUE.toString().equalsIgnoreCase(create) && !Boolean.FALSE.toString().equalsIgnoreCase(create))) {
+ ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_CREATE_HISTORY_PARTITION, create);
+ }
+ return Boolean.valueOf(create);
+ }
+
private static void checkStartDayOfMonth(String val) throws DdlException {
if (Strings.isNullOrEmpty(val)) {
throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_MONTH);
@@ -204,13 +225,15 @@ public static boolean checkInputDynamicPartitionProperties(Map p
String end = properties.get(DynamicPartitionProperty.END);
String buckets = properties.get(DynamicPartitionProperty.BUCKETS);
String enable = properties.get(DynamicPartitionProperty.ENABLE);
+ String createHistoryPartition = properties.get(DynamicPartitionProperty.CREATE_HISTORY_PARTITION);
if (!(Strings.isNullOrEmpty(enable) &&
Strings.isNullOrEmpty(timeUnit) &&
Strings.isNullOrEmpty(timeZone) &&
Strings.isNullOrEmpty(prefix) &&
Strings.isNullOrEmpty(start) &&
Strings.isNullOrEmpty(end) &&
- Strings.isNullOrEmpty(buckets))) {
+ Strings.isNullOrEmpty(buckets) &&
+ Strings.isNullOrEmpty(createHistoryPartition))) {
if (Strings.isNullOrEmpty(enable)) {
properties.put(DynamicPartitionProperty.ENABLE, "true");
}
@@ -232,6 +255,9 @@ public static boolean checkInputDynamicPartitionProperties(Map p
if (Strings.isNullOrEmpty(timeZone)) {
properties.put(DynamicPartitionProperty.TIME_ZONE, TimeUtils.getSystemTimeZone().getID());
}
+ if (Strings.isNullOrEmpty(createHistoryPartition)) {
+ properties.put(DynamicPartitionProperty.CREATE_HISTORY_PARTITION, "false");
+ }
}
return true;
}
@@ -267,12 +293,7 @@ public static Map analyzeDynamicPartition(Map pr
properties.remove(DynamicPartitionProperty.PREFIX);
analyzedProperties.put(DynamicPartitionProperty.PREFIX, prefixValue);
}
- if (properties.containsKey(DynamicPartitionProperty.END)) {
- String endValue = properties.get(DynamicPartitionProperty.END);
- checkEnd(endValue);
- properties.remove(DynamicPartitionProperty.END);
- analyzedProperties.put(DynamicPartitionProperty.END, endValue);
- }
+
if (properties.containsKey(DynamicPartitionProperty.BUCKETS)) {
String bucketsValue = properties.get(DynamicPartitionProperty.BUCKETS);
checkBuckets(bucketsValue);
@@ -285,14 +306,41 @@ public static Map analyzeDynamicPartition(Map pr
properties.remove(DynamicPartitionProperty.ENABLE);
analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue);
}
- // If dynamic property is not specified.Use Integer.MIN_VALUE as default
+
+ // If dynamic property "start" is not specified, use Integer.MIN_VALUE as default
+ int start = DynamicPartitionProperty.MIN_START_OFFSET;
if (properties.containsKey(DynamicPartitionProperty.START)) {
String startValue = properties.get(DynamicPartitionProperty.START);
- checkStart(startValue);
+ start = checkStart(startValue);
properties.remove(DynamicPartitionProperty.START);
analyzedProperties.put(DynamicPartitionProperty.START, startValue);
}
+ int end = DynamicPartitionProperty.MAX_END_OFFSET;
+ if (properties.containsKey(DynamicPartitionProperty.END)) {
+ String endValue = properties.get(DynamicPartitionProperty.END);
+ end = checkEnd(endValue);
+ properties.remove(DynamicPartitionProperty.END);
+ analyzedProperties.put(DynamicPartitionProperty.END, endValue);
+ }
+
+ boolean createHistoryPartition = false;
+ if (properties.containsKey(DynamicPartitionProperty.CREATE_HISTORY_PARTITION)) {
+ String val = properties.get(DynamicPartitionProperty.CREATE_HISTORY_PARTITION);
+ createHistoryPartition = checkCreateHistoryPartition(val);
+ properties.remove(DynamicPartitionProperty.CREATE_HISTORY_PARTITION);
+ analyzedProperties.put(DynamicPartitionProperty.CREATE_HISTORY_PARTITION, val);
+ }
+
+ // Check the number of dynamic partitions that need to be created to avoid creating too many partitions at once.
+ // If create_history_partition is false, history partition is not considered.
+ if (!createHistoryPartition) {
+ start = 0;
+ }
+ if (end - start > Config.max_dynamic_partition_num) {
+ throw new DdlException("Too many dynamic partitions: " + (end - start) + ". Limit: " + Config.max_dynamic_partition_num);
+ }
+
if (properties.containsKey(DynamicPartitionProperty.START_DAY_OF_MONTH)) {
String val = properties.get(DynamicPartitionProperty.START_DAY_OF_MONTH);
checkStartDayOfMonth(val);
@@ -414,9 +462,9 @@ public static String getPartitionRangeString(DynamicPartitionProperty property,
return getPartitionRangeOfDay(current, offset, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), format);
- } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { // MONTH
+ } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return getPartitionRangeOfHour(current, offset, format);
- } else {
+ } else { // MONTH
return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), format);
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index d7dc04cec3d166..28e890b30fc7b1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -19,11 +19,14 @@
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
-
import org.apache.doris.utframe.UtFrameUtils;
+
+import com.clearspring.analytics.util.Lists;
+
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -33,9 +36,11 @@
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Iterator;
+import java.util.List;
import java.util.UUID;
public class DynamicPartitionTableTest {
@@ -468,16 +473,75 @@ public void testCreateDynamicPartitionImmediately() throws Exception {
"\"dynamic_partition.buckets\" = \"1\"\n" +
");";
createTable(createOlapTblStmt);
- OlapTable emptyDynamicTable = (OlapTable)Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("empty_dynamic_partition");
+ OlapTable emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("empty_dynamic_partition");
Assert.assertTrue(emptyDynamicTable.getAllPartitions().size() == 4);
+ Iterator partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ List partNames = Lists.newArrayList();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ partNames.add(partitionName.substring(1));
+ }
+ Collections.sort(partNames);
+
int partitionCount = 0;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+ for (String partName : partNames) {
+ Date partitionDate = sdf.parse(partName);
+ Date date = new Date();
+ Calendar calendar = new GregorianCalendar();
+ calendar.setTime(date);
+ calendar.add(calendar.DATE, partitionCount);
+ date = calendar.getTime();
+
+ Assert.assertEquals(partitionDate.getYear(), date.getYear());
+ Assert.assertEquals(partitionDate.getMonth(), date.getMonth());
+ Assert.assertEquals(partitionDate.getDay(), date.getDay());
+
+ partitionCount++;
+ }
+ }
+
+ @Test
+ public void testFillHistoryDynamicPartition() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`histo_dynamic_partition` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ OlapTable emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("histo_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
Iterator partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
- while (partitionCount < 4) {
+ List partNames = Lists.newArrayList();
+ while (partitionIterator.hasNext()) {
String partitionName = partitionIterator.next().getName();
- SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
- Date partitionDate = sdf.parse(partitionName.substring(1));
+ partNames.add(partitionName.substring(1));
+ }
+ Collections.sort(partNames);
+ int partitionCount = -3;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+ for (String partName : partNames) {
+ Date partitionDate = sdf.parse(partName);
Date date = new Date();
Calendar calendar = new GregorianCalendar();
calendar.setTime(date);
@@ -491,4 +555,221 @@ public void testCreateDynamicPartitionImmediately() throws Exception {
partitionCount++;
}
}
+
+ @Test(expected = DdlException.class)
+ public void testFillHistoryDynamicPartition2() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`histo_dynamic_partition2` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3000\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ // exceed the max dynamic partition limit
+ Config.max_dynamic_partition_num = 1000;
+ createTable(createOlapTblStmt);
+ }
+
+ @Test
+ public void testAllTypeDynamicPartition() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`hour_dynamic_partition` (\n" +
+ " `k1` datetime NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"hour\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ OlapTable emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("hour_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ Iterator partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(11, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`week_dynamic_partition` (\n" +
+ " `k1` datetime NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"week\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("week_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(8, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`month_dynamic_partition` (\n" +
+ " `k1` datetime NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"month\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("month_dynamic_partition");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(7, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`int_dynamic_partition_day` (\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("int_dynamic_partition_day");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(9, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`int_dynamic_partition_week` (\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"week\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("int_dynamic_partition_week");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(8, partitionName.length());
+ }
+
+ createOlapTblStmt = "CREATE TABLE test.`int_dynamic_partition_month` (\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"month\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ emptyDynamicTable = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("int_dynamic_partition_month");
+ Assert.assertEquals(7, emptyDynamicTable.getAllPartitions().size());
+
+ partitionIterator = emptyDynamicTable.getAllPartitions().iterator();
+ while (partitionIterator.hasNext()) {
+ String partitionName = partitionIterator.next().getName();
+ Assert.assertEquals(7, partitionName.length());
+ }
+ }
+
+ @Test(expected = DdlException.class)
+ public void testHourDynamicPartitionWithIntType() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`int_dynamic_partition_hour` (\n" +
+ " `k1` int NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "PARTITION BY RANGE(`k1`)\n" +
+ "()\n" +
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n" +
+ "\"dynamic_partition.time_unit\" = \"hour\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ }
+
}