Skip to content

[enhancement](auto-partition) Forbid use Auto and Dynamic partition at the same time #33736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HappenLee merged 1 commit into from
Apr 19, 2024
Merged

[enhancement](auto-partition) Forbid use Auto and Dynamic partition at the same time #33736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 21 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ public List<String> getPartitionColNames() {
// 1. partition by list (column) : now support one slotRef
// 2. partition by range(column/function(column)) : support slotRef and some
// special function eg: date_trunc, date_floor/ceil
// not only for auto partition. maybe we should check for project partitiion also
public static List<String> getColNamesFromExpr(ArrayList<Expr> exprs, boolean isListPartition,
boolean isAutoPartition)
throws AnalysisException {
Expand Down Expand Up @@ -206,16 +205,27 @@ public void analyze(List<ColumnDef> columnDefs, Map<String, String> otherPropert
throw new AnalysisException(
"The partition column must be NOT NULL with allow_partition_column_nullable OFF");
}
if (this instanceof RangePartitionDesc && isAutoCreatePartitions && columnDef.isAllowNull()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
}
if (this instanceof RangePartitionDesc && partitionExprs != null) {
if (partitionExprs.get(0) instanceof FunctionCallExpr) {
if (!columnDef.getType().isDateType()) {
throw new AnalysisException(
"Auto range partition needs Date/DateV2/"
+ "Datetime/DatetimeV2 column as partition column"
+ partitionExprs.get(0).toSql());
if (this instanceof RangePartitionDesc && isAutoCreatePartitions) {
if (columnDef.isAllowNull()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
}
if (partitionExprs != null) {
for (Expr expr : partitionExprs) {
if (!(expr instanceof FunctionCallExpr) || !RANGE_PARTITION_FUNCTIONS
.contains(((FunctionCallExpr) expr).getFnName().getFunction())) {
throw new AnalysisException(
"auto create partition only support slotRef and "
+ "date_trunc/date_floor/date_ceil function in range partitions. "
+ expr.toSql());
}
}
if (partitionExprs.get(0) instanceof FunctionCallExpr) {
if (!columnDef.getType().isDateType()) {
throw new AnalysisException(
"Auto range partition needs Date/DateV2/"
+ "Datetime/DatetimeV2 column as partition column but got"
+ partitionExprs.get(0).toSql());
}
}
}
}
Expand Down
6 changes: 6 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,12 @@ public static Map<String, String> analyzeDynamicPartition(Map<String, String> pr
analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue);
}

if (Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true"))
&& olapTable.getPartitionInfo().enableAutomaticPartition()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
}

// If dynamic property "start" is not specified, use Integer.MIN_VALUE as default
int start = DynamicPartitionProperty.MIN_START_OFFSET;
if (properties.containsKey(DynamicPartitionProperty.START)) {
Expand Down
17 changes: 2 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2735,21 +2735,8 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
.getDynamicPartitionProperty();
if (dynamicProperty.isExist() && dynamicProperty.getEnable()
&& partitionDesc.isAutoCreatePartitions()) {
String dynamicUnit = dynamicProperty.getTimeUnit();
ArrayList<Expr> autoExprs = partitionDesc.getPartitionExprs();
for (Expr autoExpr : autoExprs) {
Expr func = (FunctionCallExpr) autoExpr;
for (Expr child : func.getChildren()) {
if (child instanceof LiteralExpr) {
String autoUnit = ((LiteralExpr) child).getStringValue();
if (!dynamicUnit.equalsIgnoreCase(autoUnit)) {
throw new AnalysisException(
"If support auto partition and dynamic partition at same time, "
+ "they must have the same interval unit.");
}
}
}
}
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
}
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
Expand Down
6 changes: 4 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,10 @@ private void validatePartitionColumn(ColumnDefinition column, ConnectContext ctx
throw new AnalysisException(
"The partition column must be NOT NULL with allow_partition_column_nullable OFF");
}
if (isAutoPartition && partitionType.equalsIgnoreCase(PartitionType.RANGE.name()) && column.isNullable()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
if (partitionType.equalsIgnoreCase(PartitionType.RANGE.name()) && isAutoPartition) {
if (column.isNullable()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
}
}
}

Expand Down
53 changes: 51 additions & 2 deletions regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,16 @@ suite("test_auto_partition_behavior") {
exception "AUTO RANGE PARTITION doesn't support NULL column"
}

sql "drop table if exists test_dynamic"
sql """
create table test_dynamic(
k0 DATE not null
)
auto partition by range (date_trunc(k0, 'year')) ()
DISTRIBUTED BY HASH(`k0`) BUCKETS auto
properties("replication_num" = "1");
"""

// PROHIBIT different timeunit of interval when use both auto & dynamic partition
sql "set experimental_enable_nereids_planner=true;"
test{
Expand All @@ -263,7 +273,19 @@ suite("test_auto_partition_behavior") {
"dynamic_partition.buckets" = "8"
);
"""
exception "If support auto partition and dynamic partition at same time, they must have the same interval unit."
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
test {
sql """
ALTER TABLE test_dynamic set (
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "DAY",
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}

sql "set experimental_enable_nereids_planner=false;"
Expand All @@ -288,7 +310,19 @@ suite("test_auto_partition_behavior") {
"dynamic_partition.buckets" = "8"
);
"""
exception "If support auto partition and dynamic partition at same time, they must have the same interval unit."
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
test {
sql """
ALTER TABLE test_dynamic set (
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "DAY",
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}

// prohibit too long value for partition column
Expand Down Expand Up @@ -359,4 +393,19 @@ suite("test_auto_partition_behavior") {
"""
exception "partition expr date_trunc is illegal!"
}
sql "set experimental_enable_nereids_planner=false;"
test{
sql """
create table illegal(
k0 datetime(6) NOT null,
k1 int NOT null
)
auto partition by range (date_trunc(k1, 'hour'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties("replication_num" = "1");
"""
exception "Auto range partition needs Date/DateV2/Datetime/DatetimeV2 column as partition column"
}
}