Skip to content

Commit 0dca9ef

Browse files
authored
[Enhancement](alter) support add partitions in batch (#37114)
## Proposed changes Issue Number: close #32524
1 parent aaf7a96 commit 0dca9ef

File tree

6 files changed

+226
-1
lines changed

6 files changed

+226
-1
lines changed

fe/fe-core/src/main/cup/sql_parser.cup

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1746,6 +1746,14 @@ alter_table_clause ::=
17461746
{:
17471747
RESULT = new ModifyEngineClause(engine, properties);
17481748
:}
1749+
| KW_ADD opt_tmp:isTempPartition KW_PARTITIONS KW_FROM LPAREN partition_key_list:lower RPAREN KW_TO LPAREN partition_key_list:upper RPAREN KW_INTERVAL INTEGER_LITERAL:time_interval ident:time_unit opt_properties:properties
1750+
{:
1751+
RESULT = new AlterMultiPartitionClause(PartitionKeyDesc.createMultiFixed(lower, upper, time_interval, time_unit), properties, isTempPartition);
1752+
:}
1753+
| KW_ADD opt_tmp:isTempPartition KW_PARTITIONS KW_FROM LPAREN partition_key_list:lower RPAREN KW_TO LPAREN partition_key_list:upper RPAREN KW_INTERVAL INTEGER_LITERAL:num_interval opt_properties:properties
1754+
{:
1755+
RESULT = new AlterMultiPartitionClause(PartitionKeyDesc.createMultiFixed(lower, upper, num_interval), properties, isTempPartition);
1756+
:}
17491757
;
17501758

17511759
opt_enable_feature_properties ::=

fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.analysis.AddPartitionClause;
2121
import org.apache.doris.analysis.AddPartitionLikeClause;
2222
import org.apache.doris.analysis.AlterClause;
23+
import org.apache.doris.analysis.AlterMultiPartitionClause;
2324
import org.apache.doris.analysis.AlterSystemStmt;
2425
import org.apache.doris.analysis.AlterTableStmt;
2526
import org.apache.doris.analysis.AlterViewStmt;
@@ -252,7 +253,8 @@ private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable,
252253
} else if (alterClause instanceof DropPartitionFromIndexClause) {
253254
// do nothing
254255
} else if (alterClause instanceof AddPartitionClause
255-
|| alterClause instanceof AddPartitionLikeClause) {
256+
|| alterClause instanceof AddPartitionLikeClause
257+
|| alterClause instanceof AlterMultiPartitionClause) {
256258
needProcessOutsideTableLock = true;
257259
} else {
258260
throw new DdlException("Invalid alter operation: " + alterClause.getOpType());
@@ -508,6 +510,12 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException {
508510
} else if (alterClause instanceof ModifyTablePropertiesClause) {
509511
Map<String, String> properties = alterClause.getProperties();
510512
((SchemaChangeHandler) schemaChangeHandler).updateTableProperties(db, tableName, properties);
513+
} else if (alterClause instanceof AlterMultiPartitionClause) {
514+
if (!((AlterMultiPartitionClause) alterClause).isTempPartition()) {
515+
DynamicPartitionUtil.checkAlterAllowed(
516+
(OlapTable) db.getTableOrMetaException(tableName, TableType.OLAP));
517+
}
518+
Env.getCurrentEnv().addMultiPartitions(db, tableName, (AlterMultiPartitionClause) alterClause);
511519
} else {
512520
throw new DdlException("Invalid alter operation: " + alterClause.getOpType());
513521
}
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.analysis;
19+
20+
import org.apache.doris.alter.AlterOpType;
21+
import org.apache.doris.common.AnalysisException;
22+
23+
import java.util.Map;
24+
25+
public class AlterMultiPartitionClause extends AlterTableClause {
26+
private PartitionKeyDesc partitionKeyDesc;
27+
private Map<String, String> properties;
28+
private boolean isTempPartition;
29+
30+
public AlterMultiPartitionClause(PartitionKeyDesc partitionKeyDesc, Map<String, String> properties,
31+
boolean isTempPartition) {
32+
super(AlterOpType.ADD_PARTITION);
33+
this.partitionKeyDesc = partitionKeyDesc;
34+
this.properties = properties;
35+
this.isTempPartition = isTempPartition;
36+
}
37+
38+
@Override
39+
public void analyze(Analyzer analyzer) throws AnalysisException {
40+
}
41+
42+
@Override
43+
public String toSql() {
44+
return String.format("ADD PARTITIONS %s", partitionKeyDesc.toSql());
45+
}
46+
47+
@Override
48+
public String toString() {
49+
return toSql();
50+
}
51+
52+
@Override
53+
public boolean allowOpMTMV() {
54+
return false;
55+
}
56+
57+
@Override
58+
public boolean needChangeMTMVState() {
59+
return false;
60+
}
61+
62+
public PartitionKeyDesc getPartitionKeyDesc() {
63+
return partitionKeyDesc;
64+
}
65+
66+
public Map<String, String> getProperties() {
67+
return properties;
68+
}
69+
70+
public boolean isTempPartition() {
71+
return isTempPartition;
72+
}
73+
}

fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.doris.analysis.AlterDatabaseQuotaStmt;
3939
import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType;
4040
import org.apache.doris.analysis.AlterDatabaseRename;
41+
import org.apache.doris.analysis.AlterMultiPartitionClause;
4142
import org.apache.doris.analysis.AlterSystemStmt;
4243
import org.apache.doris.analysis.AlterTableStmt;
4344
import org.apache.doris.analysis.AlterViewStmt;
@@ -3278,6 +3279,11 @@ public PartitionPersistInfo addPartition(Database db, String tableName, AddParti
32783279
isCreateTable, generatedPartitionId, writeEditLog);
32793280
}
32803281

3282+
public void addMultiPartitions(Database db, String tableName, AlterMultiPartitionClause multiPartitionClause)
3283+
throws DdlException {
3284+
getInternalCatalog().addMultiPartitions(db, tableName, multiPartitionClause);
3285+
}
3286+
32813287
public void addPartitionLike(Database db, String tableName, AddPartitionLikeClause addPartitionLikeClause)
32823288
throws DdlException {
32833289
getInternalCatalog().addPartitionLike(db, tableName, addPartitionLikeClause);

fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.analysis.AlterDatabaseQuotaStmt;
2626
import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType;
2727
import org.apache.doris.analysis.AlterDatabaseRename;
28+
import org.apache.doris.analysis.AlterMultiPartitionClause;
2829
import org.apache.doris.analysis.Analyzer;
2930
import org.apache.doris.analysis.ColumnDef;
3031
import org.apache.doris.analysis.ColumnDef.DefaultValue;
@@ -42,6 +43,7 @@
4243
import org.apache.doris.analysis.HashDistributionDesc;
4344
import org.apache.doris.analysis.KeysDesc;
4445
import org.apache.doris.analysis.LiteralExpr;
46+
import org.apache.doris.analysis.MultiPartitionDesc;
4547
import org.apache.doris.analysis.PartitionDesc;
4648
import org.apache.doris.analysis.PartitionKeyDesc;
4749
import org.apache.doris.analysis.PartitionNames;
@@ -1867,6 +1869,24 @@ public PartitionPersistInfo addPartition(Database db, String tableName, AddParti
18671869
}
18681870
}
18691871

1872+
public void addMultiPartitions(Database db, String tableName, AlterMultiPartitionClause multiPartitionClause)
1873+
throws DdlException {
1874+
List<SinglePartitionDesc> singlePartitionDescs;
1875+
try {
1876+
MultiPartitionDesc multiPartitionDesc = new MultiPartitionDesc(multiPartitionClause.getPartitionKeyDesc(),
1877+
multiPartitionClause.getProperties());
1878+
singlePartitionDescs = multiPartitionDesc.getSinglePartitionDescList();
1879+
} catch (AnalysisException e) {
1880+
throw new DdlException("Failed to analyze multi partition clause: " + e.getMessage());
1881+
}
1882+
1883+
for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) {
1884+
AddPartitionClause addPartitionClause = new AddPartitionClause(singlePartitionDesc, null,
1885+
multiPartitionClause.getProperties(), false);
1886+
addPartition(db, tableName, addPartitionClause, false, 0, true);
1887+
}
1888+
}
1889+
18701890
public void replayAddPartition(PartitionPersistInfo info) throws MetaNotFoundException {
18711891
Database db = (Database) getDbOrMetaException(info.getDbId());
18721892
OlapTable olapTable = (OlapTable) db.getTableOrMetaException(info.getTableId(),
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_alter_add_multi_partition") {
19+
sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
20+
21+
// Check if you can create a partition for a leap month
22+
sql """
23+
CREATE TABLE IF NOT EXISTS add_multi_partition
24+
(
25+
`k1` LARGEINT NOT NULL,
26+
`date` DATE NOT NULL,
27+
`k2` VARCHAR(20)
28+
)
29+
ENGINE=OLAP
30+
UNIQUE KEY(`k1`, `date`)
31+
PARTITION BY RANGE(`date`)
32+
(
33+
PARTITION `p_20000201` VALUES [("2000-02-01"), ("2000-02-05"))
34+
)
35+
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
36+
PROPERTIES
37+
(
38+
"replication_num" = "1"
39+
);
40+
"""
41+
List<List<Object>> result1 = sql "show partitions from add_multi_partition;"
42+
assertEquals(result1.size(), 1)
43+
sql "ALTER TABLE add_multi_partition ADD PARTITIONS FROM ('2000-02-05') TO ('2000-03-01') INTERVAL 4 DAY;"
44+
List<List<Object>> result2 = sql "show partitions from add_multi_partition;"
45+
assertEquals(result2.size(), 8)
46+
def partitionName = sql "show partitions from add_multi_partition where PartitionName = 'p_20000229';"
47+
for (pn in partitionName) {
48+
assertTrue(pn[1] == "p_20000229")
49+
}
50+
sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
51+
52+
53+
sql """
54+
CREATE TABLE `add_multi_partition` (
55+
`k1` LARGEINT NOT NULL,
56+
`date` DATE NOT NULL,
57+
`k2` VARCHAR(20) NULL
58+
) ENGINE=OLAP
59+
UNIQUE KEY(`k1`, `date`)
60+
PARTITION BY RANGE(`date`)
61+
(PARTITION p_2024_01 VALUES [('2024-01-01'), ('2024-01-08')),
62+
PARTITION p_2024_02 VALUES [('2024-01-08'), ('2024-01-15')),
63+
PARTITION p_2024_03 VALUES [('2024-01-15'), ('2024-01-22')),
64+
PARTITION p_2024_04 VALUES [('2024-01-22'), ('2024-01-29')),
65+
PARTITION p_2024_05 VALUES [('2024-01-29'), ('2024-02-05')),
66+
PARTITION p_2024_06 VALUES [('2024-02-05'), ('2024-02-12')),
67+
PARTITION p_2024_07 VALUES [('2024-02-12'), ('2024-02-19')),
68+
PARTITION p_2024_08 VALUES [('2024-02-19'), ('2024-02-26')),
69+
PARTITION p_2024_09 VALUES [('2024-02-26'), ('2024-03-01')))
70+
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
71+
PROPERTIES
72+
(
73+
"replication_num" = "1"
74+
);
75+
"""
76+
List<List<Object>> result3 = sql "show partitions from add_multi_partition;"
77+
assertEquals(result3.size(), 9)
78+
sql "ALTER TABLE add_multi_partition ADD PARTITIONS FROM ('2024-04-01') TO ('2025-01-01') INTERVAL 1 WEEK;"
79+
List<List<Object>> result4 = sql "show partitions from add_multi_partition;"
80+
assertEquals(result4.size(), 49)
81+
sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
82+
83+
84+
sql """
85+
CREATE TABLE IF NOT EXISTS add_multi_partition
86+
(
87+
`k1` LARGEINT NOT NULL,
88+
`age` SMALLINT,
89+
`k2` VARCHAR(20)
90+
)
91+
ENGINE=OLAP
92+
UNIQUE KEY(`k1`, `age`)
93+
PARTITION BY RANGE(`age`)
94+
(
95+
FROM (1) TO (100) INTERVAL 10
96+
)
97+
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
98+
PROPERTIES
99+
(
100+
"replication_num" = "1"
101+
);
102+
"""
103+
List<List<Object>> result7 = sql "show partitions from add_multi_partition;"
104+
assertEquals(result7.size(), 10)
105+
sql "ALTER TABLE add_multi_partition ADD PARTITIONS FROM (100) TO (200) INTERVAL 10;"
106+
List<List<Object>> result8 = sql "show partitions from add_multi_partition;"
107+
assertEquals(result8.size(), 20)
108+
sql "DROP TABLE IF EXISTS add_multi_partition FORCE"
109+
110+
}

0 commit comments

Comments
 (0)