From c8fdeb7a87d4f673e31cc297a53d858530c5b942 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 25 Dec 2024 20:52:14 +0800 Subject: [PATCH 1/7] [Bug](auto-partition) fix auto partition could set storage_medium properties --- .../apache/doris/analysis/PartitionExprUtil.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java index 9302b6485aa720..795ed050e171ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java @@ -18,11 +18,13 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.thrift.TNullableStringLiteral; import com.google.common.base.Objects; @@ -187,7 +189,16 @@ public static Map getAddPartitionClauseFromPartition SinglePartitionDesc singleRangePartitionDesc = new SinglePartitionDesc(true, partitionName, partitionKeyDesc, partitionProperties); - + // as the auto partition table maybe no any partitions firstly, it's should same + // as table's storage medium rather than default storage medium + if (!olapTable.getStorageMedium() + .equals(singleRangePartitionDesc.getPartitionDataProperty().getStorageMedium())) { + singleRangePartitionDesc.getPartitionDataProperty().setStorageMedium(olapTable.getStorageMedium()); + singleRangePartitionDesc.getPartitionDataProperty().setStorageMediumSpecified( + !olapTable.getStorageMedium().equals(DataProperty.DEFAULT_STORAGE_MEDIUM)); + partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, + olapTable.getStorageMedium().name()); + } AddPartitionClause addPartitionClause = new AddPartitionClause(singleRangePartitionDesc, distributionDesc, partitionProperties, false); result.put(partitionName, addPartitionClause); From e82892cd4121cfd215a5c7f43cc4a1c4d66c3eb8 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 26 Dec 2024 14:40:54 +0800 Subject: [PATCH 2/7] update --- .../org/apache/doris/analysis/PartitionExprUtil.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java index 795ed050e171ba..a0e7fb91bb45c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java @@ -189,13 +189,9 @@ public static Map getAddPartitionClauseFromPartition SinglePartitionDesc singleRangePartitionDesc = new SinglePartitionDesc(true, partitionName, partitionKeyDesc, partitionProperties); - // as the auto partition table maybe no any partitions firstly, it's should same - // as table's storage medium rather than default storage medium - if (!olapTable.getStorageMedium() - .equals(singleRangePartitionDesc.getPartitionDataProperty().getStorageMedium())) { - singleRangePartitionDesc.getPartitionDataProperty().setStorageMedium(olapTable.getStorageMedium()); - singleRangePartitionDesc.getPartitionDataProperty().setStorageMediumSpecified( - !olapTable.getStorageMedium().equals(DataProperty.DEFAULT_STORAGE_MEDIUM)); + // iff table's storage medium is not equal default storage medium, + // should add storage medium in partition properties + if (!DataProperty.DEFAULT_STORAGE_MEDIUM.equals(olapTable.getStorageMedium())) { partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, olapTable.getStorageMedium().name()); } From 5436469c4fe8841c3a5f1a75d050e4c0210dc60c Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Tue, 31 Dec 2024 10:22:45 +0800 Subject: [PATCH 3/7] add case --- .../test_auto_range_partition.groovy | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy index 557560dfefc477..74b7706a218c8a 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy @@ -243,4 +243,28 @@ suite("test_auto_range_partition") { sql "insert into awh_test_range_auto values (1,'20201212')" exception "date_trunc function second param only support argument is" } + + sql "drop table if exists DAILY_TRADE_VALUE" + sql """ + CREATE TABLE `DAILY_TRADE_VALUE` + ( + `TRADE_DATE` datev2 NOT NULL COMMENT '交易日期', + `TRADE_ID` varchar(40) NOT NULL COMMENT '交易编号' + ) + UNIQUE KEY(`TRADE_DATE`, `TRADE_ID`) + AUTO PARTITION BY RANGE (date_trunc(`TRADE_DATE`, 'year')) + ( + ) + DISTRIBUTED BY HASH(`TRADE_DATE`) BUCKETS 10 + PROPERTIES ( + "storage_medium" = "ssd", + "replication_num" = "1" + ); + """ + sql " insert into DAILY_TRADE_VALUE values ('2022-02-02','asd'); " + result2 = sql "show partitions from DAILY_TRADE_VALUE" + logger.info("${result2}") + assertEquals(result2.size(), 1) + // check storage medium, and have check the pipeline P0 disk is all ssd + assertEquals(result2[0][10], "SSD") } From faa8dcab7c3c62e2e021f2de55a314a5e1e0cfa3 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Tue, 31 Dec 2024 14:11:08 +0800 Subject: [PATCH 4/7] update --- .../service/FrontendServiceImplTest.java | 45 +++++++++++++++++++ .../test_auto_range_partition.groovy | 24 ---------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java index 2e5a4e0745631e..f865bf3bd3d1aa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java @@ -27,6 +27,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; import org.apache.doris.tablefunction.BackendsTableValuedFunction; import org.apache.doris.thrift.TBackendsMetadataParams; import org.apache.doris.thrift.TCreatePartitionRequest; @@ -132,6 +133,50 @@ public void testCreatePartitionRange() throws Exception { Assert.assertNotNull(p20230807); } + @Test + public void testCreatePartitionRangeMedium() throws Exception { + String createOlapTblStmt = new String("CREATE TABLE test.partition_range2(\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100)\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code)\n" + + "AUTO PARTITION BY range (date_trunc( event_day,'day')) (\n" + + "\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 2\n" + + "PROPERTIES(\"storage_medium\" = \"ssd\",\"replication_num\" = \"1\");"); + + createTable(createOlapTblStmt); + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("test"); + OlapTable table = (OlapTable) db.getTableOrAnalysisException("partition_range2"); + + List> partitionValues = new ArrayList<>(); + List values = new ArrayList<>(); + + TNullableStringLiteral start = new TNullableStringLiteral(); + start.setValue("2023-08-07 00:00:00"); + values.add(start); + + partitionValues.add(values); + + FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv); + TCreatePartitionRequest request = new TCreatePartitionRequest(); + request.setDbId(db.getId()); + request.setTableId(table.getId()); + request.setPartitionValues(partitionValues); + TCreatePartitionResult partition = impl.createPartition(request); + + Assert.assertEquals(partition.getStatus().getStatusCode(), TStatusCode.OK); + Partition p20230807 = table.getPartition("p20230807000000"); + Assert.assertNotNull(p20230807); + + ShowResultSet result = UtFrameUtils.showPartitionsByName(connectContext, "test.partition_range2"); + String showCreateTableResultSql = result.getResultRows().get(0).get(10); + System.out.println(showCreateTableResultSql); + Assert.assertEquals(showCreateTableResultSql, "SSD"); + } + @Test public void testCreatePartitionList() throws Exception { String createOlapTblStmt = new String("CREATE TABLE test.partition_list(\n" diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy index 74b7706a218c8a..557560dfefc477 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy @@ -243,28 +243,4 @@ suite("test_auto_range_partition") { sql "insert into awh_test_range_auto values (1,'20201212')" exception "date_trunc function second param only support argument is" } - - sql "drop table if exists DAILY_TRADE_VALUE" - sql """ - CREATE TABLE `DAILY_TRADE_VALUE` - ( - `TRADE_DATE` datev2 NOT NULL COMMENT '交易日期', - `TRADE_ID` varchar(40) NOT NULL COMMENT '交易编号' - ) - UNIQUE KEY(`TRADE_DATE`, `TRADE_ID`) - AUTO PARTITION BY RANGE (date_trunc(`TRADE_DATE`, 'year')) - ( - ) - DISTRIBUTED BY HASH(`TRADE_DATE`) BUCKETS 10 - PROPERTIES ( - "storage_medium" = "ssd", - "replication_num" = "1" - ); - """ - sql " insert into DAILY_TRADE_VALUE values ('2022-02-02','asd'); " - result2 = sql "show partitions from DAILY_TRADE_VALUE" - logger.info("${result2}") - assertEquals(result2.size(), 1) - // check storage medium, and have check the pipeline P0 disk is all ssd - assertEquals(result2[0][10], "SSD") } From 17a131c2692fc757f7461ccaf9c360694cd30c61 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Tue, 31 Dec 2024 15:45:11 +0800 Subject: [PATCH 5/7] update --- .../java/org/apache/doris/service/FrontendServiceImplTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java index f865bf3bd3d1aa..24a15e7b83dde5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.common.Config; +import org.apache.doris.common.ConfigBase; import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSet; @@ -135,6 +136,7 @@ public void testCreatePartitionRange() throws Exception { @Test public void testCreatePartitionRangeMedium() throws Exception { + ConfigBase.setMutableConfig("disable_storage_medium_check", "true"); String createOlapTblStmt = new String("CREATE TABLE test.partition_range2(\n" + " event_day DATETIME NOT NULL,\n" + " site_id INT DEFAULT '10',\n" From bc63995395b7291d2d6e0f187d6d1f2e155317a6 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Tue, 31 Dec 2024 16:29:54 +0800 Subject: [PATCH 6/7] update --- .../java/org/apache/doris/service/FrontendServiceImplTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java index 24a15e7b83dde5..072092d412a2ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java @@ -232,7 +232,8 @@ public void testGetDBNames() throws Exception { TGetDbsResult dbNames = impl.getDbNames(params); Assert.assertEquals(dbNames.getDbs().size(), 2); - Assert.assertEquals(dbNames.getDbs(), Arrays.asList("test", "test_")); + Assert.assertTrue(dbNames.getDbs().contains("test")); + Assert.assertTrue(dbNames.getDbs().contains("test_")); } @Test From 80b5ac2d4c3527ad5e10277c7f2492d9d533ee55 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Tue, 31 Dec 2024 16:46:31 +0800 Subject: [PATCH 7/7] update --- .../java/org/apache/doris/service/FrontendServiceImplTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java index 072092d412a2ec..e145b0c688048b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java @@ -55,7 +55,6 @@ import org.junit.rules.ExpectedException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.UUID;