From 79a7fe24fd84e50c6eb1db70ded2c32420fb203a Mon Sep 17 00:00:00 2001 From: yujun777 Date: Tue, 16 Jul 2024 17:22:17 +0800 Subject: [PATCH 1/6] create dynamic partition with overlap not throw exception --- .../org/apache/doris/clone/DynamicPartitionScheduler.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 4636eb30ca29fd..a7f92f02ebdee6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -233,7 +233,7 @@ private static int getBucketsNum(DynamicPartitionProperty property, OlapTable ta } private ArrayList getAddPartitionClause(Database db, OlapTable olapTable, - Column partitionColumn, String partitionFormat, boolean executeFirstTime) throws DdlException { + Column partitionColumn, String partitionFormat, boolean executeFirstTime) { ArrayList addPartitionClauses = new ArrayList<>(); DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty(); RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); @@ -278,9 +278,7 @@ private ArrayList getAddPartitionClause(Database db, OlapTab // IllegalArgumentException: lb is greater than ub LOG.warn("Error in gen addPartitionKeyRange. db: {}, table: {}, partition idx: {}", db.getFullName(), olapTable.getName(), idx, e); - recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), - e.getMessage(), olapTable.getId()); - throw new DdlException(e.getMessage()); + continue; } for (PartitionItem partitionItem : rangePartitionInfo.getIdToItem(false).values()) { // only support single column partition now @@ -299,7 +297,6 @@ private ArrayList getAddPartitionClause(Database db, OlapTab addPartitionKeyRange, db.getFullName(), olapTable.getName(), idx, e); recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId()); - throw new DdlException(e.getMessage()); } break; } From ca86b406e396bb28a9c9f2f7a0bfe21b9176b43b Mon Sep 17 00:00:00 2001 From: yujun777 Date: Tue, 16 Jul 2024 17:34:46 +0800 Subject: [PATCH 2/6] add test --- .../test_dynamic_partition_failed.groovy | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy index 3447834a2af3ac..a0a60eb30e271e 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy @@ -26,6 +26,7 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { PROPERTIES ( "replication_num" = "1", + "dynamic_partition.replication_num" = "1", "dynamic_partition.enable" = "true", "dynamic_partition.end" = "3", "dynamic_partition.time_unit" = "day", @@ -46,6 +47,26 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { setFeConfig('max_dynamic_partition_num', Integer.MAX_VALUE) + sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_ok2 FORCE' + sql '''CREATE TABLE test_dynamic_partition_failed_ok2 + ( `k1` date NULL ) + PARTITION BY RANGE(k1) ( + PARTITION `phistory` VALUES less than ('2020-01-01') + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES + ( + "replication_num" = "1", + "dynamic_partition.replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.end" = "3", + "dynamic_partition.time_unit" = "YEAR", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.start" = "-20", + "dynamic_partition.create_history_partition" = "true" + )''' + sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_2' test { sql '''CREATE TABLE test_dynamic_partition_failed_2 @@ -55,6 +76,7 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { PROPERTIES ( "replication_num" = "1", + "dynamic_partition.replication_num" = "1", "dynamic_partition.enable" = "true", "dynamic_partition.end" = "3", "dynamic_partition.time_unit" = "day", @@ -74,6 +96,7 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { } finally { setFeConfig('max_dynamic_partition_num', old_max_dynamic_partition_num) sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_ok1 FORCE' + sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_ok2 FORCE' sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_2' } } From e575aa4affc57f738fcf406fb5fb8f7140aaadd4 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Tue, 16 Jul 2024 17:35:24 +0800 Subject: [PATCH 3/6] update --- .../dynamic_partition/test_dynamic_partition_failed.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy index a0a60eb30e271e..065f44fb95feea 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy @@ -63,7 +63,7 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { "dynamic_partition.time_unit" = "YEAR", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "1", - "dynamic_partition.start" = "-20", + "dynamic_partition.start" = "-10", "dynamic_partition.create_history_partition" = "true" )''' From d7fa47aae428d04d495e1caecc7a4376092fada6 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Tue, 16 Jul 2024 22:37:57 +0800 Subject: [PATCH 4/6] update --- .../org/apache/doris/clone/DynamicPartitionScheduler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index a7f92f02ebdee6..c322f66edaa7ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -233,7 +233,7 @@ private static int getBucketsNum(DynamicPartitionProperty property, OlapTable ta } private ArrayList getAddPartitionClause(Database db, OlapTable olapTable, - Column partitionColumn, String partitionFormat, boolean executeFirstTime) { + Column partitionColumn, String partitionFormat, boolean executeFirstTime) throws DdlException { ArrayList addPartitionClauses = new ArrayList<>(); DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty(); RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); @@ -278,6 +278,9 @@ private ArrayList getAddPartitionClause(Database db, OlapTab // IllegalArgumentException: lb is greater than ub LOG.warn("Error in gen addPartitionKeyRange. db: {}, table: {}, partition idx: {}", db.getFullName(), olapTable.getName(), idx, e); + if (executeFirstTime) { + throw new DdlException(e.getMessage()); + } continue; } for (PartitionItem partitionItem : rangePartitionInfo.getIdToItem(false).values()) { From eb442b443dd58fb152fea6b10bef31ad78421ca1 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Tue, 16 Jul 2024 22:43:40 +0800 Subject: [PATCH 5/6] update --- .../java/org/apache/doris/clone/DynamicPartitionScheduler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index c322f66edaa7ef..74f62bddb2dbc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -279,7 +279,8 @@ private ArrayList getAddPartitionClause(Database db, OlapTab LOG.warn("Error in gen addPartitionKeyRange. db: {}, table: {}, partition idx: {}", db.getFullName(), olapTable.getName(), idx, e); if (executeFirstTime) { - throw new DdlException(e.getMessage()); + throw new DdlException("maybe dynamic_partition.start is too small, error: " + + e.getMessage()); } continue; } From e04ede29c13e12ef8c43a3ebc59b6d7b02f0a1a5 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Tue, 16 Jul 2024 22:57:42 +0800 Subject: [PATCH 6/6] update