From 226d476363cb17a94b830fd5d5841a098e789853 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Mon, 27 May 2024 17:01:26 +0800 Subject: [PATCH 01/11] fix-dynamic-partiton-start-fail --- .../clone/DynamicPartitionScheduler.java | 20 ++++++++-------- .../common/util/DynamicPartitionUtil.java | 23 +++++++++++-------- .../catalog/DynamicPartitionTableTest.java | 19 +++++++++++++++ 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index e2a700c92bcd1d..ff40b39e4059d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -226,15 +226,10 @@ private ArrayList getAddPartitionClause(Database db, OlapTab boolean createHistoryPartition = dynamicPartitionProperty.isCreateHistoryPartition(); int idx; - int start = dynamicPartitionProperty.getStart(); - int historyPartitionNum = dynamicPartitionProperty.getHistoryPartitionNum(); // When enable create_history_partition, will check the valid value from start and history_partition_num. if (createHistoryPartition) { - if (historyPartitionNum == DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM) { - idx = start; - } else { - idx = Math.max(start, -historyPartitionNum); - } + idx = DynamicPartitionUtil.getRealStart(dynamicPartitionProperty.getStart(), + dynamicPartitionProperty.getHistoryPartitionNum()); } else { idx = 0; } @@ -263,11 +258,13 @@ private ArrayList getAddPartitionClause(Database db, OlapTab PartitionKey upperBound = PartitionKey.createPartitionKey(Collections.singletonList(upperValue), Collections.singletonList(partitionColumn)); addPartitionKeyRange = Range.closedOpen(lowerBound, upperBound); - } catch (AnalysisException | IllegalArgumentException e) { + } catch (Exception e) { // AnalysisException: keys.size is always equal to column.size, cannot reach this exception // IllegalArgumentException: lb is greater than ub LOG.warn("Error in gen addPartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), db.getFullName(), olapTable.getName()); + recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), + e.getMessage(), olapTable.getId()); continue; } for (PartitionItem partitionItem : rangePartitionInfo.getIdToItem(false).values()) { @@ -416,9 +413,11 @@ private ArrayList getDropPartitionClause(Database db, OlapT return dropPartitionClauses; } + int realStart = DynamicPartitionUtil.getRealStart(dynamicPartitionProperty.getStart(), + dynamicPartitionProperty.getHistoryPartitionNum()); ZonedDateTime now = ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId()); String lowerBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, - now, dynamicPartitionProperty.getStart(), partitionFormat); + now, realStart, partitionFormat); PartitionValue lowerPartitionValue = new PartitionValue(lowerBorder); List> reservedHistoryPartitionKeyRangeList = new ArrayList>(); Range reservePartitionKeyRange; @@ -427,11 +426,12 @@ private ArrayList getDropPartitionClause(Database db, OlapT Collections.singletonList(partitionColumn)); reservePartitionKeyRange = Range.atLeast(lowerBound); reservedHistoryPartitionKeyRangeList.add(reservePartitionKeyRange); - } catch (AnalysisException | IllegalArgumentException e) { + } catch (Exception e) { // AnalysisException: keys.size is always equal to column.size, cannot reach this exception // IllegalArgumentException: lb is greater than ub LOG.warn("Error in gen reservePartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), db.getFullName(), olapTable.getName()); + recordDropPartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId()); return dropPartitionClauses; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index fe29dfe7eea9b6..6f001ddb2d48a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -583,21 +583,18 @@ public static Map analyzeDynamicPartition(Map pr long expectCreatePartitionNum = 0; if (!createHistoryPartition) { start = 0; - expectCreatePartitionNum = (long) end - start; } else { int historyPartitionNum = Integer.parseInt(analyzedProperties.getOrDefault( DynamicPartitionProperty.HISTORY_PARTITION_NUM, String.valueOf(DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM))); - if (historyPartitionNum != DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM) { - expectCreatePartitionNum = (long) end - Math.max(start, -historyPartitionNum); - } else { - if (start == Integer.MIN_VALUE) { - throw new DdlException("Provide start or history_partition_num property" - + " when create_history_partition=true. Otherwise set create_history_partition=false"); - } - expectCreatePartitionNum = (long) end - start; + start = getRealStart(start, historyPartitionNum); + if (start == Integer.MIN_VALUE) { + throw new DdlException("Provide start or history_partition_num property" + + " when create_history_partition=true. Otherwise set create_history_partition=false"); } } + expectCreatePartitionNum = (long) end - start; + if (hasEnd && (expectCreatePartitionNum > Config.max_dynamic_partition_num) && Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true"))) { throw new DdlException("Too many dynamic partitions: " @@ -683,6 +680,14 @@ public static Map analyzeDynamicPartition(Map pr return analyzedProperties; } + public static int getRealStart(int start, int historyPartitionNum) { + if (historyPartitionNum == DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM) { + return start; + } else { + return Math.max(start, -historyPartitionNum); + } + } + public static void checkAlterAllowed(OlapTable olapTable) throws DdlException { TableProperty tableProperty = olapTable.getTableProperty(); if (tableProperty != null && tableProperty.getDynamicPartitionProperty() != null diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index f5493dedb76c00..9573a389cfe9b3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -67,6 +67,7 @@ public static void beforeClass() throws Exception { FeConstants.default_scheduler_interval_millisecond = 1000; FeConstants.runningUnitTest = true; Config.disable_storage_medium_check = true; + Config.dynamic_partition_enable = false; // disable auto create dynamic partition UtFrameUtils.createDorisCluster(runningDir); @@ -722,6 +723,24 @@ public void testFillHistoryDynamicPartition3() throws Exception { ExceptionChecker.expectThrowsNoException(() -> alterTable(alter4)); Env.getCurrentEnv().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(db.getId(), tbl.getId()); Assert.assertEquals(24, tbl.getPartitionNames().size()); + + String createOlapTblStmt5 = + "CREATE TABLE test.`dynamic_partition4` (\n" + " `k1` datetime NULL COMMENT \"\"\n" + ")\n" + + "PARTITION BY RANGE (k1)\n" + "()\n" + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + + "PROPERTIES (\n" + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\",\n" + "\"dynamic_partition.start\" = \"-99999999\",\n" + + "\"dynamic_partition.history_partition_num\" = \"5\",\n" + + "\"dynamic_partition.create_history_partition\" = \"true\"\n" + ");"; + // start and history_partition_num are set, create ok + ExceptionChecker.expectThrowsNoException(() -> createTable(createOlapTblStmt5)); + OlapTable tbl4 = (OlapTable) db.getTableOrAnalysisException("dynamic_partition4"); + Assert.assertEquals(9, tbl4.getPartitionNames().size()); + + String alter5 = "alter table test.dynamic_partition4 set ('dynamic_partition.history_partition_num' = '3')"; + Env.getCurrentEnv().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(db.getId(), tbl4.getId()); + Assert.assertEquals(7, tbl4.getPartitionNames().size()); } @Test From e1faf23a183575a66deceab3f5e811d64faee4d9 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Wed, 29 May 2024 10:08:53 +0800 Subject: [PATCH 02/11] add test --- .../org/apache/doris/catalog/DynamicPartitionTableTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 9573a389cfe9b3..61d33de00dfd7e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -81,7 +81,7 @@ public static void beforeClass() throws Exception { @AfterClass public static void tearDown() { - UtFrameUtils.cleanDorisFeDir(runningDir); + //UtFrameUtils.cleanDorisFeDir(runningDir); } private static void changeBeDisk(TStorageMedium storageMedium) { From dc1b690984d648e2e1699f52576728b774aa127c Mon Sep 17 00:00:00 2001 From: yujun777 Date: Wed, 29 May 2024 10:09:23 +0800 Subject: [PATCH 03/11] create dynamic partition first time throw exception --- .../clone/DynamicPartitionScheduler.java | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index ff40b39e4059d3..114095ae3c421c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -53,6 +53,7 @@ import org.apache.doris.thrift.TStorageMedium; import com.google.common.base.Strings; +import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; @@ -101,7 +102,7 @@ public DynamicPartitionScheduler(String name, long intervalMs) { this.initialize = false; } - public void executeDynamicPartitionFirstTime(Long dbId, Long tableId) { + public void executeDynamicPartitionFirstTime(Long dbId, Long tableId) throws DdlException { List> tempDynamicPartitionTableInfo = Lists.newArrayList(Pair.of(dbId, tableId)); executeDynamicPartition(tempDynamicPartitionTableInfo, true); } @@ -218,7 +219,7 @@ private static int getBucketsNum(DynamicPartitionProperty property, OlapTable ta } private ArrayList getAddPartitionClause(Database db, OlapTable olapTable, - Column partitionColumn, String partitionFormat, boolean executeFirstTime) { + Column partitionColumn, String partitionFormat, boolean executeFirstTime) throws DdlException { ArrayList addPartitionClauses = new ArrayList<>(); DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty(); RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); @@ -261,11 +262,11 @@ private ArrayList getAddPartitionClause(Database db, OlapTab } catch (Exception e) { // AnalysisException: keys.size is always equal to column.size, cannot reach this exception // IllegalArgumentException: lb is greater than ub - LOG.warn("Error in gen addPartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), - db.getFullName(), olapTable.getName()); + LOG.warn("Error in gen addPartitionKeyRange. db: {}, table: {}, partition idx: {}", + db.getFullName(), olapTable.getName(), idx, e); recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId()); - continue; + throw new DdlException(Throwables.getStackTraceAsString(e)); } for (PartitionItem partitionItem : rangePartitionInfo.getIdToItem(false).values()) { // only support single column partition now @@ -275,13 +276,16 @@ private ArrayList getAddPartitionClause(Database db, OlapTab isPartitionExists = true; if (addPartitionKeyRange.equals(partitionItem.getItems())) { if (LOG.isDebugEnabled()) { - LOG.debug("partition range {} exist in table {}, clear fail msg", - addPartitionKeyRange, olapTable.getName()); + LOG.debug("partition range {} exist in db {} table {} partition idx {}, clear fail msg", + addPartitionKeyRange, db.getFullName(), olapTable.getName(), idx); } clearCreatePartitionFailedMsg(olapTable.getId()); } else { + LOG.warn("check partition range {} in db {} table {} partiton idx {} fail", + addPartitionKeyRange, db.getFullName(), olapTable.getName(), idx, e); recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId()); + throw new DdlException(Throwables.getStackTraceAsString(e)); } break; } @@ -429,8 +433,8 @@ private ArrayList getDropPartitionClause(Database db, OlapT } catch (Exception e) { // AnalysisException: keys.size is always equal to column.size, cannot reach this exception // IllegalArgumentException: lb is greater than ub - LOG.warn("Error in gen reservePartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), - db.getFullName(), olapTable.getName()); + LOG.warn("Error in gen reservePartitionKeyRange. db: {}, table: {}", + db.getFullName(), olapTable.getName(), e); recordDropPartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId()); return dropPartitionClauses; } @@ -485,7 +489,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT } private void executeDynamicPartition(Collection> dynamicPartitionTableInfoCol, - boolean executeFirstTime) { + boolean executeFirstTime) throws DdlException { Iterator> iterator = dynamicPartitionTableInfoCol.iterator(); while (iterator.hasNext()) { Pair tableInfo = iterator.next(); @@ -549,8 +553,11 @@ private void executeDynamicPartition(Collection> dynamicPartiti } dropPartitionClauses = getDropPartitionClause(db, olapTable, partitionColumn, partitionFormat); tableName = olapTable.getName(); - } catch (DdlException e) { - LOG.warn("should not happen", e); + } catch (Exception e) { + LOG.warn("has error", e); + if (executeFirstTime) { + throw new DdlException(Throwables.getStackTraceAsString(e)); + } } finally { olapTable.readUnlock(); } @@ -564,6 +571,9 @@ private void executeDynamicPartition(Collection> dynamicPartiti clearDropPartitionFailedMsg(olapTable.getId()); } catch (Exception e) { recordDropPartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId()); + if (executeFirstTime) { + throw new DdlException(Throwables.getStackTraceAsString(e)); + } } finally { olapTable.writeUnlock(); } @@ -576,6 +586,9 @@ private void executeDynamicPartition(Collection> dynamicPartiti clearCreatePartitionFailedMsg(olapTable.getId()); } catch (Exception e) { recordCreatePartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId()); + if (executeFirstTime) { + throw new DdlException(Throwables.getStackTraceAsString(e)); + } } } } @@ -633,7 +646,14 @@ protected void runAfterCatalogReady() { } setInterval(Config.dynamic_partition_check_interval_seconds * 1000L); if (Config.dynamic_partition_enable) { - executeDynamicPartition(dynamicPartitionTableInfo, false); + try { + executeDynamicPartition(dynamicPartitionTableInfo, false); + } catch (Exception e) { + // previous had log DdlException + if (LOG.isDebugEnabled()) { + LOG.debug("dynamic partition has error: ", e); + } + } } } } From e63ea5ba0279de82560b6fe7833ad96b7b1fa145 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Thu, 30 May 2024 17:18:32 +0800 Subject: [PATCH 04/11] update test --- .../org/apache/doris/catalog/DynamicPartitionTableTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 61d33de00dfd7e..d457be0324fa98 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -81,7 +81,7 @@ public static void beforeClass() throws Exception { @AfterClass public static void tearDown() { - //UtFrameUtils.cleanDorisFeDir(runningDir); + UtFrameUtils.cleanDorisFeDir(runningDir); } private static void changeBeDisk(TStorageMedium storageMedium) { @@ -739,6 +739,7 @@ public void testFillHistoryDynamicPartition3() throws Exception { Assert.assertEquals(9, tbl4.getPartitionNames().size()); String alter5 = "alter table test.dynamic_partition4 set ('dynamic_partition.history_partition_num' = '3')"; + ExceptionChecker.expectThrowsNoException(() -> alterTable(alter5)); Env.getCurrentEnv().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(db.getId(), tbl4.getId()); Assert.assertEquals(7, tbl4.getPartitionNames().size()); } From 90d2e622cac8c7f9586797d91a62aeb4bd5fd845 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Thu, 30 May 2024 17:23:30 +0800 Subject: [PATCH 05/11] update --- .../org/apache/doris/clone/DynamicPartitionScheduler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 114095ae3c421c..d687ab453eb604 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -556,7 +556,7 @@ private void executeDynamicPartition(Collection> dynamicPartiti } catch (Exception e) { LOG.warn("has error", e); if (executeFirstTime) { - throw new DdlException(Throwables.getStackTraceAsString(e)); + throw new DdlException(e.getMessage()); } } finally { olapTable.readUnlock(); @@ -572,7 +572,7 @@ private void executeDynamicPartition(Collection> dynamicPartiti } catch (Exception e) { recordDropPartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId()); if (executeFirstTime) { - throw new DdlException(Throwables.getStackTraceAsString(e)); + throw new DdlException(e.getMessage()); } } finally { olapTable.writeUnlock(); @@ -587,7 +587,7 @@ private void executeDynamicPartition(Collection> dynamicPartiti } catch (Exception e) { recordCreatePartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId()); if (executeFirstTime) { - throw new DdlException(Throwables.getStackTraceAsString(e)); + throw new DdlException(e.getMessage()); } } } From b92b6ca783583c42baf0d7669cbfb147694e07a8 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Thu, 30 May 2024 17:26:49 +0800 Subject: [PATCH 06/11] update --- .../java/org/apache/doris/clone/DynamicPartitionScheduler.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index d687ab453eb604..d883cbfe569053 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -571,6 +571,7 @@ private void executeDynamicPartition(Collection> dynamicPartiti clearDropPartitionFailedMsg(olapTable.getId()); } catch (Exception e) { recordDropPartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId()); + LOG.warn("has error", e); if (executeFirstTime) { throw new DdlException(e.getMessage()); } @@ -586,6 +587,7 @@ private void executeDynamicPartition(Collection> dynamicPartiti clearCreatePartitionFailedMsg(olapTable.getId()); } catch (Exception e) { recordCreatePartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId()); + LOG.warn("has error", e); if (executeFirstTime) { throw new DdlException(e.getMessage()); } From 441c7c7fff8272c5e41420c806b1a7bf8e823122 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Fri, 31 May 2024 18:08:55 +0800 Subject: [PATCH 07/11] add test --- .../test_dynamic_partition_failed.groovy | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy new file mode 100644 index 00000000000000..23f709b4c086bf --- /dev/null +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_dynamic_partition_failed', 'nonConcurrent') { + def old_max_dynamic_partition_num = getFeConfig('max_dynamic_partition_num') + try { + sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_1' + sql '''CREATE TABLE test_dynamic_partition_failed_1 + ( `k1` datetime NULL COMMENT ) + PARTITION BY RANGE (k1)() + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES + ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.end" = "3", + "dynamic_partition.time_unit" = "day", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.start" = "-99999999", + "dynamic_partition.history_partition_num" = "5", + "dynamic_partition.create_history_partition" = "true" + )''' + + setFeConfig('max_dynamic_partition_num', Integer.MAX_VALUE) + + sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_2' + test { + sql '''CREATE TABLE test_dynamic_partition_failed_2 + ( `k1` datetime NULL COMMENT ) + PARTITION BY RANGE (k1)() + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES + ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.end" = "3", + "dynamic_partition.time_unit" = "day", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.start" = "-99999999", + "dynamic_partition.create_history_partition" = "true" + )''' + exception 'InternalCatalog.createPartitionWithIndices.createFailed' + } + } finally { + setFeConfig('max_dynamic_partition_num', old_max_dynamic_partition_num) + sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_1' + sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_2' + } +} From 1ddb5708771dbd24e5577945bb7e99a9e699be11 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Fri, 31 May 2024 18:10:59 +0800 Subject: [PATCH 08/11] update --- .../dynamic_partition/test_dynamic_partition_failed.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy index 23f709b4c086bf..79c56db86c04b7 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy @@ -55,7 +55,7 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { "dynamic_partition.start" = "-99999999", "dynamic_partition.create_history_partition" = "true" )''' - exception 'InternalCatalog.createPartitionWithIndices.createFailed' + exception 'date/datetime literal [+271768-09-11 00:00:00] is invalid' } } finally { setFeConfig('max_dynamic_partition_num', old_max_dynamic_partition_num) From 7908fbebcad5fe378c888be4c4ae1af192761d43 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Mon, 3 Jun 2024 10:30:18 +0800 Subject: [PATCH 09/11] update --- .../doris/clone/DynamicPartitionScheduler.java | 13 ++++++------- .../test_dynamic_partition_failed.groovy | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index d883cbfe569053..ad7faeaf6afa91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -53,7 +53,6 @@ import org.apache.doris.thrift.TStorageMedium; import com.google.common.base.Strings; -import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; @@ -266,7 +265,7 @@ private ArrayList getAddPartitionClause(Database db, OlapTab db.getFullName(), olapTable.getName(), idx, e); recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId()); - throw new DdlException(Throwables.getStackTraceAsString(e)); + throw new DdlException(e.getMessage()); } for (PartitionItem partitionItem : rangePartitionInfo.getIdToItem(false).values()) { // only support single column partition now @@ -285,7 +284,7 @@ private ArrayList getAddPartitionClause(Database db, OlapTab addPartitionKeyRange, db.getFullName(), olapTable.getName(), idx, e); recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId()); - throw new DdlException(Throwables.getStackTraceAsString(e)); + throw new DdlException(e.getMessage()); } break; } @@ -395,11 +394,11 @@ private Range getClosedRange(Database db, OlapTable olapTable, Col PartitionKey upperBorderBound = PartitionKey.createPartitionKey( Collections.singletonList(upperBorderPartitionValue), Collections.singletonList(partitionColumn)); reservedHistoryPartitionKeyRange = Range.closed(lowerBorderBound, upperBorderBound); - } catch (AnalysisException e) { + } catch (AnalysisException | org.apache.doris.nereids.exceptions.AnalysisException e) { // AnalysisException: keys.size is always equal to column.size, cannot reach this exception // IllegalArgumentException: lb is greater than ub - LOG.warn("Error in gen reservePartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), - db.getFullName(), olapTable.getName()); + LOG.warn("Error in gen reservePartitionKeyRange. {}, table: {}", + db.getFullName(), olapTable.getName(), e); } return reservedHistoryPartitionKeyRange; } @@ -456,7 +455,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange); } catch (IllegalArgumentException e) { return dropPartitionClauses; - } catch (AnalysisException e) { + } catch (AnalysisException | org.apache.doris.nereids.exceptions.AnalysisException e) { throw new DdlException(e.getMessage()); } } diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy index 79c56db86c04b7..7da145a36bcd0e 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy @@ -20,7 +20,7 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { try { sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_1' sql '''CREATE TABLE test_dynamic_partition_failed_1 - ( `k1` datetime NULL COMMENT ) + ( `k1` datetime NULL ) PARTITION BY RANGE (k1)() DISTRIBUTED BY HASH(`k1`) BUCKETS 1 PROPERTIES @@ -36,12 +36,15 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { "dynamic_partition.create_history_partition" = "true" )''' + def partitions = sql_return_maparray "SHOW PARTITIONS FROM test_dynamic_partition_failed_1" + assertEquals(9, partitions.size()); + setFeConfig('max_dynamic_partition_num', Integer.MAX_VALUE) sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_2' test { sql '''CREATE TABLE test_dynamic_partition_failed_2 - ( `k1` datetime NULL COMMENT ) + ( `k1` datetime NULL ) PARTITION BY RANGE (k1)() DISTRIBUTED BY HASH(`k1`) BUCKETS 1 PROPERTIES @@ -55,7 +58,13 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') { "dynamic_partition.start" = "-99999999", "dynamic_partition.create_history_partition" = "true" )''' - exception 'date/datetime literal [+271768-09-11 00:00:00] is invalid' + check { result, exception, startTime, endTime -> + assertNotNull(exception) + def msg = exception.toString() + logger.info("exception: " + msg) + // 'date/datetime literal [+271768-09-11 00:00:00] is invalid' + assertTrue(msg.contains('date/datetime literal') && msg.contains('is invalid')) + } } } finally { setFeConfig('max_dynamic_partition_num', old_max_dynamic_partition_num) From 25ff2d2d95109285c5851dff52580a06cd24e4b8 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Mon, 3 Jun 2024 10:47:50 +0800 Subject: [PATCH 10/11] update --- .../org/apache/doris/clone/DynamicPartitionScheduler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index ad7faeaf6afa91..28e9df38751a55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -39,7 +39,6 @@ import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.Table; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; @@ -394,7 +393,7 @@ private Range getClosedRange(Database db, OlapTable olapTable, Col PartitionKey upperBorderBound = PartitionKey.createPartitionKey( Collections.singletonList(upperBorderPartitionValue), Collections.singletonList(partitionColumn)); reservedHistoryPartitionKeyRange = Range.closed(lowerBorderBound, upperBorderBound); - } catch (AnalysisException | org.apache.doris.nereids.exceptions.AnalysisException e) { + } catch (org.apache.doris.common.AnalysisException | org.apache.doris.nereids.exceptions.AnalysisException e) { // AnalysisException: keys.size is always equal to column.size, cannot reach this exception // IllegalArgumentException: lb is greater than ub LOG.warn("Error in gen reservePartitionKeyRange. {}, table: {}", @@ -455,7 +454,8 @@ private ArrayList getDropPartitionClause(Database db, OlapT reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange); } catch (IllegalArgumentException e) { return dropPartitionClauses; - } catch (AnalysisException | org.apache.doris.nereids.exceptions.AnalysisException e) { + } catch (org.apache.doris.common.AnalysisException + | org.apache.doris.nereids.exceptions.AnalysisException e) { throw new DdlException(e.getMessage()); } } From 2799ded39298dc55b1c399a42f0bc94976e157f4 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Mon, 3 Jun 2024 15:04:09 +0800 Subject: [PATCH 11/11] update --- .../org/apache/doris/catalog/ColocateGroupSchema.java | 5 +++-- .../main/java/org/apache/doris/common/ErrorCode.java | 4 ++-- .../test/java/org/apache/doris/alter/AlterTest.java | 10 ++++++++-- .../org/apache/doris/catalog/ColocateTableTest.java | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java index 40b3405069d8b4..72ed921633a5ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java @@ -89,12 +89,13 @@ public void checkDistribution(DistributionInfo distributionInfo) throws DdlExcep HashDistributionInfo info = (HashDistributionInfo) distributionInfo; // buckets num if (info.getBucketNum() != bucketsNum) { - ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM, bucketsNum); + ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM, + info.getBucketNum(), bucketsNum); } // distribution col size if (info.getDistributionColumns().size() != distributionColTypes.size()) { ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_SIZE, - distributionColTypes.size()); + info.getDistributionColumns().size(), distributionColTypes.size()); } // distribution col type for (int i = 0; i < distributionColTypes.size(); i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java index 28b9142dc76962..e66bdef133c957 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java @@ -1123,9 +1123,9 @@ public enum ErrorCode { ERR_COLOCATE_TABLE_MUST_HAS_SAME_REPLICATION_ALLOCATION(5063, new byte[]{'4', '2', '0', '0', '0'}, "Colocate tables must have same replication allocation: { %s } should be { %s }"), ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM(5063, new byte[]{'4', '2', '0', '0', '0'}, - "Colocate tables must have same bucket num: %s"), + "Colocate tables must have same bucket num: %s should be %s"), ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_SIZE(5063, new byte[]{'4', '2', '0', '0', '0'}, - "Colocate tables distribution columns size must be same : %s"), + "Colocate tables distribution columns size must be same: %s should be %s"), ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_TYPE(5063, new byte[]{'4', '2', '0', '0', '0'}, "Colocate tables distribution columns must have the same data type: %s should be %s"), ERR_COLOCATE_NOT_COLOCATE_TABLE(5064, new byte[]{'4', '2', '0', '0', '0'}, diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index f0c988607a2d35..c2032355156755 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -163,11 +163,12 @@ public static void beforeClass() throws Exception { + "PROPERTIES\n" + "(\n" + "\"colocate_with\" = \"group_3\",\n" + + "\"replication_num\" = \"1\",\n" + "\"dynamic_partition.enable\" = \"true\",\n" + "\"dynamic_partition.time_unit\" = \"DAY\",\n" + "\"dynamic_partition.end\" = \"3\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" - + "\"dynamic_partition.buckets\" = \"32\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + "\"dynamic_partition.replication_num\" = \"1\",\n" + "\"dynamic_partition.create_history_partition\"=\"true\",\n" + "\"dynamic_partition.start\" = \"-3\"\n" @@ -254,7 +255,12 @@ public static void tearDown() { private static void createTable(String sql) throws Exception { Config.enable_odbc_mysql_broker_table = true; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); - Env.getCurrentEnv().createTable(createTableStmt); + try { + Env.getCurrentEnv().createTable(createTableStmt); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } } private static void createRemoteStorageResource(String sql) throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java index 87b51808431a42..6def5f1774b9eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java @@ -222,7 +222,7 @@ public void testBucketNum() throws Exception { + ");"); expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Colocate tables must have same bucket num: 1"); + expectedEx.expectMessage("Colocate tables must have same bucket num: 2 should be 1"); createTable("create table " + dbName + "." + tableName2 + " (\n" + " `k1` int NULL COMMENT \"\",\n" + " `k2` varchar(10) NULL COMMENT \"\"\n" @@ -282,7 +282,7 @@ public void testDistributionColumnsSize() throws Exception { + ");"); expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Colocate tables distribution columns size must be same : 2"); + expectedEx.expectMessage("Colocate tables distribution columns size must be same: 1 should be 2"); createTable("create table " + dbName + "." + tableName2 + " (\n" + " `k1` int NULL COMMENT \"\",\n" + " `k2` varchar(10) NULL COMMENT \"\"\n"