From a3d6a79c27187246d89009754ebcfe463c6d167f Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Thu, 2 Sep 2021 10:21:00 +0000 Subject: [PATCH 01/16] modify dynamic partition. --- .../analysis/ShowDynamicPartitionStmt.java | 2 + .../catalog/DynamicPartitionProperty.java | 32 +++++- .../clone/DynamicPartitionScheduler.java | 54 +++++++--- .../org/apache/doris/common/ErrorCode.java | 8 ++ .../common/util/DynamicPartitionUtil.java | 102 +++++++++++++++++- .../org/apache/doris/qe/ShowExecutor.java | 4 +- 6 files changed, 184 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java index ab8994ae5fae0c..065771441b9b14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java @@ -46,6 +46,8 @@ public class ShowDynamicPartitionStmt extends ShowStmt { .addColumn(new Column("State", ScalarType.createVarchar(20))) .addColumn(new Column("LastCreatePartitionMsg", ScalarType.createVarchar(20))) .addColumn(new Column("LastDropPartitionMsg", ScalarType.createVarchar(20))) + .addColumn(new Column("ReservedHistoryStarts(part)", ScalarType.createVarchar(21))) + .addColumn(new Column("ReservedHistoryEnds(part)", ScalarType.createVarchar(21))) .build(); ShowDynamicPartitionStmt(String db) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index ecbe7f26419fbf..677dfbfb8f7652 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -43,11 +43,16 @@ public class DynamicPartitionProperty { public static final String CREATE_HISTORY_PARTITION = "dynamic_partition.create_history_partition"; public static final String HISTORY_PARTITION_NUM = "dynamic_partition.history_partition_num"; public static final String HOT_PARTITION_NUM = "dynamic_partition.hot_partition_num"; + public static final String RESERVED_HISTORY_STARTS = "dynamic_partition.reserved_history_starts"; + public static final String RESERVED_HISTORY_ENDS = "dynamic_partition.reserved_history_ends"; + public static final int MIN_START_OFFSET = Integer.MIN_VALUE; public static final int MAX_END_OFFSET = Integer.MAX_VALUE; public static final int NOT_SET_REPLICATION_NUM = -1; public static final int NOT_SET_HISTORY_PARTITION_NUM = -1; + public static final String NOT_SET_RESERVED_HISTORY_STARTS = "0000-01-01"; + public static final String NOT_SET_RESERVED_HISTORY_ENDS = "0000-01-01"; private boolean exist; @@ -67,6 +72,8 @@ public class DynamicPartitionProperty { // This property are used to describe the number of partitions that need to be reserved on the high-speed storage. // If not set, default is 0 private int hotPartitionNum; + private String reservedHistoryStarts; + private String reservedHistoryEnds; public DynamicPartitionProperty(Map properties) { if (properties != null && !properties.isEmpty()) { @@ -83,6 +90,8 @@ public DynamicPartitionProperty(Map properties) { this.createHistoryPartition = Boolean.parseBoolean(properties.get(CREATE_HISTORY_PARTITION)); this.historyPartitionNum = Integer.parseInt(properties.getOrDefault(HISTORY_PARTITION_NUM, String.valueOf(NOT_SET_HISTORY_PARTITION_NUM))); this.hotPartitionNum = Integer.parseInt(properties.getOrDefault(HOT_PARTITION_NUM, "0")); + this.reservedHistoryStarts = properties.getOrDefault(RESERVED_HISTORY_STARTS, NOT_SET_RESERVED_HISTORY_STARTS); + this.reservedHistoryEnds = properties.getOrDefault(RESERVED_HISTORY_ENDS, NOT_SET_RESERVED_HISTORY_ENDS); createStartOfs(properties); } else { this.exist = false; @@ -114,6 +123,8 @@ private void createStartOfs(Map properties) { } } + + public boolean isExist() { return exist; } @@ -180,6 +191,22 @@ public ReplicaAllocation getReplicaAllocation() { return replicaAlloc; } + public String getReservedHistoryStarts() { + return reservedHistoryStarts; + } + + public void setReservedHistoryStarts(String reservedHistoryStarts) { + this.reservedHistoryStarts = reservedHistoryStarts; + } + + public String getReservedHistoryEnds() { + return reservedHistoryEnds; + } + + public void setReservedHistoryEnds(String reservedHistoryEnds) { + this.reservedHistoryEnds = reservedHistoryEnds; + } + /** * use table replication_num as dynamic_partition.replication_num default value */ @@ -195,12 +222,15 @@ public String getProperties(ReplicaAllocation tableReplicaAlloc) { ",\n\"" + BUCKETS + "\" = \"" + buckets + "\"" + ",\n\"" + CREATE_HISTORY_PARTITION + "\" = \"" + createHistoryPartition + "\"" + ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\"" + - ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\""; + ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\"" + + ",\n\"" + RESERVED_HISTORY_STARTS + "\" = \"" + reservedHistoryStarts + "\"" + + ",\n\"" + RESERVED_HISTORY_ENDS + "\" = \"" + reservedHistoryEnds + "\""; if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) { res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\""; } else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) { res += ",\n\"" + START_DAY_OF_MONTH + "\" = \"" + startOfMonth.day + "\""; } + return res; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 62905a75aa498e..8819378d7d78fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -37,7 +37,6 @@ import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DynamicPartitionUtil; @@ -261,14 +260,16 @@ private ArrayList getDropPartitionClause(Database db, OlapT String lowerBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, dynamicPartitionProperty.getStart(), partitionFormat); String upperBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, - now, 0, partitionFormat); + now, dynamicPartitionProperty.getEnd(), partitionFormat); PartitionValue lowerPartitionValue = new PartitionValue(lowerBorder); PartitionValue upperPartitionValue = new PartitionValue(upperBorder); + List> reservedHistoryPartitionKeyRangeList = new ArrayList>(); Range reservePartitionKeyRange; try { PartitionKey lowerBound = PartitionKey.createPartitionKey(Collections.singletonList(lowerPartitionValue), Collections.singletonList(partitionColumn)); PartitionKey upperBound = PartitionKey.createPartitionKey(Collections.singletonList(upperPartitionValue), Collections.singletonList(partitionColumn)); reservePartitionKeyRange = Range.closedOpen(lowerBound, upperBound); + reservedHistoryPartitionKeyRangeList.add(reservePartitionKeyRange); } catch (AnalysisException | IllegalArgumentException e) { // AnalysisException: keys.size is always equal to column.size, cannot reach this exception // IllegalArgumentException: lb is greater than ub @@ -276,23 +277,50 @@ private ArrayList getDropPartitionClause(Database db, OlapT db.getFullName(), olapTable.getName()); return dropPartitionClauses; } + String[] reservedHistoryStarts = dynamicPartitionProperty.getReservedHistoryStarts().split(","); + String[] reservedHistoryEnds = dynamicPartitionProperty.getReservedHistoryEnds().split(","); + + Range reservedHistoryPartitionKeyRange; + for (int i = 0; i < reservedHistoryStarts.length; i++) { + String lowBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, reservedHistoryStarts[i], partitionFormat); + String upBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, reservedHistoryEnds[i], partitionFormat); + PartitionValue lowBorderPartitionValue = new PartitionValue(lowBorderOfReservedHistory); + PartitionValue upBorderPartitionValue = new PartitionValue(upBorderOfReservedHistory); + try { + PartitionKey lowerBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(lowBorderPartitionValue), Collections.singletonList(partitionColumn)); + PartitionKey upperBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(upBorderPartitionValue), Collections.singletonList(partitionColumn)); + reservedHistoryPartitionKeyRange = Range.closedOpen(lowerBorderBound, upperBorderBound); + reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange); + + } catch (AnalysisException | IllegalArgumentException e) { + LOG.warn("Error in gen reserveHistoryPartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), + db.getFullName(), olapTable.getName()); + return dropPartitionClauses; + } + } + RangePartitionInfo info = (RangePartitionInfo) (olapTable.getPartitionInfo()); List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem) o.getValue()).getItems().upperEndpoint())); + Map isContaineds = new HashMap<>(); for (Map.Entry idToItem : idToItems) { - try { - Long checkDropPartitionId = idToItem.getKey(); - Range checkDropPartitionKey = idToItem.getValue().getItems(); - RangeUtils.checkRangeIntersect(reservePartitionKeyRange, checkDropPartitionKey); - if (checkDropPartitionKey.upperEndpoint().compareTo(reservePartitionKeyRange.lowerEndpoint()) <= 0) { - String dropPartitionName = olapTable.getPartition(checkDropPartitionId).getName(); - // Do not drop the partition "by force", or the partition will be dropped directly instread of being in - // catalog recycle bin. This is for safe reason. - dropPartitionClauses.add(new DropPartitionClause(false, dropPartitionName, false, false)); + isContaineds.put(idToItem.getKey(), false); + Long checkDropPartitionId = idToItem.getKey(); + Range checkDropPartitionKey = idToItem.getValue().getItems(); + for (Range reserveHistoryPartitionKeyRange : reservedHistoryPartitionKeyRangeList) { + if (reserveHistoryPartitionKeyRange.contains(checkDropPartitionKey.lowerEndpoint()) || checkDropPartitionKey.lowerEndpoint().compareTo(reserveHistoryPartitionKeyRange.upperEndpoint()) == 0) { + isContaineds.put(checkDropPartitionId, true); } - } catch (DdlException e) { - break; + } + } + for (Long dropPartitionId : isContaineds.keySet() + ) { + // Do not drop the partition "by force", or the partition will be dropped directly instread of being in + // catalog recycle bin. This is for safe reason. + if(!isContaineds.get(dropPartitionId)) { + String dropPartitionName = olapTable.getPartition(dropPartitionId).getName(); + dropPartitionClauses.add(new DropPartitionClause(false, dropPartitionName, false, false)); } } return dropPartitionClauses; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java index d6497dd3b85630..6bdc81d55dad91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java @@ -244,6 +244,14 @@ public enum ErrorCode { "Invalid dynamic partition create_history_partition: %s. Expected true or false"), ERROR_DYNAMIC_PARTITION_HISTORY_PARTITION_NUM_ZERO(5075, new byte[] {'4', '2', '0', '0', '0'}, "Dynamic history partition num must greater than 0"), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_EMPTY(5076, new byte[] {'4', '2', '0', '0', '0'}, + "Dynamic reserved history starts is empty."), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_ENDS_EMPTY(5077, new byte[] {'4', '2', '0', '0', '0'}, + "Dynamic reserved history ends is empty."), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL(5078, new byte[] {'4', '2', '0', '0', '0'}, + "RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: %s, RESERVED_HISTORY_STARTS length: %s."), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_LARGER_THAN_ENDS(5079, new byte[] {'4', '2', '0', '0', '0'}, + "RESERVED_HISTORY_STARTS %s is larger than RESERVED_HISTORY_ENDS %s."), ERROR_LDAP_CONFIGURATION_ERR(5080, new byte[] {'4', '2', '0', '0', '0'}, "LDAP configuration is incorrect or LDAP admin password is not set."), ERROR_LDAP_USER_NOT_UNIQUE_ERR(5081, new byte[] {'4', '2', '0', '0', '0'}, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 174673bb887eae..21e452fae16ad9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -45,16 +45,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.DayOfWeek; import java.time.Month; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.Calendar; import java.util.HashMap; import java.util.Map; import java.util.TimeZone; +import java.util.Date; +import java.util.Calendar; public class DynamicPartitionUtil { private static final Logger LOG = LogManager.getLogger(DynamicPartitionUtil.class); @@ -230,6 +233,60 @@ private static void checkHotPartitionNum(String val) throws DdlException { } } + private static void checkReservedHistoryStarts(String reservedHistoryStarts) throws DdlException{ + String[] starts = reservedHistoryStarts.split(","); + if (starts.length == 0) { + //ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_EMPTY); + throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS); + } + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Date date = null; + try { + for (int i = 0; i < starts.length; i++) { + date = sdf.parse(starts[i]); + if (!starts[i].equals(sdf.format(date))) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be correct DATE value."); + } + } + } catch (ParseException e) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be \"yyyy-MM-dd\"."); + } + } + + private static void checkReservedHistoryEnds(String reservedHistoryEnds) throws DdlException{ + String[] ends = reservedHistoryEnds.split(","); + if (ends.length == 0) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_ENDS_EMPTY); + //throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + } + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Date date = null; + try { + for (int i = 0; i < ends.length; i++) { + date = sdf.parse(ends[i]); + if (!ends[i].equals(sdf.format(date))) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be correct DATE value."); + } + } + } catch (ParseException e) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be \"yyyy-MM-dd\"."); + } + } + + private static void checkReservedHistoryPeriodValidate(String reservedHistoryStarts, String reservedHistoryEnds) throws DdlException { + String[] starts = reservedHistoryStarts.split(","); + String[] ends = reservedHistoryEnds.split(","); + if (starts.length != ends.length) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, starts.length, ends.length); + //throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " and " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + } + for (int i = 0; i < starts.length; i++) { + if (starts[i].compareTo(ends[i]) > 0) { + //ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_LARGER_THAN_ENDS, starts[i], ends[i]); + throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " is larger than " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + } + } + } public static boolean checkDynamicPartitionPropertiesExist(Map properties) { if (properties == null) { return false; @@ -261,6 +318,9 @@ public static boolean checkInputDynamicPartitionProperties(Map p String enable = properties.get(DynamicPartitionProperty.ENABLE); String createHistoryPartition = properties.get(DynamicPartitionProperty.CREATE_HISTORY_PARTITION); String historyPartitionNum = properties.get(DynamicPartitionProperty.HISTORY_PARTITION_NUM); + String reservedHistoryStarts = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_STARTS); + String reservedHistoryEnds = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + if (!(Strings.isNullOrEmpty(enable) && Strings.isNullOrEmpty(timeUnit) && Strings.isNullOrEmpty(timeZone) && @@ -269,7 +329,9 @@ public static boolean checkInputDynamicPartitionProperties(Map p Strings.isNullOrEmpty(end) && Strings.isNullOrEmpty(buckets) && Strings.isNullOrEmpty(createHistoryPartition) && - Strings.isNullOrEmpty(historyPartitionNum))) { + Strings.isNullOrEmpty(historyPartitionNum) && + Strings.isNullOrEmpty(reservedHistoryStarts) && + Strings.isNullOrEmpty(reservedHistoryEnds))) { if (Strings.isNullOrEmpty(enable)) { properties.put(DynamicPartitionProperty.ENABLE, "true"); } @@ -298,6 +360,14 @@ public static boolean checkInputDynamicPartitionProperties(Map p properties.put(DynamicPartitionProperty.HISTORY_PARTITION_NUM, String.valueOf(DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM)); } + if (Strings.isNullOrEmpty(reservedHistoryStarts)) { + properties.put(DynamicPartitionProperty.RESERVED_HISTORY_STARTS, + String.valueOf(DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_STARTS)); + } + if (Strings.isNullOrEmpty(reservedHistoryEnds)) { + properties.put(DynamicPartitionProperty.RESERVED_HISTORY_ENDS, + String.valueOf(DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_ENDS)); + } } return true; } @@ -450,7 +520,18 @@ public static Map analyzeDynamicPartition(Map pr properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM); analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val); } - + if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_STARTS)) { + String val = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_STARTS); + checkReservedHistoryStarts(val); + properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_STARTS); + analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_STARTS, val); + } + if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_ENDS)) { + String val = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + checkReservedHistoryEnds(val); + properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_ENDS, val); + } return analyzedProperties; } @@ -553,6 +634,21 @@ public static String getPartitionRangeString(DynamicPartitionProperty property, } } + public static String getHistoryPartitionRangeString(DynamicPartitionProperty dynamicPartitionProperty, String time, String format) { + ZoneId zoneId = dynamicPartitionProperty.getTimeZone().toZoneId(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + Date date = null; + try { + date = simpleDateFormat.parse(time); + } catch (ParseException e) { + e.printStackTrace(); + } + Timestamp timestamp = new Timestamp(date.getTime()); + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.s").withZone(zoneId); + + return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); + } + /** * return formatted string of partition range in HOUR granularity. * offset: The offset from the current hour. 0 means current hour, 1 means next hour, -1 last hour. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8778104433c616..a9bfdccd1a03e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1832,7 +1832,9 @@ private void handleShowDynamicPartition() { dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.LAST_SCHEDULER_TIME), dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DYNAMIC_PARTITION_STATE), dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.CREATE_PARTITION_MSG), - dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DROP_PARTITION_MSG))); + dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DROP_PARTITION_MSG), + dynamicPartitionProperty.getReservedHistoryStarts(), + dynamicPartitionProperty.getReservedHistoryEnds())); } finally { olapTable.readUnlock(); } From bc7b70f1993e0d8619e04911bf26321fd8ffdf96 Mon Sep 17 00:00:00 2001 From: wuhangze Date: Fri, 3 Sep 2021 14:30:15 +0800 Subject: [PATCH 02/16] add some ut and add documents --- .../administrator-guide/dynamic-partition.md | 21 ++ .../operation/doris-error-code.md | 5 +- .../Create/CREATE-TABLE.md | 2 + .../Data Definition/CREATE TABLE.md | 2 + .../administrator-guide/dynamic-partition.md | 41 ++- .../operation/doris-error-code.md | 4 + .../Create/CREATE-TABLE.md | 2 + .../Data Definition/CREATE TABLE.md | 2 + .../analysis/ShowDynamicPartitionStmt.java | 6 +- .../catalog/DynamicPartitionProperty.java | 12 +- .../clone/DynamicPartitionScheduler.java | 10 +- .../common/util/DynamicPartitionUtil.java | 39 +- .../catalog/DynamicPartitionTableTest.java | 332 ++++++++++++++++++ 13 files changed, 433 insertions(+), 45 deletions(-) diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md index 9a947dd80fa630..d9099b2461dea4 100644 --- a/docs/en/administrator-guide/dynamic-partition.md +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -151,6 +151,27 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: p20210523: ["2021-05-23", "2021-05-24") storage_medium=SSD storage_cooldown_time=2021-05-25 00:00:00 ``` +* `dynamic_partition.reserved_history_starts` + + Start times of reserved history period. It should be in the form of `yyyy-MM-dd,yyyy-MM-dd,...`. The default value is `"9999-12-31"`, which means it is not set. + +* `dynamic_partition.reserved_history_ends` + + End times of reserved history period. It should be in the form of `yyyy-MM-dd,yyyy-MM-dd,...`. The default value is `"9999-12-31"`, which means it is not set. + + Let us give an example. Suppose today is 2021-09-06，partitioned by day, and the properties of dynamic partition are set to: + + ```end=3, start=-3, reserved_history_starts="2020-06-01,2020-10-31", reserved_history_ends="2020-06-20,2020-11-15"```。 + + The the system will automatically reserve following partitions in following period : + + ``` + ["2020-06-01","2020-06-20"), + ["2020-10-31","2020-11-15") + ``` + + Otherwise, `reserved_history_starts` and `reserved_history_ends` are a couple of properties. They shoule be set at the same time, and the length should also be the same. And `reserved_history_starts` can't larger than `reserved_history_ends` in the corresponding position. + #### Create History Partition Rules When `create_history_partition` is `true`, i.e. history partition creation is enabled, Doris determines the number of history partitions to be created based on `dynamic_partition.start` and `dynamic_partition.history_partition_num`. diff --git a/docs/en/administrator-guide/operation/doris-error-code.md b/docs/en/administrator-guide/operation/doris-error-code.md index 1ee7bedcaabfc2..5501139c72b27a 100644 --- a/docs/en/administrator-guide/operation/doris-error-code.md +++ b/docs/en/administrator-guide/operation/doris-error-code.md @@ -172,4 +172,7 @@ under the License. | 5072 | The dynamic partition copy value is not a valid number | | 5073 | The original created table stmt is empty | | 5074 | Create historical dynamic partition parameters: create_history_partition is invalid, what is expected is: true or false | - +| 5076 | The specified dynamic partition reserved_history_starts is null | +| 5077 | The specified dynamic partition reserved_history_ends is null | +| 5078 | The length of specified dynamic partition reserved_history_starts is not equal to reserved_history_ends' | +| 5079 | The specified dynamic partition reserved_history_starts is larger than reserved_history_ends | diff --git a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md index d7dbeb305fa5ab..aa8532f9e905b3 100644 --- a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -287,6 +287,8 @@ distribution_info * `dynamic_partition.buckets`: Used to specify the number of partition buckets that are automatically created. * `dynamic_partition.create_history_partition`: Whether to create a history partition. * `dynamic_partition.history_partition_num`: Specify the number of historical partitions to be created. + * `dynamic_partition.reserved_history_starts`: Used to specify the starting dates of reserved history periods. + * `dynamic_partition.reserved_history_ends`: Used to specify the ending dates of reserved history periods. ### Example diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 2664a002089faa..2912bf11285d81 100644 --- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -311,6 +311,8 @@ Syntax: dynamic_partition.buckets: specifies the number of partition buckets that are automatically created dynamic_partition.create_history_partition: specifies whether create history partitions, default value is false dynamic_partition.history_partition_num: used to specify the number of history partitions when enable create_history_partition + dynamic_partition.reserved_history_starts: Used to specify the starting dates of reserved history periods + dynamic_partition.reserved_history_ends: Used to specify the ending dates of reserved history periods ``` 5) You can create multiple Rollups in bulk when building a table grammar: diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md index efe84fa8ad8055..583933f46d6528 100644 --- a/docs/zh-CN/administrator-guide/dynamic-partition.md +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -149,6 +149,27 @@ under the License. p20210523：["2021-05-23", "2021-05-24") storage_medium=SSD storage_cooldown_time=2021-05-25 00:00:00 ``` +* `dynamic_partition.reserved_history_starts` + + 需要保留的历史分区的开始时间。需要以 `yyyy-MM-dd,yyyy-MM-dd,...` 格式进行设置。如果不设置，默认为 `"9999-12-31"`。 + +* `dynamic_partition.reserved_history_ends` + + 需要保留的历史分区的结束时间。需要以 `yyyy-MM-dd,yyyy-MM-dd,...` 格式进行设置。如果不设置，默认为 `"9999-12-31"`。 + + 我们举例说明。假设今天是 2021-09-06，按天分类，动态分区的属性设置为： + + ```end=3, start=-3, reserved_history_starts="2020-06-01,2020-10-31", reserved_history_ends="2020-06-20,2020-11-15"```。 + + 则系统会自动保留： + + ``` + ["2020-06-01","2020-06-20"), + ["2020-10-31","2020-11-15") + ``` + + 这两个时间段的分区。其中，`reserved_history_starts` 和 `reserved_history_ends` 是一对设置项，两者需要同时被设置，且两者的长度需要一致。另外，对应位置的 `reserved_history_starts` 不能大于 `reserved_history_ends`。 + #### 创建历史分区规则 当 `create_history_partition` 为 `true`，即开启创建历史分区功能时，Doris 会根据 `dynamic_partition.start` 和 `dynamic_partition.history_partition_num` 来决定创建历史分区的个数。 @@ -355,16 +376,16 @@ p20200521: ["2020-05-21", "2020-05-22") ``` mysql> SHOW DYNAMIC PARTITION TABLES; -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+ -| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+ -| d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+ ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+--------------------------------------------------------+--------------------------------------------------------+ +| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | ReservedHistoryStarts | ReservedHistoryEnds | ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+--------------------------------------------------------+--------------------------------------------------------+ +| d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 2020-06-01,2020-10-25,2021-06-01,2021-10-25 | 2020-06-20,2020-11-15,2021-06-20,2021-11-15 | +| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 2020-01-01 | 2021-06-01 | +| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | +| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | +| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | +| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+--------------------------------------------------------+--------------------------------------------------------+ 7 rows in set (0.02 sec) ``` diff --git a/docs/zh-CN/administrator-guide/operation/doris-error-code.md b/docs/zh-CN/administrator-guide/operation/doris-error-code.md index 07123ed0936700..f050d7f20da100 100644 --- a/docs/zh-CN/administrator-guide/operation/doris-error-code.md +++ b/docs/zh-CN/administrator-guide/operation/doris-error-code.md @@ -172,4 +172,8 @@ under the License. | 5072 | 动态分区副本值不是有效的数字 | | 5073 | 原始创建表stmt为空 | | 5074 | 创建历史动态分区参数：create_history_partition无效，期望的是：true或者false | +| 5076 | 指定的保留历史分区的起始时间为空 | +| 5077 | 指定的保留历史分区的结束时间为空 | +| 5078 | 指定的保留历史分区的起始时间和结束时间长度不一致 | +| 5079 | 指定的保留历史分区对应位置的的起始时间比结束时间大 | diff --git a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md index fdb00a47edb343..df5179c591d892 100644 --- a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -287,6 +287,8 @@ distribution_info * `dynamic_partition.buckets`: 用于指定自动创建的分区分桶数量。 * `dynamic_partition.create_history_partition`: 是否创建历史分区。 * `dynamic_partition.history_partition_num`: 指定创建历史分区的数量。 + * `dynamic_partition.reserved_history_starts`: 用于指定保留的历史分区的起始时间。 + * `dynamic_partition.reserved_history_ends`: 用于指定保留的历史分区的结束时间。 ### Example diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 15abbbfbab036c..bfacf02945dec5 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -334,6 +334,8 @@ under the License. dynamic_partition.buckets: 用于指定自动创建的分区分桶数量 dynamic_partition.create_history_partition: 用于创建历史分区功能是否开启。默认为 false。 dynamic_partition.history_partition_num: 当开启创建历史分区功能时，用于指定创建历史分区数量。 + dynamic_partition.reserved_history_starts: 用于指定保留的历史分区的起始时间。 + dynamic_partition.reserved_history_ends: 用于指定保留的历史分区的结束时间。 5) 建表时可以批量创建多个 Rollup 语法： diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java index 065771441b9b14..6c7cc4580aeba9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java @@ -46,8 +46,8 @@ public class ShowDynamicPartitionStmt extends ShowStmt { .addColumn(new Column("State", ScalarType.createVarchar(20))) .addColumn(new Column("LastCreatePartitionMsg", ScalarType.createVarchar(20))) .addColumn(new Column("LastDropPartitionMsg", ScalarType.createVarchar(20))) - .addColumn(new Column("ReservedHistoryStarts(part)", ScalarType.createVarchar(21))) - .addColumn(new Column("ReservedHistoryEnds(part)", ScalarType.createVarchar(21))) + .addColumn(new Column("ReservedHistoryStarts", ScalarType.createVarchar(20))) + .addColumn(new Column("ReservedHistoryEnds", ScalarType.createVarchar(20))) .build(); ShowDynamicPartitionStmt(String db) { @@ -97,4 +97,4 @@ public ShowResultSetMetaData getMetaData() { public RedirectStatus getRedirectStatus() { return RedirectStatus.FORWARD_NO_SYNC; } -} \ No newline at end of file +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 677dfbfb8f7652..3ee91627e9fec5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -46,13 +46,12 @@ public class DynamicPartitionProperty { public static final String RESERVED_HISTORY_STARTS = "dynamic_partition.reserved_history_starts"; public static final String RESERVED_HISTORY_ENDS = "dynamic_partition.reserved_history_ends"; - public static final int MIN_START_OFFSET = Integer.MIN_VALUE; public static final int MAX_END_OFFSET = Integer.MAX_VALUE; public static final int NOT_SET_REPLICATION_NUM = -1; public static final int NOT_SET_HISTORY_PARTITION_NUM = -1; - public static final String NOT_SET_RESERVED_HISTORY_STARTS = "0000-01-01"; - public static final String NOT_SET_RESERVED_HISTORY_ENDS = "0000-01-01"; + public static final String NOT_SET_RESERVED_HISTORY_STARTS = "9999-12-31"; + public static final String NOT_SET_RESERVED_HISTORY_ENDS = "9999-12-31"; private boolean exist; @@ -90,8 +89,8 @@ public DynamicPartitionProperty(Map properties) { this.createHistoryPartition = Boolean.parseBoolean(properties.get(CREATE_HISTORY_PARTITION)); this.historyPartitionNum = Integer.parseInt(properties.getOrDefault(HISTORY_PARTITION_NUM, String.valueOf(NOT_SET_HISTORY_PARTITION_NUM))); this.hotPartitionNum = Integer.parseInt(properties.getOrDefault(HOT_PARTITION_NUM, "0")); - this.reservedHistoryStarts = properties.getOrDefault(RESERVED_HISTORY_STARTS, NOT_SET_RESERVED_HISTORY_STARTS); - this.reservedHistoryEnds = properties.getOrDefault(RESERVED_HISTORY_ENDS, NOT_SET_RESERVED_HISTORY_ENDS); + this.reservedHistoryStarts = properties.getOrDefault(RESERVED_HISTORY_STARTS, NOT_SET_RESERVED_HISTORY_STARTS).replace(" ", ""); + this.reservedHistoryEnds = properties.getOrDefault(RESERVED_HISTORY_ENDS, NOT_SET_RESERVED_HISTORY_ENDS).replace(" ", ""); createStartOfs(properties); } else { this.exist = false; @@ -123,8 +122,6 @@ private void createStartOfs(Map properties) { } } - - public boolean isExist() { return exist; } @@ -230,7 +227,6 @@ public String getProperties(ReplicaAllocation tableReplicaAlloc) { } else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) { res += ",\n\"" + START_DAY_OF_MONTH + "\" = \"" + startOfMonth.day + "\""; } - return res; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 8819378d7d78fb..a123b17aadd365 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -260,7 +260,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT String lowerBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, dynamicPartitionProperty.getStart(), partitionFormat); String upperBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, - now, dynamicPartitionProperty.getEnd(), partitionFormat); + now, dynamicPartitionProperty.getEnd() + 1, partitionFormat); PartitionValue lowerPartitionValue = new PartitionValue(lowerBorder); PartitionValue upperPartitionValue = new PartitionValue(upperBorder); List> reservedHistoryPartitionKeyRangeList = new ArrayList>(); @@ -298,7 +298,6 @@ private ArrayList getDropPartitionClause(Database db, OlapT return dropPartitionClauses; } } - RangePartitionInfo info = (RangePartitionInfo) (olapTable.getPartitionInfo()); List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); @@ -309,13 +308,14 @@ private ArrayList getDropPartitionClause(Database db, OlapT Long checkDropPartitionId = idToItem.getKey(); Range checkDropPartitionKey = idToItem.getValue().getItems(); for (Range reserveHistoryPartitionKeyRange : reservedHistoryPartitionKeyRangeList) { - if (reserveHistoryPartitionKeyRange.contains(checkDropPartitionKey.lowerEndpoint()) || checkDropPartitionKey.lowerEndpoint().compareTo(reserveHistoryPartitionKeyRange.upperEndpoint()) == 0) { + if (reserveHistoryPartitionKeyRange.contains(checkDropPartitionKey.lowerEndpoint()) + || (reserveHistoryPartitionKeyRange.lowerEndpoint().compareTo(checkDropPartitionKey.lowerEndpoint()) >=0 && reserveHistoryPartitionKeyRange.upperEndpoint().compareTo(checkDropPartitionKey.upperEndpoint()) <=0)) { isContaineds.put(checkDropPartitionId, true); } } } - for (Long dropPartitionId : isContaineds.keySet() - ) { + + for (Long dropPartitionId : isContaineds.keySet()) { // Do not drop the partition "by force", or the partition will be dropped directly instread of being in // catalog recycle bin. This is for safe reason. if(!isContaineds.get(dropPartitionId)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 21e452fae16ad9..3ac70773b86923 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -245,11 +245,11 @@ private static void checkReservedHistoryStarts(String reservedHistoryStarts) thr for (int i = 0; i < starts.length; i++) { date = sdf.parse(starts[i]); if (!starts[i].equals(sdf.format(date))) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be correct DATE value."); + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be correct DATE value (yyyy-MM-dd)"); } } } catch (ParseException e) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be \"yyyy-MM-dd\"."); + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be like \"yyyy-MM-dd\""); } } @@ -257,7 +257,6 @@ private static void checkReservedHistoryEnds(String reservedHistoryEnds) throws String[] ends = reservedHistoryEnds.split(","); if (ends.length == 0) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_ENDS_EMPTY); - //throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS); } SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); Date date = null; @@ -265,11 +264,11 @@ private static void checkReservedHistoryEnds(String reservedHistoryEnds) throws for (int i = 0; i < ends.length; i++) { date = sdf.parse(ends[i]); if (!ends[i].equals(sdf.format(date))) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be correct DATE value."); + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be correct DATE value (yyyy-MM-dd)"); } } } catch (ParseException e) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be \"yyyy-MM-dd\"."); + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be like \"yyyy-MM-dd\""); } } @@ -278,15 +277,19 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistorySta String[] ends = reservedHistoryEnds.split(","); if (starts.length != ends.length) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, starts.length, ends.length); - //throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " and " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + } else if (("9999-12-31".equals(starts[0]) && !"9999-12-31".equals(ends[0]))) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, 0, ends.length); + } else if (("9999-12-31".equals(ends[0]) && !"9999-12-31".equals(starts[0]))) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, starts.length, 0); } for (int i = 0; i < starts.length; i++) { - if (starts[i].compareTo(ends[i]) > 0) { - //ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_LARGER_THAN_ENDS, starts[i], ends[i]); - throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " is larger than " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + if (starts[i].compareTo(ends[i]) >= 0) { + throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + ": \"" + starts[i] + + "\" is equal to or larger than " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + ": \"" + ends[i] + "\""); } } } + public static boolean checkDynamicPartitionPropertiesExist(Map properties) { if (properties == null) { return false; @@ -520,17 +523,17 @@ public static Map analyzeDynamicPartition(Map pr properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM); analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val); } - if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_STARTS)) { - String val = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_STARTS); - checkReservedHistoryStarts(val); + if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_STARTS) && properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_ENDS)) { + String starts = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_STARTS).replace(" ", ""); + String ends = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_ENDS).replace(" ", ""); + + checkReservedHistoryStarts(starts); + checkReservedHistoryEnds(ends); + checkReservedHistoryPeriodValidate(starts, ends); properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_STARTS); - analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_STARTS, val); - } - if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_ENDS)) { - String val = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_ENDS); - checkReservedHistoryEnds(val); properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_ENDS); - analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_ENDS, val); + analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_STARTS, starts); + analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_ENDS, ends); } return analyzedProperties; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index efc2685d704e78..92d9cbfbc6d7f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1186,4 +1186,336 @@ public void testRuntimeInfo() throws Exception { scheduler.removeRuntimeInfo(tableId); Assert.assertTrue(scheduler.getRuntimeInfo(tableId, key1) == FeConstants.null_string); } + + @Test + public void testMissReservedHistoryStartsAndEnds() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_miss_reserved_history_starts_and_ends` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\"\n" + + ");"; + createTable(createOlapTblStmt); + OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_miss_reserved_history_starts_and_ends"); + Assert.assertEquals("9999-12-31", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryStarts()); + Assert.assertEquals("9999-12-31", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryEnds()); + } + + @Test + public void testNormalReservedHisrotyStartsAndEnds() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal_reserved_history_starts_and_ends` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\"),\n" + + "PARTITION p4 VALUES LESS THAN (\"2020-06-01\"),\n" + + "PARTITION p5 VALUES LESS THAN (\"2020-06-20\"),\n" + + "PARTITION p6 VALUES LESS THAN (\"2020-10-25\"),\n" + + "PARTITION p7 VALUES LESS THAN (\"2020-11-01\"),\n" + + "PARTITION p8 VALUES LESS THAN (\"2020-11-11\"),\n" + + "PARTITION p9 VALUES LESS THAN (\"2020-11-21\"),\n" + + "PARTITION p10 VALUES LESS THAN (\"2021-04-20\"),\n" + + "PARTITION p11 VALUES LESS THAN (\"2021-05-20\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"2020-06-01,2020-10-25,2021-06-01\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"2020-06-20,2020-11-15,2021-06-20\"\n" + + ");"; + createTable(createOlapTblStmt); + OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_normal_reserved_history_starts_and_ends"); + Assert.assertEquals("2020-06-01,2020-10-25,2021-06-01", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryStarts()); + Assert.assertEquals("2020-06-20,2020-11-15,2021-06-20", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryEnds()); + Assert.assertEquals(table.getAllPartitions().size(), 8); + } + + @Test + public void testInvalidReservedHistoryStarts() throws Exception { + String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_starts1` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"20210101\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"2021-10-10\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_starts value. It must be like \"yyyy-MM-dd\"", + () -> createTable(createOlapTblStmt1)); + + String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_starts2` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"0000-00-00\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"2021-10-10\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_starts value. It must be correct DATE value (yyyy-MM-dd)", + () -> createTable(createOlapTblStmt2)); + } + + @Test + public void testInvalidReservedHistoryEnds() throws Exception { + String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_ends` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"2021-10-12\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"20211010\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_ends value. It must be like \"yyyy-MM-dd\"", + () -> createTable(createOlapTblStmt1)); + + String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_ends2` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"2021-01-01\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"0000-00-00\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_ends value. It must be correct DATE value (yyyy-MM-dd)", + () -> createTable(createOlapTblStmt2)); + } + + @Test + public void testReservedHistoryPeriodValidate() throws Exception { + String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_reserved_history_period_validate1` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"2021-01-01\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: 1, RESERVED_HISTORY_STARTS length: 0.", + () -> createTable(createOlapTblStmt1)); + + String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_reserved_history_period_validate2` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"2021-01-01\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: 0, RESERVED_HISTORY_STARTS length: 1.", + () -> createTable(createOlapTblStmt2)); + + String createOlapTblStmt3 = "CREATE TABLE test.`dynamic_partition_reserved_history_starts_is_larger_than_ends` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"2020-01-01,2021-10-01\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"2020-03-01,2021-09-01\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = Invalid properties: dynamic_partition.reserved_history_starts: \"2021-10-01\" is larger than dynamic_partition.reserved_history_ends: \"2021-09-01\"", + () -> createTable(createOlapTblStmt3)); + + String createOlapTblStmt4 = "CREATE TABLE test.`dynamic_partition_reserved_history_starts_and_ends_length` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_starts\" = \"2020-01-01,2021-08-01,2021-11-01\",\n" + + "\"dynamic_partition.reserved_history_ends\" = \"2020-03-01,2021-09-01\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: 3, RESERVED_HISTORY_STARTS length: 2.", + () -> createTable(createOlapTblStmt4)); + } } From 78a7214090ec77e1e686bb27cdcc9ee35f1bad81 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Tue, 7 Sep 2021 17:57:03 +0800 Subject: [PATCH 03/16] fix --- .../java/org/apache/doris/common/util/DynamicPartitionUtil.java | 2 +- .../org/apache/doris/catalog/DynamicPartitionTableTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 3ac70773b86923..79f28ea98b543f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -283,7 +283,7 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistorySta ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, starts.length, 0); } for (int i = 0; i < starts.length; i++) { - if (starts[i].compareTo(ends[i]) >= 0) { + if (!"9999-12-31".equals(starts[i]) && starts[i].compareTo(ends[i]) >= 0) { throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + ": \"" + starts[i] + "\" is equal to or larger than " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + ": \"" + ends[i] + "\""); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 92d9cbfbc6d7f1..a4d01443f2a377 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1484,7 +1484,7 @@ public void testReservedHistoryPeriodValidate() throws Exception { "\"dynamic_partition.reserved_history_ends\" = \"2020-03-01,2021-09-01\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid properties: dynamic_partition.reserved_history_starts: \"2021-10-01\" is larger than dynamic_partition.reserved_history_ends: \"2021-09-01\"", + "errCode = 2, detailMessage = Invalid properties: dynamic_partition.reserved_history_starts: \"2021-10-01\" is equal to or larger than dynamic_partition.reserved_history_ends: \"2021-09-01\"", () -> createTable(createOlapTblStmt3)); String createOlapTblStmt4 = "CREATE TABLE test.`dynamic_partition_reserved_history_starts_and_ends_length` (\n" + From fed84fb82007abe4aef69235e5f366d5ec3b6d80 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Thu, 9 Sep 2021 14:24:32 +0800 Subject: [PATCH 04/16] change reserved history period range closedOpen to closed. --- docs/en/administrator-guide/dynamic-partition.md | 4 ++-- docs/zh-CN/administrator-guide/dynamic-partition.md | 4 ++-- .../org/apache/doris/clone/DynamicPartitionScheduler.java | 5 ++--- .../org/apache/doris/common/util/DynamicPartitionUtil.java | 7 +++---- .../apache/doris/catalog/DynamicPartitionTableTest.java | 4 ++-- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md index d9099b2461dea4..45c81fff320dc0 100644 --- a/docs/en/administrator-guide/dynamic-partition.md +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -166,8 +166,8 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: The the system will automatically reserve following partitions in following period : ``` - ["2020-06-01","2020-06-20"), - ["2020-10-31","2020-11-15") + ["2020-06-01","2020-06-20"], + ["2020-10-31","2020-11-15"] ``` Otherwise, `reserved_history_starts` and `reserved_history_ends` are a couple of properties. They shoule be set at the same time, and the length should also be the same. And `reserved_history_starts` can't larger than `reserved_history_ends` in the corresponding position. diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md index 583933f46d6528..84acb87dfdd029 100644 --- a/docs/zh-CN/administrator-guide/dynamic-partition.md +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -164,8 +164,8 @@ under the License. 则系统会自动保留： ``` - ["2020-06-01","2020-06-20"), - ["2020-10-31","2020-11-15") + ["2020-06-01","2020-06-20"], + ["2020-10-31","2020-11-15"] ``` 这两个时间段的分区。其中，`reserved_history_starts` 和 `reserved_history_ends` 是一对设置项，两者需要同时被设置，且两者的长度需要一致。另外，对应位置的 `reserved_history_starts` 不能大于 `reserved_history_ends`。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index a123b17aadd365..8d7acd40b0f287 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -289,7 +289,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT try { PartitionKey lowerBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(lowBorderPartitionValue), Collections.singletonList(partitionColumn)); PartitionKey upperBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(upBorderPartitionValue), Collections.singletonList(partitionColumn)); - reservedHistoryPartitionKeyRange = Range.closedOpen(lowerBorderBound, upperBorderBound); + reservedHistoryPartitionKeyRange = Range.closed(lowerBorderBound, upperBorderBound); reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange); } catch (AnalysisException | IllegalArgumentException e) { @@ -308,8 +308,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT Long checkDropPartitionId = idToItem.getKey(); Range checkDropPartitionKey = idToItem.getValue().getItems(); for (Range reserveHistoryPartitionKeyRange : reservedHistoryPartitionKeyRangeList) { - if (reserveHistoryPartitionKeyRange.contains(checkDropPartitionKey.lowerEndpoint()) - || (reserveHistoryPartitionKeyRange.lowerEndpoint().compareTo(checkDropPartitionKey.lowerEndpoint()) >=0 && reserveHistoryPartitionKeyRange.upperEndpoint().compareTo(checkDropPartitionKey.upperEndpoint()) <=0)) { + if (reserveHistoryPartitionKeyRange.contains(checkDropPartitionKey.lowerEndpoint())) { isContaineds.put(checkDropPartitionId, true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 79f28ea98b543f..32ca6ea8841bad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -236,8 +236,7 @@ private static void checkHotPartitionNum(String val) throws DdlException { private static void checkReservedHistoryStarts(String reservedHistoryStarts) throws DdlException{ String[] starts = reservedHistoryStarts.split(","); if (starts.length == 0) { - //ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_EMPTY); - throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS); + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_EMPTY); } SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); Date date = null; @@ -283,9 +282,9 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistorySta ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, starts.length, 0); } for (int i = 0; i < starts.length; i++) { - if (!"9999-12-31".equals(starts[i]) && starts[i].compareTo(ends[i]) >= 0) { + if (!"9999-12-31".equals(starts[i]) && starts[i].compareTo(ends[i]) > 0) { throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + ": \"" + starts[i] + - "\" is equal to or larger than " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + ": \"" + ends[i] + "\""); + "\" is larger than " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + ": \"" + ends[i] + "\""); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index a4d01443f2a377..70bd7a720c51b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1261,7 +1261,7 @@ public void testNormalReservedHisrotyStartsAndEnds() throws Exception { OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_normal_reserved_history_starts_and_ends"); Assert.assertEquals("2020-06-01,2020-10-25,2021-06-01", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryStarts()); Assert.assertEquals("2020-06-20,2020-11-15,2021-06-20", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryEnds()); - Assert.assertEquals(table.getAllPartitions().size(), 8); + Assert.assertEquals(table.getAllPartitions().size(), 9); } @Test @@ -1484,7 +1484,7 @@ public void testReservedHistoryPeriodValidate() throws Exception { "\"dynamic_partition.reserved_history_ends\" = \"2020-03-01,2021-09-01\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid properties: dynamic_partition.reserved_history_starts: \"2021-10-01\" is equal to or larger than dynamic_partition.reserved_history_ends: \"2021-09-01\"", + "errCode = 2, detailMessage = Invalid properties: dynamic_partition.reserved_history_starts: \"2021-10-01\" is larger than dynamic_partition.reserved_history_ends: \"2021-09-01\"", () -> createTable(createOlapTblStmt3)); String createOlapTblStmt4 = "CREATE TABLE test.`dynamic_partition_reserved_history_starts_and_ends_length` (\n" + From 5a0d88154afaa7c797a786ef801f733faf351532 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Mon, 13 Sep 2021 12:28:47 +0800 Subject: [PATCH 05/16] change it to list --- .../administrator-guide/dynamic-partition.md | 32 ++-- .../operation/doris-error-code.md | 8 +- .../Create/CREATE-TABLE.md | 3 +- .../Data Definition/CREATE TABLE.md | 3 +- .../administrator-guide/dynamic-partition.md | 32 ++-- .../operation/doris-error-code.md | 8 +- .../Create/CREATE-TABLE.md | 3 +- .../Data Definition/CREATE TABLE.md | 3 +- .../analysis/ShowDynamicPartitionStmt.java | 3 +- .../catalog/DynamicPartitionProperty.java | 38 ++--- .../clone/DynamicPartitionScheduler.java | 11 +- .../org/apache/doris/common/ErrorCode.java | 16 +- .../common/util/DynamicPartitionUtil.java | 108 ++++++------ .../org/apache/doris/qe/ShowExecutor.java | 4 +- .../catalog/DynamicPartitionTableTest.java | 154 +++--------------- 15 files changed, 148 insertions(+), 278 deletions(-) diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md index 45c81fff320dc0..58d91dfe40e35e 100644 --- a/docs/en/administrator-guide/dynamic-partition.md +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -151,17 +151,13 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: p20210523: ["2021-05-23", "2021-05-24") storage_medium=SSD storage_cooldown_time=2021-05-25 00:00:00 ``` -* `dynamic_partition.reserved_history_starts` +* `dynamic_partition.reserved_history_periods` - Start times of reserved history period. It should be in the form of `yyyy-MM-dd,yyyy-MM-dd,...`. The default value is `"9999-12-31"`, which means it is not set. - -* `dynamic_partition.reserved_history_ends` - - End times of reserved history period. It should be in the form of `yyyy-MM-dd,yyyy-MM-dd,...`. The default value is `"9999-12-31"`, which means it is not set. + The range of reserved history periods. It should be in the form of `[yyyy-MM-dd,yyyy-MM-dd],[...,...]`. The default value is `"[9999-12-31,9999-12-31]"`, which means it is not set. Let us give an example. Suppose today is 2021-09-06，partitioned by day, and the properties of dynamic partition are set to: - ```end=3, start=-3, reserved_history_starts="2020-06-01,2020-10-31", reserved_history_ends="2020-06-20,2020-11-15"```。 + ```end=3, start=-3, reserved_history_periods="[2020-06-01,2020-06-20],[2020-10-31,2020-11-15]"```。 The the system will automatically reserve following partitions in following period : @@ -170,7 +166,7 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: ["2020-10-31","2020-11-15"] ``` - Otherwise, `reserved_history_starts` and `reserved_history_ends` are a couple of properties. They shoule be set at the same time, and the length should also be the same. And `reserved_history_starts` can't larger than `reserved_history_ends` in the corresponding position. + Otherwise, every `[...,...]` in `reserved_history_periods` is a couple of properties, and they should be set at the same time. And the first date can't be larger than the second one. #### Create History Partition Rules @@ -378,16 +374,16 @@ You can further view the scheduling of dynamic partitioned tables by using the f ``` mysql> SHOW DYNAMIC PARTITION TABLES; -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+ -| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+ -| d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+ ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ +| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | ReservedHistoryPeriods | ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ +| d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [2021-12-01,2021-12-31] | +| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ 7 rows in set (0.02 sec) ``` diff --git a/docs/en/administrator-guide/operation/doris-error-code.md b/docs/en/administrator-guide/operation/doris-error-code.md index 5501139c72b27a..9e243222b0e463 100644 --- a/docs/en/administrator-guide/operation/doris-error-code.md +++ b/docs/en/administrator-guide/operation/doris-error-code.md @@ -172,7 +172,7 @@ under the License. | 5072 | The dynamic partition copy value is not a valid number | | 5073 | The original created table stmt is empty | | 5074 | Create historical dynamic partition parameters: create_history_partition is invalid, what is expected is: true or false | -| 5076 | The specified dynamic partition reserved_history_starts is null | -| 5077 | The specified dynamic partition reserved_history_ends is null | -| 5078 | The length of specified dynamic partition reserved_history_starts is not equal to reserved_history_ends' | -| 5079 | The specified dynamic partition reserved_history_starts is larger than reserved_history_ends | +| 5076 | The specified dynamic partition reserved_history_periods is null or empty | +| 5077 | The specified dynamic partition reserved_history_periods is invalid | +| 5078 | The length of specified dynamic partition reserved_history_periods must have pairs of date value | +| 5079 | The specified dynamic partition reserved_history_periods' first date is larger than the second one | diff --git a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md index aa8532f9e905b3..47bf39e05bc64d 100644 --- a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -287,8 +287,7 @@ distribution_info * `dynamic_partition.buckets`: Used to specify the number of partition buckets that are automatically created. * `dynamic_partition.create_history_partition`: Whether to create a history partition. * `dynamic_partition.history_partition_num`: Specify the number of historical partitions to be created. - * `dynamic_partition.reserved_history_starts`: Used to specify the starting dates of reserved history periods. - * `dynamic_partition.reserved_history_ends`: Used to specify the ending dates of reserved history periods. + * `dynamic_partition.reserved_history_periods`: Used to specify the range of reserved history periods. ### Example diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 2912bf11285d81..ddf25d81c410ed 100644 --- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -311,8 +311,7 @@ Syntax: dynamic_partition.buckets: specifies the number of partition buckets that are automatically created dynamic_partition.create_history_partition: specifies whether create history partitions, default value is false dynamic_partition.history_partition_num: used to specify the number of history partitions when enable create_history_partition - dynamic_partition.reserved_history_starts: Used to specify the starting dates of reserved history periods - dynamic_partition.reserved_history_ends: Used to specify the ending dates of reserved history periods + dynamic_partition.reserved_history_periods: Used to specify the range of reserved history periods ``` 5) You can create multiple Rollups in bulk when building a table grammar: diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md index 84acb87dfdd029..7ac0326834c89d 100644 --- a/docs/zh-CN/administrator-guide/dynamic-partition.md +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -149,17 +149,13 @@ under the License. p20210523：["2021-05-23", "2021-05-24") storage_medium=SSD storage_cooldown_time=2021-05-25 00:00:00 ``` -* `dynamic_partition.reserved_history_starts` +* `dynamic_partition.reserved_history_periods` - 需要保留的历史分区的开始时间。需要以 `yyyy-MM-dd,yyyy-MM-dd,...` 格式进行设置。如果不设置，默认为 `"9999-12-31"`。 - -* `dynamic_partition.reserved_history_ends` - - 需要保留的历史分区的结束时间。需要以 `yyyy-MM-dd,yyyy-MM-dd,...` 格式进行设置。如果不设置，默认为 `"9999-12-31"`。 + 需要保留的历史分区的时间范围。需要以 `[yyyy-MM-dd,yyyy-MM-dd],[...,...]` 格式进行设置。如果不设置，默认为 `"[9999-12-31,9999-12-31]"`。 我们举例说明。假设今天是 2021-09-06，按天分类，动态分区的属性设置为： - ```end=3, start=-3, reserved_history_starts="2020-06-01,2020-10-31", reserved_history_ends="2020-06-20,2020-11-15"```。 + ```end=3, start=-3, reserved_history_periods="[2020-06-01,2020-06-20],[2020-10-31,2020-11-15]"```。 则系统会自动保留： @@ -168,7 +164,7 @@ under the License. ["2020-10-31","2020-11-15"] ``` - 这两个时间段的分区。其中，`reserved_history_starts` 和 `reserved_history_ends` 是一对设置项，两者需要同时被设置，且两者的长度需要一致。另外，对应位置的 `reserved_history_starts` 不能大于 `reserved_history_ends`。 + 这两个时间段的分区。其中，`reserved_history_periods` 的每一个 `[...,...]` 是一对设置项，两者需要同时被设置，且第一个时间不能大于第二个时间``。 #### 创建历史分区规则 @@ -376,16 +372,16 @@ p20200521: ["2020-05-21", "2020-05-22") ``` mysql> SHOW DYNAMIC PARTITION TABLES; -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+--------------------------------------------------------+--------------------------------------------------------+ -| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | ReservedHistoryStarts | ReservedHistoryEnds | -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+--------------------------------------------------------+--------------------------------------------------------+ -| d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 2020-06-01,2020-10-25,2021-06-01,2021-10-25 | 2020-06-20,2020-11-15,2021-06-20,2021-11-15 | -| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 2020-01-01 | 2021-06-01 | -| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | -| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | -| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | -| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | 9999-12-31 | 9999-12-31 | -+-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+--------------------------------------------------------+--------------------------------------------------------+ ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ +| TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | ReservedHistoryPeriods | ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ +| d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [2021-12-01,2021-12-31] | +| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | ++-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ 7 rows in set (0.02 sec) ``` diff --git a/docs/zh-CN/administrator-guide/operation/doris-error-code.md b/docs/zh-CN/administrator-guide/operation/doris-error-code.md index f050d7f20da100..122c0d1b590da2 100644 --- a/docs/zh-CN/administrator-guide/operation/doris-error-code.md +++ b/docs/zh-CN/administrator-guide/operation/doris-error-code.md @@ -172,8 +172,8 @@ under the License. | 5072 | 动态分区副本值不是有效的数字 | | 5073 | 原始创建表stmt为空 | | 5074 | 创建历史动态分区参数：create_history_partition无效，期望的是：true或者false | -| 5076 | 指定的保留历史分区的起始时间为空 | -| 5077 | 指定的保留历史分区的结束时间为空 | -| 5078 | 指定的保留历史分区的起始时间和结束时间长度不一致 | -| 5079 | 指定的保留历史分区对应位置的的起始时间比结束时间大 | +| 5076 | 指定的保留历史分区时间段为空 | +| 5077 | 指定的保留历史分区时间段无效 | +| 5078 | 指定的保留历史分区时间段必须是成对的时间 | +| 5079 | 指定的保留历史分区时间段对应位置的第一个时间比第二个时间大（起始时间大于结束时间） | diff --git a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md index df5179c591d892..e61af64443fa58 100644 --- a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -287,8 +287,7 @@ distribution_info * `dynamic_partition.buckets`: 用于指定自动创建的分区分桶数量。 * `dynamic_partition.create_history_partition`: 是否创建历史分区。 * `dynamic_partition.history_partition_num`: 指定创建历史分区的数量。 - * `dynamic_partition.reserved_history_starts`: 用于指定保留的历史分区的起始时间。 - * `dynamic_partition.reserved_history_ends`: 用于指定保留的历史分区的结束时间。 + * `dynamic_partition.reserved_history_periods`: 用于指定保留的历史分区的时间段。 ### Example diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index bfacf02945dec5..e287fd42c84cd8 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -334,8 +334,7 @@ under the License. dynamic_partition.buckets: 用于指定自动创建的分区分桶数量 dynamic_partition.create_history_partition: 用于创建历史分区功能是否开启。默认为 false。 dynamic_partition.history_partition_num: 当开启创建历史分区功能时，用于指定创建历史分区数量。 - dynamic_partition.reserved_history_starts: 用于指定保留的历史分区的起始时间。 - dynamic_partition.reserved_history_ends: 用于指定保留的历史分区的结束时间。 + dynamic_partition.reserved_history_periods: 用于指定保留的历史分区的时间段。 5) 建表时可以批量创建多个 Rollup 语法： diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java index 6c7cc4580aeba9..a80e8e5de863b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java @@ -46,8 +46,7 @@ public class ShowDynamicPartitionStmt extends ShowStmt { .addColumn(new Column("State", ScalarType.createVarchar(20))) .addColumn(new Column("LastCreatePartitionMsg", ScalarType.createVarchar(20))) .addColumn(new Column("LastDropPartitionMsg", ScalarType.createVarchar(20))) - .addColumn(new Column("ReservedHistoryStarts", ScalarType.createVarchar(20))) - .addColumn(new Column("ReservedHistoryEnds", ScalarType.createVarchar(20))) + .addColumn(new Column("ReservedHistoryPeriods", ScalarType.createVarchar(20))) .build(); ShowDynamicPartitionStmt(String db) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 3ee91627e9fec5..95747c2b29c5e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -24,6 +24,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.TimeUtils; +import java.util.Arrays; import java.util.Map; import java.util.TimeZone; @@ -43,15 +44,13 @@ public class DynamicPartitionProperty { public static final String CREATE_HISTORY_PARTITION = "dynamic_partition.create_history_partition"; public static final String HISTORY_PARTITION_NUM = "dynamic_partition.history_partition_num"; public static final String HOT_PARTITION_NUM = "dynamic_partition.hot_partition_num"; - public static final String RESERVED_HISTORY_STARTS = "dynamic_partition.reserved_history_starts"; - public static final String RESERVED_HISTORY_ENDS = "dynamic_partition.reserved_history_ends"; + public static final String RESERVED_HISTORY_PERIODS = "dynamic_partition.reserved_history_periods"; public static final int MIN_START_OFFSET = Integer.MIN_VALUE; public static final int MAX_END_OFFSET = Integer.MAX_VALUE; public static final int NOT_SET_REPLICATION_NUM = -1; public static final int NOT_SET_HISTORY_PARTITION_NUM = -1; - public static final String NOT_SET_RESERVED_HISTORY_STARTS = "9999-12-31"; - public static final String NOT_SET_RESERVED_HISTORY_ENDS = "9999-12-31"; + public static final String NOT_SET_RESERVED_HISTORY_PERIODS = "[9999-12-31,9999-12-31]"; private boolean exist; @@ -71,8 +70,7 @@ public class DynamicPartitionProperty { // This property are used to describe the number of partitions that need to be reserved on the high-speed storage. // If not set, default is 0 private int hotPartitionNum; - private String reservedHistoryStarts; - private String reservedHistoryEnds; + private String reservedHistoryPeriods; public DynamicPartitionProperty(Map properties) { if (properties != null && !properties.isEmpty()) { @@ -89,8 +87,7 @@ public DynamicPartitionProperty(Map properties) { this.createHistoryPartition = Boolean.parseBoolean(properties.get(CREATE_HISTORY_PARTITION)); this.historyPartitionNum = Integer.parseInt(properties.getOrDefault(HISTORY_PARTITION_NUM, String.valueOf(NOT_SET_HISTORY_PARTITION_NUM))); this.hotPartitionNum = Integer.parseInt(properties.getOrDefault(HOT_PARTITION_NUM, "0")); - this.reservedHistoryStarts = properties.getOrDefault(RESERVED_HISTORY_STARTS, NOT_SET_RESERVED_HISTORY_STARTS).replace(" ", ""); - this.reservedHistoryEnds = properties.getOrDefault(RESERVED_HISTORY_ENDS, NOT_SET_RESERVED_HISTORY_ENDS).replace(" ", ""); + this.reservedHistoryPeriods = properties.getOrDefault(RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS).replace(" ", ""); createStartOfs(properties); } else { this.exist = false; @@ -188,20 +185,18 @@ public ReplicaAllocation getReplicaAllocation() { return replicaAlloc; } - public String getReservedHistoryStarts() { - return reservedHistoryStarts; + public String getReservedHistoryPeriods() { + return reservedHistoryPeriods; } - public void setReservedHistoryStarts(String reservedHistoryStarts) { - this.reservedHistoryStarts = reservedHistoryStarts; - } - - public String getReservedHistoryEnds() { - return reservedHistoryEnds; - } - - public void setReservedHistoryEnds(String reservedHistoryEnds) { - this.reservedHistoryEnds = reservedHistoryEnds; + public String getSortedReservedHistoryPeriods(String reservedHistoryPeriods) { + String[] periods = reservedHistoryPeriods.replace(" ", "").replaceFirst("\\[","").substring(0, reservedHistoryPeriods.length() - 2).split("],\\["); + Arrays.sort(periods); + StringBuilder stringBuilder = new StringBuilder("["); + for (int i = 0; i < periods.length; i++) { + stringBuilder.append(periods[i]).append("],["); + } + return stringBuilder.delete(stringBuilder.length()-2,stringBuilder.length()).toString(); } /** @@ -220,8 +215,7 @@ public String getProperties(ReplicaAllocation tableReplicaAlloc) { ",\n\"" + CREATE_HISTORY_PARTITION + "\" = \"" + createHistoryPartition + "\"" + ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\"" + ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\"" + - ",\n\"" + RESERVED_HISTORY_STARTS + "\" = \"" + reservedHistoryStarts + "\"" + - ",\n\"" + RESERVED_HISTORY_ENDS + "\" = \"" + reservedHistoryEnds + "\""; + ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" + reservedHistoryPeriods + "\""; if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) { res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\""; } else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 8d7acd40b0f287..bec9d14e1cf37e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -277,8 +277,15 @@ private ArrayList getDropPartitionClause(Database db, OlapT db.getFullName(), olapTable.getName()); return dropPartitionClauses; } - String[] reservedHistoryStarts = dynamicPartitionProperty.getReservedHistoryStarts().split(","); - String[] reservedHistoryEnds = dynamicPartitionProperty.getReservedHistoryEnds().split(","); + StringBuilder reservedHistoryStartsBuilder = new StringBuilder(); + StringBuilder reservedHistoryEndsBuilder = new StringBuilder(); + String[] reservedHistoryPeriods = dynamicPartitionProperty.getReservedHistoryPeriods().replace(" ", "").replace("[", "").replace("]", "").split(","); + for (int i = 0; i < reservedHistoryPeriods.length; i+=2) { + reservedHistoryStartsBuilder.append(reservedHistoryPeriods[i] + ","); + reservedHistoryEndsBuilder.append(reservedHistoryPeriods[i+1] + ","); + } + String[] reservedHistoryStarts = reservedHistoryStartsBuilder.deleteCharAt(reservedHistoryStartsBuilder.length() - 1).toString().split(","); + String[] reservedHistoryEnds = reservedHistoryEndsBuilder.deleteCharAt(reservedHistoryEndsBuilder.length() - 1).toString().split(","); Range reservedHistoryPartitionKeyRange; for (int i = 0; i < reservedHistoryStarts.length; i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java index 6bdc81d55dad91..08e61ed7727775 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java @@ -244,14 +244,14 @@ public enum ErrorCode { "Invalid dynamic partition create_history_partition: %s. Expected true or false"), ERROR_DYNAMIC_PARTITION_HISTORY_PARTITION_NUM_ZERO(5075, new byte[] {'4', '2', '0', '0', '0'}, "Dynamic history partition num must greater than 0"), - ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_EMPTY(5076, new byte[] {'4', '2', '0', '0', '0'}, - "Dynamic reserved history starts is empty."), - ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_ENDS_EMPTY(5077, new byte[] {'4', '2', '0', '0', '0'}, - "Dynamic reserved history ends is empty."), - ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL(5078, new byte[] {'4', '2', '0', '0', '0'}, - "RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: %s, RESERVED_HISTORY_STARTS length: %s."), - ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_LARGER_THAN_ENDS(5079, new byte[] {'4', '2', '0', '0', '0'}, - "RESERVED_HISTORY_STARTS %s is larger than RESERVED_HISTORY_ENDS %s."), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_EMPTY(5076, new byte[] {'4', '2', '0', '0', '0'}, + "Dynamic reserved history periods is empty."), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID(5077, new byte[] {'4', '2', '0', '0', '0'}, + "Invalid \" %s \" value %s. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_ENDS_LENGTH_NOT_EQUAL(5078, new byte[] {'4', '2', '0', '0', '0'}, + "RESERVED_HISTORY_PERIODS must have pairs of date value. The input %s is not valid."), + ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_LARGER_THAN_ENDS(5079, new byte[] {'4', '2', '0', '0', '0'}, + "The first date is larger than the second date, [%s,%s] is invalid."), ERROR_LDAP_CONFIGURATION_ERR(5080, new byte[] {'4', '2', '0', '0', '0'}, "LDAP configuration is incorrect or LDAP admin password is not set."), ERROR_LDAP_USER_NOT_UNIQUE_ERR(5081, new byte[] {'4', '2', '0', '0', '0'}, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 32ca6ea8841bad..4a0e6ed8214606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -58,6 +58,8 @@ import java.util.TimeZone; import java.util.Date; import java.util.Calendar; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class DynamicPartitionUtil { private static final Logger LOG = LogManager.getLogger(DynamicPartitionUtil.class); @@ -233,58 +235,52 @@ private static void checkHotPartitionNum(String val) throws DdlException { } } - private static void checkReservedHistoryStarts(String reservedHistoryStarts) throws DdlException{ - String[] starts = reservedHistoryStarts.split(","); - if (starts.length == 0) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_EMPTY); + private static void checkReservedHistoryPeriodValidate(String reservedHistoryPeriods) throws DdlException { + if (Strings.isNullOrEmpty(reservedHistoryPeriods) ) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_EMPTY); } - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - Date date = null; - try { - for (int i = 0; i < starts.length; i++) { - date = sdf.parse(starts[i]); - if (!starts[i].equals(sdf.format(date))) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be correct DATE value (yyyy-MM-dd)"); - } + // it has 5 kinds of situation + // 1. "dynamic_partition.reserved_history_periods" = "[2021-07-01,]," invalid one + // 2. "dynamic_partition.reserved_history_periods" = "2021-07-01", invalid. It must be surrounded by [] + // 1. "dynamic_partition.reserved_history_periods" = "[2021-07-01,]" invalid one, needs pairs of values + // 2. "dynamic_partition.reserved_history_periods" = "[,2021-08-01]" invalid one, needs pairs of values + // 3. "dynamic_partition.reserved_history_periods" = "[2021-07-01,2020-08-01,]" invalid format + String reservedHistoryPeriodsWithoutSpace = reservedHistoryPeriods.replace(" ", ""); + if (!reservedHistoryPeriodsWithoutSpace.startsWith("[") || !reservedHistoryPeriodsWithoutSpace.endsWith("]")) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriodsWithoutSpace); + } + + String[] ranges = reservedHistoryPeriodsWithoutSpace.replaceFirst("\\[","").substring(0, reservedHistoryPeriodsWithoutSpace.length() - 2).split("],\\["); + Pattern pattern = Pattern.compile("[0-9-]{10}[,]{1}[0-9-]{10}"); + for (String reservedHistoryPeriod : ranges) { + LOG.info("whz_test_999 reservedHistoryPeriod: {}", reservedHistoryPeriod); + Matcher matcher = pattern.matcher(reservedHistoryPeriod); + if (!matcher.matches()) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriodsWithoutSpace); } - } catch (ParseException e) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + " value. It must be like \"yyyy-MM-dd\""); } - } - private static void checkReservedHistoryEnds(String reservedHistoryEnds) throws DdlException{ - String[] ends = reservedHistoryEnds.split(","); - if (ends.length == 0) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_ENDS_EMPTY); + String[] periods = reservedHistoryPeriods.replace(" ", "").replace("[", "").replace("]", "").split(","); + if (periods.length % 2 == 1) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_ENDS_LENGTH_NOT_EQUAL, reservedHistoryPeriodsWithoutSpace); } + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); Date date = null; try { - for (int i = 0; i < ends.length; i++) { - date = sdf.parse(ends[i]); - if (!ends[i].equals(sdf.format(date))) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be correct DATE value (yyyy-MM-dd)"); + for (int i = 0; i < periods.length; i++) { + date = sdf.parse(periods[i]); + if (!periods[i].equals(sdf.format(date))) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be correct DATE value \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); } } - } catch (ParseException e) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + " value. It must be like \"yyyy-MM-dd\""); + } catch (ParseException e) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); } - } - private static void checkReservedHistoryPeriodValidate(String reservedHistoryStarts, String reservedHistoryEnds) throws DdlException { - String[] starts = reservedHistoryStarts.split(","); - String[] ends = reservedHistoryEnds.split(","); - if (starts.length != ends.length) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, starts.length, ends.length); - } else if (("9999-12-31".equals(starts[0]) && !"9999-12-31".equals(ends[0]))) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, 0, ends.length); - } else if (("9999-12-31".equals(ends[0]) && !"9999-12-31".equals(starts[0]))) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_STARTS_ENDS_LENGTH_NOT_EQUAL, starts.length, 0); - } - for (int i = 0; i < starts.length; i++) { - if (!"9999-12-31".equals(starts[i]) && starts[i].compareTo(ends[i]) > 0) { - throw new DdlException("Invalid properties: " + DynamicPartitionProperty.RESERVED_HISTORY_STARTS + ": \"" + starts[i] + - "\" is larger than " + DynamicPartitionProperty.RESERVED_HISTORY_ENDS + ": \"" + ends[i] + "\""); + for (int i = 0; i < periods.length; i+=2) { + if (periods[i].compareTo(periods[i+1]) > 0) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_LARGER_THAN_ENDS, periods[i], periods[i+1]); } } } @@ -320,8 +316,7 @@ public static boolean checkInputDynamicPartitionProperties(Map p String enable = properties.get(DynamicPartitionProperty.ENABLE); String createHistoryPartition = properties.get(DynamicPartitionProperty.CREATE_HISTORY_PARTITION); String historyPartitionNum = properties.get(DynamicPartitionProperty.HISTORY_PARTITION_NUM); - String reservedHistoryStarts = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_STARTS); - String reservedHistoryEnds = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_ENDS); + String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS); if (!(Strings.isNullOrEmpty(enable) && Strings.isNullOrEmpty(timeUnit) && @@ -332,8 +327,7 @@ public static boolean checkInputDynamicPartitionProperties(Map p Strings.isNullOrEmpty(buckets) && Strings.isNullOrEmpty(createHistoryPartition) && Strings.isNullOrEmpty(historyPartitionNum) && - Strings.isNullOrEmpty(reservedHistoryStarts) && - Strings.isNullOrEmpty(reservedHistoryEnds))) { + Strings.isNullOrEmpty(reservedHistoryPeriods))) { if (Strings.isNullOrEmpty(enable)) { properties.put(DynamicPartitionProperty.ENABLE, "true"); } @@ -362,13 +356,9 @@ public static boolean checkInputDynamicPartitionProperties(Map p properties.put(DynamicPartitionProperty.HISTORY_PARTITION_NUM, String.valueOf(DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM)); } - if (Strings.isNullOrEmpty(reservedHistoryStarts)) { - properties.put(DynamicPartitionProperty.RESERVED_HISTORY_STARTS, - String.valueOf(DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_STARTS)); - } - if (Strings.isNullOrEmpty(reservedHistoryEnds)) { - properties.put(DynamicPartitionProperty.RESERVED_HISTORY_ENDS, - String.valueOf(DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_ENDS)); + if (Strings.isNullOrEmpty(reservedHistoryPeriods)) { + properties.put(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, + String.valueOf(DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_PERIODS)); } } return true; @@ -522,17 +512,11 @@ public static Map analyzeDynamicPartition(Map pr properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM); analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val); } - if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_STARTS) && properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_ENDS)) { - String starts = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_STARTS).replace(" ", ""); - String ends = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_ENDS).replace(" ", ""); - - checkReservedHistoryStarts(starts); - checkReservedHistoryEnds(ends); - checkReservedHistoryPeriodValidate(starts, ends); - properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_STARTS); - properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_ENDS); - analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_STARTS, starts); - analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_ENDS, ends); + if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS)) { + String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS).replace(" ",""); + checkReservedHistoryPeriodValidate(reservedHistoryPeriods); + properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS); + analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); } return analyzedProperties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index a9bfdccd1a03e3..8eab3ee96908b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1817,6 +1817,7 @@ private void handleShowDynamicPartition() { if (replicaAlloc.isNotSet()) { replicaAlloc = olapTable.getDefaultReplicaAllocation(); } + String unsortedReservedHistoryPeriods = dynamicPartitionProperty.getReservedHistoryPeriods(); rows.add(Lists.newArrayList( tableName, String.valueOf(dynamicPartitionProperty.getEnable()), @@ -1833,8 +1834,7 @@ private void handleShowDynamicPartition() { dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DYNAMIC_PARTITION_STATE), dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.CREATE_PARTITION_MSG), dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DROP_PARTITION_MSG), - dynamicPartitionProperty.getReservedHistoryStarts(), - dynamicPartitionProperty.getReservedHistoryEnds())); + dynamicPartitionProperty.getSortedReservedHistoryPeriods(unsortedReservedHistoryPeriods))); } finally { olapTable.readUnlock(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 70bd7a720c51b6..687b72764b0464 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1188,8 +1188,8 @@ public void testRuntimeInfo() throws Exception { } @Test - public void testMissReservedHistoryStartsAndEnds() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_miss_reserved_history_starts_and_ends` (\n" + + public void testMissReservedHistoryPeriods() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_miss_reserved_history_periods` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + @@ -1215,14 +1215,13 @@ public void testMissReservedHistoryStartsAndEnds() throws Exception { "\"dynamic_partition.prefix\" = \"p\"\n" + ");"; createTable(createOlapTblStmt); - OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_miss_reserved_history_starts_and_ends"); - Assert.assertEquals("9999-12-31", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryStarts()); - Assert.assertEquals("9999-12-31", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryEnds()); + OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_miss_reserved_history_periods"); + Assert.assertEquals("[9999-12-31,9999-12-31]", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); } @Test - public void testNormalReservedHisrotyStartsAndEnds() throws Exception { - String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal_reserved_history_starts_and_ends` (\n" + + public void testNormalReservedHisrotyPeriods() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal_reserved_history_periods` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + @@ -1254,19 +1253,17 @@ public void testNormalReservedHisrotyStartsAndEnds() throws Exception { "\"dynamic_partition.buckets\" = \"3\",\n" + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"2020-06-01,2020-10-25,2021-06-01\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"2020-06-20,2020-11-15,2021-06-20\"\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[2020-06-01,2020-06-20],[2020-10-25,2020-11-15],[2021-06-01,2021-06-20]\"\n" + ");"; createTable(createOlapTblStmt); - OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_normal_reserved_history_starts_and_ends"); - Assert.assertEquals("2020-06-01,2020-10-25,2021-06-01", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryStarts()); - Assert.assertEquals("2020-06-20,2020-11-15,2021-06-20", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryEnds()); + OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_normal_reserved_history_periods"); + Assert.assertEquals("[2020-06-01,2020-06-20],[2020-10-25,2020-11-15],[2021-06-01,2021-06-20]", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); Assert.assertEquals(table.getAllPartitions().size(), 9); } @Test - public void testInvalidReservedHistoryStarts() throws Exception { - String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_starts1` (\n" + + public void testInvalidReservedHistoryPeriods() throws Exception { + String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_periods1` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + @@ -1290,14 +1287,13 @@ public void testInvalidReservedHistoryStarts() throws Exception { "\"dynamic_partition.buckets\" = \"3\",\n" + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"20210101\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"2021-10-10\"\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[20210101,2021-10-10]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_starts value. It must be like \"yyyy-MM-dd\"", + "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [20210101,2021-10-10]. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", () -> createTable(createOlapTblStmt1)); - String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_starts2` (\n" + + String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_periods2` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + @@ -1321,17 +1317,16 @@ public void testInvalidReservedHistoryStarts() throws Exception { "\"dynamic_partition.buckets\" = \"3\",\n" + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"0000-00-00\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"2021-10-10\"\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[0000-00-00,2021-10-10]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_starts value. It must be correct DATE value (yyyy-MM-dd)", + "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_periods value. It must be correct DATE value \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", () -> createTable(createOlapTblStmt2)); } @Test - public void testInvalidReservedHistoryEnds() throws Exception { - String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_ends` (\n" + + public void testReservedHistoryPeriodsValidate() throws Exception { + String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_reserved_history_periods_validate1` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + @@ -1355,14 +1350,13 @@ public void testInvalidReservedHistoryEnds() throws Exception { "\"dynamic_partition.buckets\" = \"3\",\n" + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"2021-10-12\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"20211010\"\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[2021-01-01,]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_ends value. It must be like \"yyyy-MM-dd\"", + "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [2021-01-01,]. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", () -> createTable(createOlapTblStmt1)); - String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_ends2` (\n" + + String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_reserved_history_periods_validate2` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + @@ -1386,77 +1380,13 @@ public void testInvalidReservedHistoryEnds() throws Exception { "\"dynamic_partition.buckets\" = \"3\",\n" + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"2021-01-01\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"0000-00-00\"\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[,2021-01-01]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_ends value. It must be correct DATE value (yyyy-MM-dd)", - () -> createTable(createOlapTblStmt2)); - } - - @Test - public void testReservedHistoryPeriodValidate() throws Exception { - String createOlapTblStmt1 = "CREATE TABLE test.`dynamic_partition_reserved_history_period_validate1` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.buckets\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"2021-01-01\"\n" + - ");"; - ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: 1, RESERVED_HISTORY_STARTS length: 0.", - () -> createTable(createOlapTblStmt1)); - - String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_reserved_history_period_validate2` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.buckets\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"2021-01-01\"\n" + - ");"; - ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: 0, RESERVED_HISTORY_STARTS length: 1.", + "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [,2021-01-01]. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", () -> createTable(createOlapTblStmt2)); - String createOlapTblStmt3 = "CREATE TABLE test.`dynamic_partition_reserved_history_starts_is_larger_than_ends` (\n" + + String createOlapTblStmt3 = "CREATE TABLE test.`dynamic_partition_reserved_history_period_starttime_is_larger_than_endtime` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + @@ -1480,42 +1410,10 @@ public void testReservedHistoryPeriodValidate() throws Exception { "\"dynamic_partition.buckets\" = \"3\",\n" + "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"2020-01-01,2021-10-01\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"2020-03-01,2021-09-01\"\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[2020-01-01,2020-03-01],[2021-10-01,2021-09-01]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid properties: dynamic_partition.reserved_history_starts: \"2021-10-01\" is larger than dynamic_partition.reserved_history_ends: \"2021-09-01\"", + "errCode = 2, detailMessage = The first date is larger than the second date, [2021-10-01,2021-09-01] is invalid.", () -> createTable(createOlapTblStmt3)); - - String createOlapTblStmt4 = "CREATE TABLE test.`dynamic_partition_reserved_history_starts_and_ends_length` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.buckets\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.reserved_history_starts\" = \"2020-01-01,2021-08-01,2021-11-01\",\n" + - "\"dynamic_partition.reserved_history_ends\" = \"2020-03-01,2021-09-01\"\n" + - ");"; - ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = RESERVED_HISTORY_STARTS and RESERVED_HISTORY_ENDS should have the same length. RESERVED_HISTORY_STARTS length: 3, RESERVED_HISTORY_STARTS length: 2.", - () -> createTable(createOlapTblStmt4)); } } From 5331b4ee69311f0a6253cb420f4484bf61ea2f4a Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Tue, 14 Sep 2021 13:35:06 +0800 Subject: [PATCH 06/16] fix --- .../org/apache/doris/catalog/DynamicPartitionTableTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 687b72764b0464..e5107a719cf760 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1215,7 +1215,7 @@ public void testMissReservedHistoryPeriods() throws Exception { "\"dynamic_partition.prefix\" = \"p\"\n" + ");"; createTable(createOlapTblStmt); - OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_miss_reserved_history_periods"); + OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_miss_reserved_history_periods"); Assert.assertEquals("[9999-12-31,9999-12-31]", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); } @@ -1256,7 +1256,7 @@ public void testNormalReservedHisrotyPeriods() throws Exception { "\"dynamic_partition.reserved_history_periods\" = \"[2020-06-01,2020-06-20],[2020-10-25,2020-11-15],[2021-06-01,2021-06-20]\"\n" + ");"; createTable(createOlapTblStmt); - OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDb("default_cluster:test").getTable("dynamic_partition_normal_reserved_history_periods"); + OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_normal_reserved_history_periods"); Assert.assertEquals("[2020-06-01,2020-06-20],[2020-10-25,2020-11-15],[2021-06-01,2021-06-20]", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); Assert.assertEquals(table.getAllPartitions().size(), 9); } @@ -1386,7 +1386,7 @@ public void testReservedHistoryPeriodsValidate() throws Exception { "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [,2021-01-01]. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", () -> createTable(createOlapTblStmt2)); - String createOlapTblStmt3 = "CREATE TABLE test.`dynamic_partition_reserved_history_period_starttime_is_larger_than_endtime` (\n" + + String createOlapTblStmt3 = "CREATE TABLE test.`dynamic_partition_reserved_history_periods_validate3` (\n" + " `k1` date NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\",\n" + " `k3` smallint NULL COMMENT \"\",\n" + From 2f5e360b1c376859260c24ffb73801fb7eba09b1 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Wed, 15 Sep 2021 11:22:50 +0800 Subject: [PATCH 07/16] fix --- .../java/org/apache/doris/common/util/DynamicPartitionUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 4a0e6ed8214606..96241db5b44c13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -253,7 +253,6 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistoryPer String[] ranges = reservedHistoryPeriodsWithoutSpace.replaceFirst("\\[","").substring(0, reservedHistoryPeriodsWithoutSpace.length() - 2).split("],\\["); Pattern pattern = Pattern.compile("[0-9-]{10}[,]{1}[0-9-]{10}"); for (String reservedHistoryPeriod : ranges) { - LOG.info("whz_test_999 reservedHistoryPeriod: {}", reservedHistoryPeriod); Matcher matcher = pattern.matcher(reservedHistoryPeriod); if (!matcher.matches()) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriodsWithoutSpace); From b6050669b03cfd423bde074251633142a92659e2 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Thu, 16 Sep 2021 12:00:06 +0800 Subject: [PATCH 08/16] change Pattern --- .../catalog/DynamicPartitionProperty.java | 13 +-- .../clone/DynamicPartitionScheduler.java | 1 + .../common/util/DynamicPartitionUtil.java | 83 ++++++++++++------- .../org/apache/doris/qe/ShowExecutor.java | 2 + 4 files changed, 59 insertions(+), 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 95747c2b29c5e3..f837e664a532c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -19,12 +19,13 @@ import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.DynamicPartitionUtil.StartOfDate; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.TimeUtils; -import java.util.Arrays; import java.util.Map; import java.util.TimeZone; @@ -189,14 +190,8 @@ public String getReservedHistoryPeriods() { return reservedHistoryPeriods; } - public String getSortedReservedHistoryPeriods(String reservedHistoryPeriods) { - String[] periods = reservedHistoryPeriods.replace(" ", "").replaceFirst("\\[","").substring(0, reservedHistoryPeriods.length() - 2).split("],\\["); - Arrays.sort(periods); - StringBuilder stringBuilder = new StringBuilder("["); - for (int i = 0; i < periods.length; i++) { - stringBuilder.append(periods[i]).append("],["); - } - return stringBuilder.delete(stringBuilder.length()-2,stringBuilder.length()).toString(); + public String getSortedReservedHistoryPeriods(String reservedHistoryPeriods) throws DdlException { + return DynamicPartitionUtil.sortedListedToString(reservedHistoryPeriods); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index bec9d14e1cf37e..dd17f7a5f0191a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -277,6 +277,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT db.getFullName(), olapTable.getName()); return dropPartitionClauses; } + StringBuilder reservedHistoryStartsBuilder = new StringBuilder(); StringBuilder reservedHistoryEndsBuilder = new StringBuilder(); String[] reservedHistoryPeriods = dynamicPartitionProperty.getReservedHistoryPeriods().replace(" ", "").replace("[", "").replace("]", "").split(","); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 96241db5b44c13..d2d9d666762768 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -18,6 +18,7 @@ package org.apache.doris.common.util; +import com.google.common.collect.Range; import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; @@ -53,7 +54,11 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; + +import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.Date; @@ -235,6 +240,36 @@ private static void checkHotPartitionNum(String val) throws DdlException { } } + public static List convertStringToPeriodsList(String reservedHistoryPeriods) throws DdlException { + List reservedHistoryPeriodsToRangeList = new ArrayList(); + Integer sizeOfPeriods = reservedHistoryPeriods.split("],\\[").length; + Pattern pattern = Pattern.compile("\\[([0-9]{4}-[0-9]{2}-[0-9]{2}),([0-9]{4}-[0-9]{2}-[0-9]{2})\\]"); + Matcher matcher = pattern.matcher(reservedHistoryPeriods); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + while (matcher.find()) { + String lowerBorderOfReservedHistory = matcher.group(1); + String upperBorderOfReservedHistory = matcher.group(2); + if (lowerBorderOfReservedHistory.compareTo(upperBorderOfReservedHistory) > 0) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_LARGER_THAN_ENDS, lowerBorderOfReservedHistory, upperBorderOfReservedHistory); + } else { + reservedHistoryPeriodsToRangeList.add(Range.closed(lowerBorderOfReservedHistory, upperBorderOfReservedHistory)); + } + } + return reservedHistoryPeriodsToRangeList; + } + + public static String sortedListedToString(String reservedHistoryPeriods) throws DdlException { + List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods); + reservedHistoryPeriodsToRangeList.sort(new Comparator() { + @Override + public int compare(Range o1, Range o2) { + return o1.lowerEndpoint().compareTo(o2.lowerEndpoint()); + } + }); + String sortedReservedHistoryPeriods = reservedHistoryPeriodsToRangeList.toString().replace("..",","); + return sortedReservedHistoryPeriods; + } + private static void checkReservedHistoryPeriodValidate(String reservedHistoryPeriods) throws DdlException { if (Strings.isNullOrEmpty(reservedHistoryPeriods) ) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_EMPTY); @@ -245,41 +280,27 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistoryPer // 1. "dynamic_partition.reserved_history_periods" = "[2021-07-01,]" invalid one, needs pairs of values // 2. "dynamic_partition.reserved_history_periods" = "[,2021-08-01]" invalid one, needs pairs of values // 3. "dynamic_partition.reserved_history_periods" = "[2021-07-01,2020-08-01,]" invalid format - String reservedHistoryPeriodsWithoutSpace = reservedHistoryPeriods.replace(" ", ""); - if (!reservedHistoryPeriodsWithoutSpace.startsWith("[") || !reservedHistoryPeriodsWithoutSpace.endsWith("]")) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriodsWithoutSpace); - } - - String[] ranges = reservedHistoryPeriodsWithoutSpace.replaceFirst("\\[","").substring(0, reservedHistoryPeriodsWithoutSpace.length() - 2).split("],\\["); - Pattern pattern = Pattern.compile("[0-9-]{10}[,]{1}[0-9-]{10}"); - for (String reservedHistoryPeriod : ranges) { - Matcher matcher = pattern.matcher(reservedHistoryPeriod); - if (!matcher.matches()) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriodsWithoutSpace); - } - } - - String[] periods = reservedHistoryPeriods.replace(" ", "").replace("[", "").replace("]", "").split(","); - if (periods.length % 2 == 1) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_ENDS_LENGTH_NOT_EQUAL, reservedHistoryPeriodsWithoutSpace); + if (!reservedHistoryPeriods.startsWith("[") || !reservedHistoryPeriods.endsWith("]")) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); } + List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods); + Integer sizeOfPeriods = reservedHistoryPeriods.split("],\\[").length; SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - Date date = null; - try { - for (int i = 0; i < periods.length; i++) { - date = sdf.parse(periods[i]); - if (!periods[i].equals(sdf.format(date))) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be correct DATE value \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); - } - } - } catch (ParseException e) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); - } - for (int i = 0; i < periods.length; i+=2) { - if (periods[i].compareTo(periods[i+1]) > 0) { - ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_LARGER_THAN_ENDS, periods[i], periods[i+1]); + if (reservedHistoryPeriodsToRangeList.size() != sizeOfPeriods) { + ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); + } else { + try { + for (Range range : reservedHistoryPeriodsToRangeList) { + String formattedLowerBound = sdf.format(sdf.parse(range.lowerEndpoint().toString())); + String formattedUpperBound = sdf.format(sdf.parse(range.upperEndpoint().toString())); + if (!range.lowerEndpoint().toString().equals(formattedLowerBound) || !range.upperEndpoint().toString().equals(formattedUpperBound)) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be correct DATE value \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); + } + } + } catch (ParseException e) { + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8eab3ee96908b8..b8975db13c4941 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1835,6 +1835,8 @@ private void handleShowDynamicPartition() { dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.CREATE_PARTITION_MSG), dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DROP_PARTITION_MSG), dynamicPartitionProperty.getSortedReservedHistoryPeriods(unsortedReservedHistoryPeriods))); + } catch (DdlException e) { + e.printStackTrace(); } finally { olapTable.readUnlock(); } From 91c0e3bb2331d6a6016903257e25ac08d58ca8ca Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Thu, 16 Sep 2021 19:02:33 +0800 Subject: [PATCH 09/16] refactor --- .../clone/DynamicPartitionScheduler.java | 60 +++++++++++-------- .../common/util/DynamicPartitionUtil.java | 1 - 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index dd17f7a5f0191a..06770b45be2457 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -37,6 +37,7 @@ import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DynamicPartitionUtil; @@ -244,11 +245,29 @@ private void setStorageMediumProperty(HashMap partitionPropertie partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, cooldownTime); } + private Range getClosedRange(Database db, OlapTable olapTable, Column partitionColumn, String partitionFormat, + String lowerBorderOfReservedHistory, String upperBorderOfReservedHistory) { + Range reservedHistoryPartitionKeyRange = null; + PartitionValue lowerBorderPartitionValue = new PartitionValue(lowerBorderOfReservedHistory); + PartitionValue upperBorderPartitionValue = new PartitionValue(upperBorderOfReservedHistory); + try { + PartitionKey lowerBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(lowerBorderPartitionValue), Collections.singletonList(partitionColumn)); + PartitionKey upperBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(upperBorderPartitionValue), Collections.singletonList(partitionColumn)); + reservedHistoryPartitionKeyRange = Range.closed(lowerBorderBound, upperBorderBound); + } catch (AnalysisException e) { + // AnalysisException: keys.size is always equal to column.size, cannot reach this exception + // IllegalArgumentException: lb is greater than ub + LOG.warn("Error in gen reservePartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), + db.getFullName(), olapTable.getName()); + } + return reservedHistoryPartitionKeyRange; + } + /** * 1. get the range of [start, 0) as a reserved range. * 2. get DropPartitionClause of partitions which range are before this reserved range. */ - private ArrayList getDropPartitionClause(Database db, OlapTable olapTable, Column partitionColumn, String partitionFormat) { + private ArrayList getDropPartitionClause(Database db, OlapTable olapTable, Column partitionColumn, String partitionFormat) throws DdlException { ArrayList dropPartitionClauses = new ArrayList<>(); DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty(); if (dynamicPartitionProperty.getStart() == DynamicPartitionProperty.MIN_START_OFFSET) { @@ -278,34 +297,21 @@ private ArrayList getDropPartitionClause(Database db, OlapT return dropPartitionClauses; } - StringBuilder reservedHistoryStartsBuilder = new StringBuilder(); - StringBuilder reservedHistoryEndsBuilder = new StringBuilder(); - String[] reservedHistoryPeriods = dynamicPartitionProperty.getReservedHistoryPeriods().replace(" ", "").replace("[", "").replace("]", "").split(","); - for (int i = 0; i < reservedHistoryPeriods.length; i+=2) { - reservedHistoryStartsBuilder.append(reservedHistoryPeriods[i] + ","); - reservedHistoryEndsBuilder.append(reservedHistoryPeriods[i+1] + ","); - } - String[] reservedHistoryStarts = reservedHistoryStartsBuilder.deleteCharAt(reservedHistoryStartsBuilder.length() - 1).toString().split(","); - String[] reservedHistoryEnds = reservedHistoryEndsBuilder.deleteCharAt(reservedHistoryEndsBuilder.length() - 1).toString().split(","); - - Range reservedHistoryPartitionKeyRange; - for (int i = 0; i < reservedHistoryStarts.length; i++) { - String lowBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, reservedHistoryStarts[i], partitionFormat); - String upBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, reservedHistoryEnds[i], partitionFormat); - PartitionValue lowBorderPartitionValue = new PartitionValue(lowBorderOfReservedHistory); - PartitionValue upBorderPartitionValue = new PartitionValue(upBorderOfReservedHistory); + String reservedHistoryPeriods = dynamicPartitionProperty.getReservedHistoryPeriods(); + List ranges = DynamicPartitionUtil.convertStringToPeriodsList(reservedHistoryPeriods); + + for (Range range : ranges) { try { - PartitionKey lowerBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(lowBorderPartitionValue), Collections.singletonList(partitionColumn)); - PartitionKey upperBorderBound = PartitionKey.createPartitionKey(Collections.singletonList(upBorderPartitionValue), Collections.singletonList(partitionColumn)); - reservedHistoryPartitionKeyRange = Range.closed(lowerBorderBound, upperBorderBound); + String lowerBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, range.lowerEndpoint().toString(), partitionFormat); + String upperBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, range.upperEndpoint().toString(), partitionFormat); + Range reservedHistoryPartitionKeyRange = getClosedRange(db, olapTable, partitionColumn, partitionFormat, lowerBorderOfReservedHistory,upperBorderOfReservedHistory); reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange); - - } catch (AnalysisException | IllegalArgumentException e) { - LOG.warn("Error in gen reserveHistoryPartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(), - db.getFullName(), olapTable.getName()); + } + catch (IllegalArgumentException e) { return dropPartitionClauses; } } + RangePartitionInfo info = (RangePartitionInfo) (olapTable.getPartitionInfo()); List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); @@ -346,8 +352,8 @@ private void executeDynamicPartition(Collection> dynamicPartiti } ArrayList addPartitionClauses = new ArrayList<>(); - ArrayList dropPartitionClauses; - String tableName; + ArrayList dropPartitionClauses = null; + String tableName = null; boolean skipAddPartition = false; OlapTable olapTable; olapTable = (OlapTable) db.getTableNullable(tableId); @@ -393,6 +399,8 @@ private void executeDynamicPartition(Collection> dynamicPartiti } dropPartitionClauses = getDropPartitionClause(db, olapTable, partitionColumn, partitionFormat); tableName = olapTable.getName(); + } catch (DdlException e) { + e.printStackTrace(); } finally { olapTable.readUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index d2d9d666762768..6cbbae8130f231 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -54,7 +54,6 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; - import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; From 6f5639a8027d24c68959a43355d7ff74a9c9e13a Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Fri, 17 Sep 2021 14:30:59 +0800 Subject: [PATCH 10/16] fix --- .../apache/doris/common/util/DynamicPartitionUtil.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 6cbbae8130f231..038d87a3b196b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -18,7 +18,6 @@ package org.apache.doris.common.util; -import com.google.common.collect.Range; import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; @@ -42,6 +41,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.Range; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -64,6 +64,7 @@ import java.util.Calendar; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class DynamicPartitionUtil { private static final Logger LOG = LogManager.getLogger(DynamicPartitionUtil.class); @@ -265,8 +266,10 @@ public int compare(Range o1, Range o2) { return o1.lowerEndpoint().compareTo(o2.lowerEndpoint()); } }); - String sortedReservedHistoryPeriods = reservedHistoryPeriodsToRangeList.toString().replace("..",","); - return sortedReservedHistoryPeriods; + List sortedReservedHistoryPeriods = reservedHistoryPeriodsToRangeList.stream(). + map(e -> "[" + e.lowerEndpoint() + "," + e.upperEndpoint() + "]").collect(Collectors.toList()); + + return String.join(",", sortedReservedHistoryPeriods); } private static void checkReservedHistoryPeriodValidate(String reservedHistoryPeriods) throws DdlException { From 7fe9cd9862962ed42411b3e5815d7abfe0e4d13c Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Sat, 18 Sep 2021 11:27:36 +0800 Subject: [PATCH 11/16] fix2 --- .../apache/doris/common/util/DynamicPartitionUtil.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 038d87a3b196b4..5c3edc05313796 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -646,14 +646,15 @@ public static String getHistoryPartitionRangeString(DynamicPartitionProperty dyn ZoneId zoneId = dynamicPartitionProperty.getTimeZone().toZoneId(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); Date date = null; + Timestamp timestamp = null; + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.s").withZone(zoneId); try { date = simpleDateFormat.parse(time); } catch (ParseException e) { - e.printStackTrace(); + LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage()); + return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); } - Timestamp timestamp = new Timestamp(date.getTime()); - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.s").withZone(zoneId); - + timestamp = new Timestamp(date.getTime()); return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); } From 346cc33e3de236211ad0cc8c1d90c676b94fb5d1 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Wed, 22 Sep 2021 17:18:18 +0800 Subject: [PATCH 12/16] change default value of RESERVED_HISTORY_PERIODS --- .../administrator-guide/dynamic-partition.md | 12 +++++------ .../administrator-guide/dynamic-partition.md | 12 +++++------ .../catalog/DynamicPartitionProperty.java | 2 +- .../clone/DynamicPartitionScheduler.java | 20 +++++++++---------- .../common/util/DynamicPartitionUtil.java | 11 +++++++++- .../catalog/DynamicPartitionTableTest.java | 2 +- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md index 58d91dfe40e35e..f8bd5cf85d7c60 100644 --- a/docs/en/administrator-guide/dynamic-partition.md +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -153,7 +153,7 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: * `dynamic_partition.reserved_history_periods` - The range of reserved history periods. It should be in the form of `[yyyy-MM-dd,yyyy-MM-dd],[...,...]`. The default value is `"[9999-12-31,9999-12-31]"`, which means it is not set. + The range of reserved history periods. It should be in the form of `[yyyy-MM-dd,yyyy-MM-dd],[...,...]`. The default value is `"\\N"`, which means it is not set. Let us give an example. Suppose today is 2021-09-06，partitioned by day, and the properties of dynamic partition are set to: @@ -378,11 +378,11 @@ mysql> SHOW DYNAMIC PARTITION TABLES; | TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | ReservedHistoryPeriods | +-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ | d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [2021-12-01,2021-12-31] | -| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ 7 rows in set (0.02 sec) ``` diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md index 7ac0326834c89d..bcc4ad74ecc492 100644 --- a/docs/zh-CN/administrator-guide/dynamic-partition.md +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -151,7 +151,7 @@ under the License. * `dynamic_partition.reserved_history_periods` - 需要保留的历史分区的时间范围。需要以 `[yyyy-MM-dd,yyyy-MM-dd],[...,...]` 格式进行设置。如果不设置，默认为 `"[9999-12-31,9999-12-31]"`。 + 需要保留的历史分区的时间范围。需要以 `[yyyy-MM-dd,yyyy-MM-dd],[...,...]` 格式进行设置。如果不设置，默认为 `"\\N"`。 我们举例说明。假设今天是 2021-09-06，按天分类，动态分区的属性设置为： @@ -376,11 +376,11 @@ mysql> SHOW DYNAMIC PARTITION TABLES; | TableName | Enable | TimeUnit | Start | End | Prefix | Buckets | StartOf | LastUpdateTime | LastSchedulerTime | State | LastCreatePartitionMsg | LastDropPartitionMsg | ReservedHistoryPeriods | +-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ | d3 | true | WEEK | -3 | 3 | p | 1 | MONDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [2021-12-01,2021-12-31] | -| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | -| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | [9999-12-31,9999-12-31] | +| d5 | true | DAY | -7 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d4 | true | WEEK | -3 | 3 | p | 1 | WEDNESDAY | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d6 | true | MONTH | -2147483648 | 2 | p | 8 | 3rd | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d2 | true | DAY | -3 | 3 | p | 32 | N/A | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +| d7 | true | MONTH | -2147483648 | 5 | p | 8 | 24th | N/A | 2020-05-25 14:29:24 | NORMAL | N/A | N/A | NULL | +-----------+--------+----------+-------------+------+--------+---------+-----------+----------------+---------------------+--------+------------------------+----------------------+-------------------------+ 7 rows in set (0.02 sec) ``` diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index f837e664a532c9..178dc22f696839 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -51,7 +51,7 @@ public class DynamicPartitionProperty { public static final int MAX_END_OFFSET = Integer.MAX_VALUE; public static final int NOT_SET_REPLICATION_NUM = -1; public static final int NOT_SET_HISTORY_PARTITION_NUM = -1; - public static final String NOT_SET_RESERVED_HISTORY_PERIODS = "[9999-12-31,9999-12-31]"; + public static final String NOT_SET_RESERVED_HISTORY_PERIODS = FeConstants.null_string; private boolean exist; diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 06770b45be2457..4d3d2ccd085547 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -300,18 +300,18 @@ private ArrayList getDropPartitionClause(Database db, OlapT String reservedHistoryPeriods = dynamicPartitionProperty.getReservedHistoryPeriods(); List ranges = DynamicPartitionUtil.convertStringToPeriodsList(reservedHistoryPeriods); - for (Range range : ranges) { - try { - String lowerBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, range.lowerEndpoint().toString(), partitionFormat); - String upperBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, range.upperEndpoint().toString(), partitionFormat); - Range reservedHistoryPartitionKeyRange = getClosedRange(db, olapTable, partitionColumn, partitionFormat, lowerBorderOfReservedHistory,upperBorderOfReservedHistory); - reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange); - } - catch (IllegalArgumentException e) { - return dropPartitionClauses; + if (ranges.size() != 0) { + for (Range range : ranges) { + try { + String lowerBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, range.lowerEndpoint().toString(), partitionFormat); + String upperBorderOfReservedHistory = DynamicPartitionUtil.getHistoryPartitionRangeString(dynamicPartitionProperty, range.upperEndpoint().toString(), partitionFormat); + Range reservedHistoryPartitionKeyRange = getClosedRange(db, olapTable, partitionColumn, partitionFormat, lowerBorderOfReservedHistory, upperBorderOfReservedHistory); + reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange); + } catch (IllegalArgumentException e) { + return dropPartitionClauses; + } } } - RangePartitionInfo info = (RangePartitionInfo) (olapTable.getPartitionInfo()); List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 5c3edc05313796..f21195e3649352 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -242,6 +242,9 @@ private static void checkHotPartitionNum(String val) throws DdlException { public static List convertStringToPeriodsList(String reservedHistoryPeriods) throws DdlException { List reservedHistoryPeriodsToRangeList = new ArrayList(); + if (FeConstants.null_string.equals(reservedHistoryPeriods)) { + return reservedHistoryPeriodsToRangeList; + } Integer sizeOfPeriods = reservedHistoryPeriods.split("],\\[").length; Pattern pattern = Pattern.compile("\\[([0-9]{4}-[0-9]{2}-[0-9]{2}),([0-9]{4}-[0-9]{2}-[0-9]{2})\\]"); Matcher matcher = pattern.matcher(reservedHistoryPeriods); @@ -259,6 +262,9 @@ public static List convertStringToPeriodsList(String reservedHistoryPerio } public static String sortedListedToString(String reservedHistoryPeriods) throws DdlException { + if (FeConstants.null_string.equals(reservedHistoryPeriods)) { + return reservedHistoryPeriods; + } List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods); reservedHistoryPeriodsToRangeList.sort(new Comparator() { @Override @@ -273,9 +279,12 @@ public int compare(Range o1, Range o2) { } private static void checkReservedHistoryPeriodValidate(String reservedHistoryPeriods) throws DdlException { - if (Strings.isNullOrEmpty(reservedHistoryPeriods) ) { + if (Strings.isNullOrEmpty(reservedHistoryPeriods)) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_EMPTY); } + if (FeConstants.null_string.equals(reservedHistoryPeriods)) { + return; + } // it has 5 kinds of situation // 1. "dynamic_partition.reserved_history_periods" = "[2021-07-01,]," invalid one // 2. "dynamic_partition.reserved_history_periods" = "2021-07-01", invalid. It must be surrounded by [] diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index e5107a719cf760..8b8afe2ffd345c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1216,7 +1216,7 @@ public void testMissReservedHistoryPeriods() throws Exception { ");"; createTable(createOlapTblStmt); OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_miss_reserved_history_periods"); - Assert.assertEquals("[9999-12-31,9999-12-31]", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); + Assert.assertEquals("\\N", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); } @Test From f7e426625c457c6cf3b1bcb445ecbf718c51b61d Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Thu, 23 Sep 2021 11:20:49 +0800 Subject: [PATCH 13/16] fix --- .../org/apache/doris/catalog/DynamicPartitionProperty.java | 2 +- .../org/apache/doris/common/util/DynamicPartitionUtil.java | 6 +++--- .../org/apache/doris/catalog/DynamicPartitionTableTest.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 178dc22f696839..64d8abfdcdd5dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -51,7 +51,7 @@ public class DynamicPartitionProperty { public static final int MAX_END_OFFSET = Integer.MAX_VALUE; public static final int NOT_SET_REPLICATION_NUM = -1; public static final int NOT_SET_HISTORY_PARTITION_NUM = -1; - public static final String NOT_SET_RESERVED_HISTORY_PERIODS = FeConstants.null_string; + public static final String NOT_SET_RESERVED_HISTORY_PERIODS = "NULL"; private boolean exist; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index f21195e3649352..f94915556d1ab4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -242,7 +242,7 @@ private static void checkHotPartitionNum(String val) throws DdlException { public static List convertStringToPeriodsList(String reservedHistoryPeriods) throws DdlException { List reservedHistoryPeriodsToRangeList = new ArrayList(); - if (FeConstants.null_string.equals(reservedHistoryPeriods)) { + if (DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_PERIODS.equals(reservedHistoryPeriods)) { return reservedHistoryPeriodsToRangeList; } Integer sizeOfPeriods = reservedHistoryPeriods.split("],\\[").length; @@ -262,7 +262,7 @@ public static List convertStringToPeriodsList(String reservedHistoryPerio } public static String sortedListedToString(String reservedHistoryPeriods) throws DdlException { - if (FeConstants.null_string.equals(reservedHistoryPeriods)) { + if (DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_PERIODS.equals(reservedHistoryPeriods)) { return reservedHistoryPeriods; } List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods); @@ -282,7 +282,7 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistoryPer if (Strings.isNullOrEmpty(reservedHistoryPeriods)) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_EMPTY); } - if (FeConstants.null_string.equals(reservedHistoryPeriods)) { + if (DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_PERIODS.equals(reservedHistoryPeriods)) { return; } // it has 5 kinds of situation diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 8b8afe2ffd345c..6d28f3af4a867e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1216,7 +1216,7 @@ public void testMissReservedHistoryPeriods() throws Exception { ");"; createTable(createOlapTblStmt); OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_miss_reserved_history_periods"); - Assert.assertEquals("\\N", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); + Assert.assertEquals("NULL", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); } @Test From 89fc952dce9b8d04700944115ff7cb74e9e2aa4a Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Fri, 24 Sep 2021 14:38:58 +0800 Subject: [PATCH 14/16] modify HOUR --- .../common/util/DynamicPartitionUtil.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index f94915556d1ab4..f13e7b884d883c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -653,18 +653,31 @@ public static String getPartitionRangeString(DynamicPartitionProperty property, public static String getHistoryPartitionRangeString(DynamicPartitionProperty dynamicPartitionProperty, String time, String format) { ZoneId zoneId = dynamicPartitionProperty.getTimeZone().toZoneId(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); Date date = null; Timestamp timestamp = null; + String timeUnit = dynamicPartitionProperty.getTimeUnit(); DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.s").withZone(zoneId); - try { - date = simpleDateFormat.parse(time); - } catch (ParseException e) { - LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage()); + if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + try { + date = simpleDateFormat.parse(time); + } catch (ParseException e) { + LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage()); + return getFormattedTimeWithoutMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); + } + timestamp = new Timestamp(date.getTime()); + return getFormattedTimeWithoutMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); + } else { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + try { + date = simpleDateFormat.parse(time); + } catch (ParseException e) { + LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage()); + return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); + } + timestamp = new Timestamp(date.getTime()); return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); } - timestamp = new Timestamp(date.getTime()); - return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); } /** From 1f61d204aec50f0f6034499360a53fd8523d7c39 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Fri, 24 Sep 2021 14:54:43 +0800 Subject: [PATCH 15/16] modify HOUR fix --- .../administrator-guide/dynamic-partition.md | 13 ++- .../administrator-guide/dynamic-partition.md | 14 ++- .../catalog/DynamicPartitionProperty.java | 6 +- .../clone/DynamicPartitionScheduler.java | 2 +- .../org/apache/doris/common/ErrorCode.java | 2 +- .../common/util/DynamicPartitionUtil.java | 71 +++++++++------- .../org/apache/doris/qe/ShowExecutor.java | 2 +- .../catalog/DynamicPartitionTableTest.java | 85 ++++++++++++++++++- 8 files changed, 149 insertions(+), 46 deletions(-) diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md index f8bd5cf85d7c60..402e37d7b2a681 100644 --- a/docs/en/administrator-guide/dynamic-partition.md +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -153,11 +153,11 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: * `dynamic_partition.reserved_history_periods` - The range of reserved history periods. It should be in the form of `[yyyy-MM-dd,yyyy-MM-dd],[...,...]`. The default value is `"\\N"`, which means it is not set. + The range of reserved history periods. It should be in the form of `[yyyy-MM-dd,yyyy-MM-dd],[...,...]` while the `dynamic_partition.time_unit` is "DAY, WEEK, and MONTH". And it should be in the form of `[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]` while the dynamic_partition.time_unit` is "HOUR". And no more spaces expected. The default value is `"NULL"`, which means it is not set. Let us give an example. Suppose today is 2021-09-06，partitioned by day, and the properties of dynamic partition are set to: - ```end=3, start=-3, reserved_history_periods="[2020-06-01,2020-06-20],[2020-10-31,2020-11-15]"```。 + ```time_unit="DAY/WEEK/MONTH", end=3, start=-3, reserved_history_periods="[2020-06-01,2020-06-20],[2020-10-31,2020-11-15]"```. The the system will automatically reserve following partitions in following period : @@ -165,6 +165,15 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: ["2020-06-01","2020-06-20"], ["2020-10-31","2020-11-15"] ``` + or + + ```time_unit="HOUR", end=3, start=-3, reserved_history_periods="[2020-06-01 00:00:00,2020-06-01 03:00:00]"```. + + The the system will automatically reserve following partitions in following period : + + ``` + ["2020-06-01 00:00:00","2020-06-01 03:00:00"] + ``` Otherwise, every `[...,...]` in `reserved_history_periods` is a couple of properties, and they should be set at the same time. And the first date can't be larger than the second one. diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md index bcc4ad74ecc492..246559e1884a07 100644 --- a/docs/zh-CN/administrator-guide/dynamic-partition.md +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -151,11 +151,11 @@ under the License. * `dynamic_partition.reserved_history_periods` - 需要保留的历史分区的时间范围。需要以 `[yyyy-MM-dd,yyyy-MM-dd],[...,...]` 格式进行设置。如果不设置，默认为 `"\\N"`。 + 需要保留的历史分区的时间范围。当`dynamic_partition.time_unit` 设置为 "DAY/WEEK/MONTH" 时，需要以 `[yyyy-MM-dd,yyyy-MM-dd],[...,...]` 格式进行设置。当`dynamic_partition.time_unit` 设置为 "HOUR" 时，需要以 `[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]` 的格式来进行设置。如果不设置，默认为 `"NULL"`。 我们举例说明。假设今天是 2021-09-06，按天分类，动态分区的属性设置为： - ```end=3, start=-3, reserved_history_periods="[2020-06-01,2020-06-20],[2020-10-31,2020-11-15]"```。 + ```time_unit="DAY/WEEK/MONTH", end=3, start=-3, reserved_history_periods="[2020-06-01,2020-06-20],[2020-10-31,2020-11-15]"```。 则系统会自动保留： @@ -164,6 +164,16 @@ under the License. ["2020-10-31","2020-11-15"] ``` + 或者 + + ```time_unit="HOUR", end=3, start=-3, reserved_history_periods="[2020-06-01 00:00:00,2020-06-01 03:00:00]"```. + + 则系统会自动保留： + + ``` + ["2020-06-01 00:00:00","2020-06-01 03:00:00"] + ``` + 这两个时间段的分区。其中，`reserved_history_periods` 的每一个 `[...,...]` 是一对设置项，两者需要同时被设置，且第一个时间不能大于第二个时间``。 #### 创建历史分区规则 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 64d8abfdcdd5dd..ca8bf9002ec55e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -88,7 +88,7 @@ public DynamicPartitionProperty(Map properties) { this.createHistoryPartition = Boolean.parseBoolean(properties.get(CREATE_HISTORY_PARTITION)); this.historyPartitionNum = Integer.parseInt(properties.getOrDefault(HISTORY_PARTITION_NUM, String.valueOf(NOT_SET_HISTORY_PARTITION_NUM))); this.hotPartitionNum = Integer.parseInt(properties.getOrDefault(HOT_PARTITION_NUM, "0")); - this.reservedHistoryPeriods = properties.getOrDefault(RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS).replace(" ", ""); + this.reservedHistoryPeriods = properties.getOrDefault(RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS); createStartOfs(properties); } else { this.exist = false; @@ -190,8 +190,8 @@ public String getReservedHistoryPeriods() { return reservedHistoryPeriods; } - public String getSortedReservedHistoryPeriods(String reservedHistoryPeriods) throws DdlException { - return DynamicPartitionUtil.sortedListedToString(reservedHistoryPeriods); + public String getSortedReservedHistoryPeriods(String reservedHistoryPeriods, String timeUnit) throws DdlException { + return DynamicPartitionUtil.sortedListedToString(reservedHistoryPeriods, timeUnit); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 4d3d2ccd085547..ecc5981073b0fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -298,7 +298,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT } String reservedHistoryPeriods = dynamicPartitionProperty.getReservedHistoryPeriods(); - List ranges = DynamicPartitionUtil.convertStringToPeriodsList(reservedHistoryPeriods); + List ranges = DynamicPartitionUtil.convertStringToPeriodsList(reservedHistoryPeriods, dynamicPartitionProperty.getTimeUnit()); if (ranges.size() != 0) { for (Range range : ranges) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java index 08e61ed7727775..69b9baed27c5df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java @@ -247,7 +247,7 @@ public enum ErrorCode { ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_EMPTY(5076, new byte[] {'4', '2', '0', '0', '0'}, "Dynamic reserved history periods is empty."), ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID(5077, new byte[] {'4', '2', '0', '0', '0'}, - "Invalid \" %s \" value %s. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""), + "Invalid \" %s \" value %s. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH or \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR."), ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_ENDS_LENGTH_NOT_EQUAL(5078, new byte[] {'4', '2', '0', '0', '0'}, "RESERVED_HISTORY_PERIODS must have pairs of date value. The input %s is not valid."), ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_START_LARGER_THAN_ENDS(5079, new byte[] {'4', '2', '0', '0', '0'}, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index f13e7b884d883c..1725ff49055d7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -240,15 +240,14 @@ private static void checkHotPartitionNum(String val) throws DdlException { } } - public static List convertStringToPeriodsList(String reservedHistoryPeriods) throws DdlException { + public static List convertStringToPeriodsList(String reservedHistoryPeriods, String timeUnit) throws DdlException { List reservedHistoryPeriodsToRangeList = new ArrayList(); if (DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_PERIODS.equals(reservedHistoryPeriods)) { return reservedHistoryPeriodsToRangeList; } - Integer sizeOfPeriods = reservedHistoryPeriods.split("],\\[").length; - Pattern pattern = Pattern.compile("\\[([0-9]{4}-[0-9]{2}-[0-9]{2}),([0-9]{4}-[0-9]{2}-[0-9]{2})\\]"); + + Pattern pattern = getPattern(timeUnit); Matcher matcher = pattern.matcher(reservedHistoryPeriods); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); while (matcher.find()) { String lowerBorderOfReservedHistory = matcher.group(1); String upperBorderOfReservedHistory = matcher.group(2); @@ -261,11 +260,19 @@ public static List convertStringToPeriodsList(String reservedHistoryPerio return reservedHistoryPeriodsToRangeList; } - public static String sortedListedToString(String reservedHistoryPeriods) throws DdlException { + private static Pattern getPattern(String timeUnit) { + if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { + return Pattern.compile("\\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2})\\]"); + } else { + return Pattern.compile("\\[([0-9]{4}-[0-9]{2}-[0-9]{2}),([0-9]{4}-[0-9]{2}-[0-9]{2})\\]"); + } + } + + public static String sortedListedToString(String reservedHistoryPeriods, String timeUnit) throws DdlException { if (DynamicPartitionProperty.NOT_SET_RESERVED_HISTORY_PERIODS.equals(reservedHistoryPeriods)) { return reservedHistoryPeriods; } - List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods); + List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods, timeUnit); reservedHistoryPeriodsToRangeList.sort(new Comparator() { @Override public int compare(Range o1, Range o2) { @@ -278,7 +285,7 @@ public int compare(Range o1, Range o2) { return String.join(",", sortedReservedHistoryPeriods); } - private static void checkReservedHistoryPeriodValidate(String reservedHistoryPeriods) throws DdlException { + private static void checkReservedHistoryPeriodValidate(String reservedHistoryPeriods, String timeUnit) throws DdlException { if (Strings.isNullOrEmpty(reservedHistoryPeriods)) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_EMPTY); } @@ -295,9 +302,9 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistoryPer ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); } - List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods); + List reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods, timeUnit); Integer sizeOfPeriods = reservedHistoryPeriods.split("],\\[").length; - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + SimpleDateFormat sdf = getSimpleDateFormat(timeUnit); if (reservedHistoryPeriodsToRangeList.size() != sizeOfPeriods) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID, DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); @@ -307,15 +314,27 @@ private static void checkReservedHistoryPeriodValidate(String reservedHistoryPer String formattedLowerBound = sdf.format(sdf.parse(range.lowerEndpoint().toString())); String formattedUpperBound = sdf.format(sdf.parse(range.upperEndpoint().toString())); if (!range.lowerEndpoint().toString().equals(formattedLowerBound) || !range.upperEndpoint().toString().equals(formattedUpperBound)) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be correct DATE value \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + + " value. It must be correct DATE value \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH " + + "or \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR."); } } } catch (ParseException e) { - throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + " value. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""); + throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS + + " value. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH " + + "or \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR."); } } } + private static SimpleDateFormat getSimpleDateFormat(String timeUnit) { + if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { + return new SimpleDateFormat(DATETIME_FORMAT); + } else { + return new SimpleDateFormat(DATE_FORMAT); + } + } + public static boolean checkDynamicPartitionPropertiesExist(Map properties) { if (properties == null) { return false; @@ -544,8 +563,8 @@ public static Map analyzeDynamicPartition(Map pr analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val); } if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS)) { - String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS).replace(" ",""); - checkReservedHistoryPeriodValidate(reservedHistoryPeriods); + String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS); + checkReservedHistoryPeriodValidate(reservedHistoryPeriods, analyzedProperties.get(DynamicPartitionProperty.TIME_UNIT)); properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS); analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); } @@ -657,27 +676,15 @@ public static String getHistoryPartitionRangeString(DynamicPartitionProperty dyn Timestamp timestamp = null; String timeUnit = dynamicPartitionProperty.getTimeUnit(); DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.s").withZone(zoneId); - if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) { - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - date = simpleDateFormat.parse(time); - } catch (ParseException e) { - LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage()); - return getFormattedTimeWithoutMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); - } - timestamp = new Timestamp(date.getTime()); + SimpleDateFormat simpleDateFormat = getSimpleDateFormat(timeUnit); + try { + date = simpleDateFormat.parse(time); + } catch (ParseException e) { + LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage()); return getFormattedTimeWithoutMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); - } else { - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); - try { - date = simpleDateFormat.parse(time); - } catch (ParseException e) { - LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage()); - return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); - } - timestamp = new Timestamp(date.getTime()); - return getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); } + timestamp = new Timestamp(date.getTime()); + return getFormattedTimeWithoutMinuteSecond(ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index b8975db13c4941..d40b8478dfdeed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1834,7 +1834,7 @@ private void handleShowDynamicPartition() { dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DYNAMIC_PARTITION_STATE), dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.CREATE_PARTITION_MSG), dynamicPartitionScheduler.getRuntimeInfo(olapTable.getId(), DynamicPartitionScheduler.DROP_PARTITION_MSG), - dynamicPartitionProperty.getSortedReservedHistoryPeriods(unsortedReservedHistoryPeriods))); + dynamicPartitionProperty.getSortedReservedHistoryPeriods(unsortedReservedHistoryPeriods, dynamicPartitionProperty.getTimeUnit().toUpperCase()))); } catch (DdlException e) { e.printStackTrace(); } finally { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 6d28f3af4a867e..9870fc7f9b91ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1259,6 +1259,39 @@ public void testNormalReservedHisrotyPeriods() throws Exception { OlapTable table = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_normal_reserved_history_periods"); Assert.assertEquals("[2020-06-01,2020-06-20],[2020-10-25,2020-11-15],[2021-06-01,2021-06-20]", table.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); Assert.assertEquals(table.getAllPartitions().size(), 9); + + String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_normal_reserved_history_periods2` (\n" + + " `k1` datetime NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01 00:00:00\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-01-01 03:00:00\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-01-01 04:00:00\"),\n" + + "PARTITION p4 VALUES LESS THAN (\"2020-01-01 08:00:00\"),\n" + + "PARTITION p5 VALUES LESS THAN (\"2020-06-20 00:00:00\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"hour\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[2014-01-01 00:00:00,2014-01-01 03:00:00]\"\n" + + ");"; + createTable(createOlapTblStmt2); + OlapTable table2 = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_normal_reserved_history_periods2"); + Assert.assertEquals("[2014-01-01 00:00:00,2014-01-01 03:00:00]", table2.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); + Assert.assertEquals(table2.getAllPartitions().size(), 6); } @Test @@ -1290,7 +1323,9 @@ public void testInvalidReservedHistoryPeriods() throws Exception { "\"dynamic_partition.reserved_history_periods\" = \"[20210101,2021-10-10]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [20210101,2021-10-10]. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", + "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [20210101,2021-10-10]. " + + "It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH or " + + "\"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.", () -> createTable(createOlapTblStmt1)); String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_invalid_reserved_history_periods2` (\n" + @@ -1320,7 +1355,10 @@ public void testInvalidReservedHistoryPeriods() throws Exception { "\"dynamic_partition.reserved_history_periods\" = \"[0000-00-00,2021-10-10]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_periods value. It must be correct DATE value \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", + "errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_periods value. " + + "It must be correct DATE value " + + "\"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH or " + + "\"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.", () -> createTable(createOlapTblStmt2)); } @@ -1353,7 +1391,10 @@ public void testReservedHistoryPeriodsValidate() throws Exception { "\"dynamic_partition.reserved_history_periods\" = \"[2021-01-01,]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [2021-01-01,]. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", + "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [2021-01-01,]. " + + "It must be like " + + "\"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH " + + "or \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.", () -> createTable(createOlapTblStmt1)); String createOlapTblStmt2 = "CREATE TABLE test.`dynamic_partition_reserved_history_periods_validate2` (\n" + @@ -1383,7 +1424,10 @@ public void testReservedHistoryPeriodsValidate() throws Exception { "\"dynamic_partition.reserved_history_periods\" = \"[,2021-01-01]\"\n" + ");"; ExceptionChecker.expectThrowsWithMsg(DdlException.class, - "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [,2021-01-01]. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\"", + "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [,2021-01-01]. " + + "It must be like " + + "\"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH or " + + "\"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.", () -> createTable(createOlapTblStmt2)); String createOlapTblStmt3 = "CREATE TABLE test.`dynamic_partition_reserved_history_periods_validate3` (\n" + @@ -1415,5 +1459,38 @@ public void testReservedHistoryPeriodsValidate() throws Exception { ExceptionChecker.expectThrowsWithMsg(DdlException.class, "errCode = 2, detailMessage = The first date is larger than the second date, [2021-10-01,2021-09-01] is invalid.", () -> createTable(createOlapTblStmt3)); + + String createOlapTblStmt4 = "CREATE TABLE test.`dynamic_partition_reserved_history_periods_validate4` (\n" + + " `k1` datetime NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01 00:00:00\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01 00:00:00\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01 00:00:00\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"hour\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[2020-01-01,2020-03-01]\"\n" + + ");"; + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \" value [2020-01-01,2020-03-01]. " + + "It must be like " + + "\"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH " + + "or \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.", + () -> createTable(createOlapTblStmt4)); } } From 8a23015d6c02acdbfee728fd41fa1d78b0e4b1a2 Mon Sep 17 00:00:00 2001 From: Henry2SS <1021263336@qq.com> Date: Sun, 26 Sep 2021 10:58:22 +0800 Subject: [PATCH 16/16] change contains to RangeUtils.isTwoRangesIntesect && add more ut --- .../clone/DynamicPartitionScheduler.java | 2 +- .../apache/doris/common/util/RangeUtils.java | 6 ++++ .../catalog/DynamicPartitionTableTest.java | 29 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index ecc5981073b0fe..4a6fc1d2d2e6bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -322,7 +322,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT Long checkDropPartitionId = idToItem.getKey(); Range checkDropPartitionKey = idToItem.getValue().getItems(); for (Range reserveHistoryPartitionKeyRange : reservedHistoryPartitionKeyRangeList) { - if (reserveHistoryPartitionKeyRange.contains(checkDropPartitionKey.lowerEndpoint())) { + if (RangeUtils.checkIsTwoRangesIntersect(reserveHistoryPartitionKeyRange, checkDropPartitionKey)) { isContaineds.put(checkDropPartitionId, true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java index eeef22eac4e919..13e0f5ba0fc73f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RangeUtils.java @@ -51,6 +51,12 @@ public static void checkRangeIntersect(Range range1, Range range1, Range range2) { + if (range2.isConnected(range1) && !range2.intersection(range1).isEmpty()) { + return true; + } + return false; + } /* * Pass only if the 2 range lists are exactly same * What is "exactly same"? diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 9870fc7f9b91ea..6c36ba8cf53656 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1292,6 +1292,35 @@ public void testNormalReservedHisrotyPeriods() throws Exception { OlapTable table2 = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_normal_reserved_history_periods2"); Assert.assertEquals("[2014-01-01 00:00:00,2014-01-01 03:00:00]", table2.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); Assert.assertEquals(table2.getAllPartitions().size(), 6); + + String createOlapTblStmt3 = "CREATE TABLE test.`dynamic_partition_normal_reserved_history_periods3` (\n" + + " `k1` int NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p202127 VALUES [(\"20200527\"), (\"20200628\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"1\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.reserved_history_periods\" = \"[2020-06-01,2020-06-30]\"\n" + + ");"; + createTable(createOlapTblStmt3); + OlapTable table3 = (OlapTable) Catalog.getCurrentCatalog().getDbOrAnalysisException("default_cluster:test").getTableOrAnalysisException("dynamic_partition_normal_reserved_history_periods3"); + Assert.assertEquals("[2020-06-01,2020-06-30]", table3.getTableProperty().getDynamicPartitionProperty().getReservedHistoryPeriods()); + Assert.assertEquals(table3.getAllPartitions().size(), 5); } @Test