From f75c66e07cd2a93e28b7a40f22fecc7fd6e276f1 Mon Sep 17 00:00:00 2001 From: caoyang10 <535011125@qq.com> Date: Wed, 2 Sep 2020 14:35:04 +0800 Subject: [PATCH] [Feature] support hour time unit with dynamic parition --- .../administrator-guide/dynamic-partition.md | 4 ++- .../administrator-guide/dynamic-partition.md | 4 ++- .../org/apache/doris/common/ErrorCode.java | 2 +- .../common/util/DynamicPartitionUtil.java | 30 +++++++++++++++++-- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md index 078eeced016660..0e989ca0b5ac1a 100644 --- a/docs/en/administrator-guide/dynamic-partition.md +++ b/docs/en/administrator-guide/dynamic-partition.md @@ -81,7 +81,9 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`: * `dynamic_partition.time_unit` - The unit for dynamic partition scheduling. Can be specified as `DAY`,` WEEK`, and `MONTH`, means to create or delete partitions by day, week, and month, respectively. + The unit for dynamic partition scheduling. Can be specified as `HOUR`,`DAY`,` WEEK`, and `MONTH`, means to create or delete partitions by hour, day, week, and month, respectively. + + When specified as `HOUR`, the suffix format of the dynamically created partition name is `yyyyMMddHH`, for example, `2020032501`. When specified as `DAY`, the suffix format of the dynamically created partition name is `yyyyMMdd`, for example, `20200325`. diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md index 167e1372195cdc..7f77cdf1815ba6 100644 --- a/docs/zh-CN/administrator-guide/dynamic-partition.md +++ b/docs/zh-CN/administrator-guide/dynamic-partition.md @@ -79,8 +79,10 @@ under the License. * `dynamic_partition.time_unit` - 动态分区调度的单位。可指定为 `DAY`、`WEEK`、`MONTH`。分别表示按天、按星期、按月进行分区创建或删除。 + 动态分区调度的单位。可指定为 `HOUR`、`DAY`、`WEEK`、`MONTH`。分别表示按天、按星期、按月进行分区创建或删除。 + 当指定为 `HOUR` 时，动态创建的分区名后缀格式为 `yyyyMMddHH`，例如`2020032501`。 + 当指定为 `DAY` 时，动态创建的分区名后缀格式为 `yyyyMMdd`，例如`20200325`。 当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`。即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java index 186899ccf4a0fa..75ae600b48741b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java @@ -210,7 +210,7 @@ public enum ErrorCode { "Table %s is not a colocated table"), ERR_INVALID_OPERATION(5065, new byte[] { '4', '2', '0', '0', '0' }, "Operation %s is invalid"), ERROR_DYNAMIC_PARTITION_TIME_UNIT(5065, new byte[] {'4', '2', '0', '0', '0'}, - "Unsupported time unit %s. Expect DAY/WEEK/MONTH."), + "Unsupported time unit %s. Expect HOUR/DAY/WEEK/MONTH."), ERROR_DYNAMIC_PARTITION_START_ZERO(5066, new byte[] {'4', '2', '0', '0', '0'}, "Dynamic partition start must less than 0"), ERROR_DYNAMIC_PARTITION_START_FORMAT(5066, new byte[] {'4', '2', '0', '0', '0'}, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index c83eac0e114a32..5e68161609d617 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -63,6 +63,7 @@ public class DynamicPartitionUtil { public static void checkTimeUnit(String timeUnit) throws DdlException { if (Strings.isNullOrEmpty(timeUnit) || !(timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString()) + || timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString()) || timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString()) || timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString()))) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_TIME_UNIT, timeUnit); @@ -365,6 +366,8 @@ public static String getFormattedPartitionName(TimeZone tz, String formattedDate return formattedDateStr.substring(0, 8); } else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) { return formattedDateStr.substring(0, 6); + } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { + return formattedDateStr.substring(0, 10); } else { formattedDateStr = formattedDateStr.substring(0, 8); Calendar calendar = Calendar.getInstance(tz); @@ -385,7 +388,8 @@ public static String getFormattedPartitionName(TimeZone tz, String formattedDate } // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss] - // TODO: support HOUR and YEAR + // add support: HOUR by caoyang10 + // TODO: support YEAR public static String getPartitionRangeString(DynamicPartitionProperty property, ZonedDateTime current, int offset, String format) { String timeUnit = property.getTimeUnit(); @@ -393,10 +397,27 @@ public static String getPartitionRangeString(DynamicPartitionProperty property, return getPartitionRangeOfDay(current, offset, format); } else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) { return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), format); - } else { // MONTH + } else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { // MONTH + return getPartitionRangeOfHour(current, offset, format); + } else { return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), format); } } + + /** + * return formatted string of partition range in HOUR granularity. + * offset: The offset from the current hour. 0 means current hour, 1 means next hour, -1 last hour. + * format: the format of the return date string. + * + * Eg: + * Today is 2020-05-24 00:12:34, offset = -1 + * It will return 2020-05-23 23:00:00 + * Today is 2020-05-24 00, offset = 1 + * It will return 2020-05-24 01:00:00 + */ + private static String getPartitionRangeOfHour(ZonedDateTime current, int offset, String format) { + return getFormattedTimeWithoutMinuteSecond(current.plusHours(offset), format); + } /** * return formatted string of partition range in DAY granularity. @@ -460,6 +481,11 @@ private static String getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime zone return DateTimeFormatter.ofPattern(format).format(timeWithoutHourMinuteSecond); } + private static String getFormattedTimeWithoutMinuteSecond(ZonedDateTime zonedDateTime, String format) { + ZonedDateTime timeWithoutMinuteSecond = zonedDateTime.withMinute(0).withSecond(0); + return DateTimeFormatter.ofPattern(format).format(timeWithoutMinuteSecond); + } + /** * Used to indicate the start date. * Taking the year as the granularity, it can indicate the month and day as the start date.