From 7198191dcf0b8b70c8202fd90a1f36a342fe2cf5 Mon Sep 17 00:00:00 2001 From: LiBinfeng Date: Thu, 13 Mar 2025 16:53:01 +0800 Subject: [PATCH 1/2] [fix](Nereids) fix str_to_date date value out of range --- .../DateTimeExtractAndTransform.java | 30 +++++++++++-------- .../apache/doris/nereids/util/DateUtils.java | 6 ++-- .../fold_constant_date_arithmatic.groovy | 10 +++++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index 61857f907ebd3f..476fdaa79642b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -295,14 +295,15 @@ private static LocalDateTime firstDayOfWeek(LocalDateTime dateTime) { @ExecFunction(name = "date_format") public static Expression dateFormat(DateLiteral date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter(Locale.US).format( + return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format( java.time.LocalDate.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay())))); } @ExecFunction(name = "date_format") public static Expression dateFormat(DateTimeLiteral date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter(Locale.US).format( + return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()) + .withResolverStyle(ResolverStyle.STRICT).format( java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()), ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond())))); } @@ -310,14 +311,16 @@ public static Expression dateFormat(DateTimeLiteral date, StringLikeLiteral form @ExecFunction(name = "date_format") public static Expression dateFormat(DateV2Literal date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter(Locale.US).format( + return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()) + .withResolverStyle(ResolverStyle.STRICT).format( java.time.LocalDate.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay())))); } @ExecFunction(name = "date_format") public static Expression dateFormat(DateTimeV2Literal date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter(Locale.US).format( + return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()) + .withResolverStyle(ResolverStyle.STRICT).format( java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()), ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond())))); } @@ -539,7 +542,8 @@ public static Expression unixTimestamp(DateTimeV2Literal date) { @ExecFunction(name = "unix_timestamp") public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - DateTimeFormatter formatter = DateUtils.formatBuilder(format.getValue()).toFormatter(); + DateTimeFormatter formatter = DateUtils.dateTimeFormatter(format.getValue()) + .withResolverStyle(ResolverStyle.STRICT); LocalDateTime dateObj; try { dateObj = LocalDateTime.parse(date.getValue(), formatter); @@ -630,20 +634,20 @@ public static Expression strToDate(StringLikeLiteral str, StringLikeLiteral form if (returnType instanceof DateTimeV2Type) { boolean hasMicroPart = org.apache.doris.analysis.DateLiteral .hasMicroSecondPart(format.getStringValue()); - return DateTimeV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) - .toFormatter(), str.getValue()), hasMicroPart ? 6 : 0); + return DateTimeV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils + .dateTimeFormatter(format.getValue()), str.getValue()), hasMicroPart ? 6 : 0); } else { - return DateTimeLiteral.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) - .toFormatter(), str.getValue())); + return DateTimeLiteral.fromJavaDateType(DateUtils.getTime(DateUtils + .dateTimeFormatter(format.getValue()), str.getValue())); } } else { DataType returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATE)); if (returnType instanceof DateV2Type) { - return DateV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) - .toFormatter(), str.getValue())); + return DateV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.dateTimeFormatter(format.getValue()), + str.getValue())); } else { - return DateLiteral.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) - .toFormatter(), str.getValue())); + return DateLiteral.fromJavaDateType(DateUtils.getTime(DateUtils.dateTimeFormatter(format.getValue()), + str.getValue())); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java index 0b4604a4ed7c4d..047b8d43e1084a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java @@ -27,12 +27,14 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import java.time.format.ResolverStyle; import java.time.format.SignStyle; import java.time.format.TextStyle; import java.time.temporal.ChronoField; import java.time.temporal.IsoFields; import java.time.temporal.TemporalAccessor; import java.time.temporal.WeekFields; +import java.util.Locale; import java.util.Set; /** @@ -47,7 +49,7 @@ public class DateUtils { /** * format builder. */ - public static DateTimeFormatterBuilder formatBuilder(String pattern) throws AnalysisException { + public static DateTimeFormatter dateTimeFormatter(String pattern) throws AnalysisException { DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { @@ -159,7 +161,7 @@ public static DateTimeFormatterBuilder formatBuilder(String pattern) throws Anal builder.appendLiteral(character); } } - return builder; + return builder.toFormatter(Locale.US).withResolverStyle(ResolverStyle.STRICT); } /** diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_date_arithmatic.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_date_arithmatic.groovy index e19a01b2a3868a..ccd3547deefe29 100644 --- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_date_arithmatic.groovy +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_date_arithmatic.groovy @@ -38,4 +38,14 @@ suite("fold_constant_date_arithmatic") { testFoldConst("SELECT date_format('2020-12-01 12:00:30.01', '%I');") testFoldConst("SELECT date_format('2020-12-01 12:00:30.01', '%l');") testFoldConst("SELECT date_format('2020-12-01 12:00:30.01', '%r');") + + testFoldConst("select str_to_date('2023-02-29', '%Y-%m-%d') AS result;") + testFoldConst("select str_to_date('1900-02-29', '%Y-%m-%d') AS result;") + testFoldConst("select str_to_date('2025-04-31', '%Y-%m-%d') AS result;") + testFoldConst("select str_to_date('31-12-2020 23:59:59', '%d-%m-%Y %H:%i:%s');") + testFoldConst("select str_to_date('2020-12-31T23:59:59', '%Y-%m-%dT%H:%i:%s');") + testFoldConst("select str_to_date('20201231235959', '%Y%m%d%H%i%s');") + testFoldConst("select str_to_date('31/12/2020 23:59', '%d/%m/%Y %H:%i');") + testFoldConst("select str_to_date('31/12/2020 11:59 PM', '%d/%m/%Y %h:%i %p');") + testFoldConst("select str_to_date('20201231T235959', '%Y%m%dT%H%i%s');") } From 3cc97608940c29cf5b8b57c041071e419bda7441 Mon Sep 17 00:00:00 2001 From: LiBinfeng Date: Mon, 17 Mar 2025 11:38:46 +0800 Subject: [PATCH 2/2] fix strict mode set twice --- .../executable/DateTimeExtractAndTransform.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index 476fdaa79642b3..e5036730822ca3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -302,8 +302,7 @@ public static Expression dateFormat(DateLiteral date, StringLikeLiteral format) @ExecFunction(name = "date_format") public static Expression dateFormat(DateTimeLiteral date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()) - .withResolverStyle(ResolverStyle.STRICT).format( + return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format( java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()), ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond())))); } @@ -311,16 +310,14 @@ public static Expression dateFormat(DateTimeLiteral date, StringLikeLiteral form @ExecFunction(name = "date_format") public static Expression dateFormat(DateV2Literal date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()) - .withResolverStyle(ResolverStyle.STRICT).format( + return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format( java.time.LocalDate.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay())))); } @ExecFunction(name = "date_format") public static Expression dateFormat(DateTimeV2Literal date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()) - .withResolverStyle(ResolverStyle.STRICT).format( + return new VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format( java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()), ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond())))); } @@ -542,8 +539,7 @@ public static Expression unixTimestamp(DateTimeV2Literal date) { @ExecFunction(name = "unix_timestamp") public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral format) { format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format); - DateTimeFormatter formatter = DateUtils.dateTimeFormatter(format.getValue()) - .withResolverStyle(ResolverStyle.STRICT); + DateTimeFormatter formatter = DateUtils.dateTimeFormatter(format.getValue()); LocalDateTime dateObj; try { dateObj = LocalDateTime.parse(date.getValue(), formatter);