From fb5d52d4de4fcac38e74ff8e95e6fb81b8be6b24 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 6 Mar 2025 18:12:09 +0800 Subject: [PATCH 1/5] update commit --- .../nereids/trees/expressions/literal/DateLiteral.java | 6 +++++- .../nereids/trees/expressions/literal/DateTimeLiteral.java | 4 ++-- .../org/apache/doris/nereids/util/TypeCoercionUtils.java | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index d0364991c1e3b2..1d72a82fab7c03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableSet; +import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Year; @@ -290,7 +291,7 @@ public static Result parseDateLiteral(String s) } /** parseDateTime */ - public static Result parseDateTime(String s) { + public static Result parseDateTime(String s) { // fast parse '2022-01-01' if (s.length() == 10 && s.charAt(4) == '-' && s.charAt(7) == '-') { TemporalAccessor date = fastParseDate(s); @@ -346,6 +347,9 @@ public static Result parseDateTime(String s } return Result.ok(dateTime); + } catch (DateTimeException e) { + return Result.err(() -> + new DateTimeException("date/datetime literal [" + originalString + "] is invalid", e)); } catch (Exception ex) { return Result.err(() -> new AnalysisException("date/datetime literal [" + originalString + "] is invalid")); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 17c5678b05170b..f056bed9169a31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -132,8 +132,8 @@ public static int determineScale(String s) { } /** parseDateTimeLiteral */ - public static Result parseDateTimeLiteral(String s, boolean isV2) { - Result parseResult = parseDateTime(s); + public static Result parseDateTimeLiteral(String s, boolean isV2) { + Result parseResult = parseDateTime(s); if (parseResult.isError()) { return parseResult.cast(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 3acf4508ac9d20..d45482926c14ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -605,7 +605,7 @@ public static Optional characterLiteralTypeCoercion(String value, Da if (parseResult.isOk()) { ret = parseResult.get(); } else { - Result parseResult2 + Result parseResult2 = DateTimeV2Literal.parseDateTimeLiteral(value, true); if (parseResult2.isOk()) { ret = parseResult2.get(); From 387751a73cd9abfafd20d8166282ee10073e7503 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 6 Mar 2025 18:26:32 +0800 Subject: [PATCH 2/5] update --- .../doris/nereids/trees/expressions/literal/DateLiteral.java | 4 ++-- .../java/org/apache/doris/nereids/util/TypeCoercionUtils.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 1d72a82fab7c03..b8eba0b2499cc0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -274,8 +274,8 @@ static Result normalize(String s) { } /** parseDateLiteral */ - public static Result parseDateLiteral(String s) { - Result parseResult = parseDateTime(s); + public static Result parseDateLiteral(String s) { + Result parseResult = parseDateTime(s); if (parseResult.isError()) { return parseResult.cast(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index d45482926c14ee..29066b0c3b2b2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -600,7 +600,7 @@ public static Optional characterLiteralTypeCoercion(String value, Da } else if (dataType.isDateTimeType() && DateTimeChecker.isValidDateTime(value)) { ret = DateTimeLiteral.parseDateTimeLiteral(value, false).orElse(null); } else if (dataType.isDateV2Type() && DateTimeChecker.isValidDateTime(value)) { - Result parseResult + Result parseResult = DateV2Literal.parseDateLiteral(value); if (parseResult.isOk()) { ret = parseResult.get(); From 7a7dee28f3e1f7759d35a8600c10b2f9670e6b43 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 6 Mar 2025 18:54:19 +0800 Subject: [PATCH 3/5] add test --- .../nereids_syntax_p0/test_cast_datetime.out | 19 +++++++++++++++++++ .../test_cast_datetime.groovy | 17 +++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/regression-test/data/nereids_syntax_p0/test_cast_datetime.out b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out index 4038c435d0101f..a0d754fe52dcf8 100644 --- a/regression-test/data/nereids_syntax_p0/test_cast_datetime.out +++ b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out @@ -2,3 +2,22 @@ -- !1 -- 1 +-- !2 -- +1 2000-01-01 2000-01-01 2000-01-01T00:00 2000-01-01T12:12:12 +2 \N \N \N \N +3 \N \N \N \N + +-- !3 -- +\N + +-- !3 -- +\N + +-- !4 -- +\N + +-- !5 -- +1 \N \N \N \N +2 \N \N \N \N +3 \N \N \N \N + diff --git a/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy index 924938c136e3a0..327fd8ef7662cf 100644 --- a/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy +++ b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy @@ -19,14 +19,14 @@ suite("test_cast_datetime") { sql "drop table if exists casttbl" sql """CREATE TABLE casttbl ( + a int null, mydate date NULL, mydatev2 DATEV2 null, mydatetime datetime null, mydatetimev2 datetimev2 null ) ENGINE=OLAP - DUPLICATE KEY(`mydate`) COMMENT 'OLAP' - DISTRIBUTED BY HASH(`mydate`) BUCKETS 1 + DISTRIBUTED BY HASH(`a`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "in_memory" = "false", @@ -36,11 +36,20 @@ suite("test_cast_datetime") { ); """ - sql "insert into casttbl values ('2000-01-01', '2000-01-01 12:12:12', '2000-01-01', '2000-01-01 12:12:12');" + sql "insert into casttbl values (1, '2000-01-01', '2000-01-01 12:12:12', '2000-01-01', '2000-01-01 12:12:12');" sql "set enable_nereids_planner=true;" + sql "set enable_fallback_to_original_planner=false" //when BE storage support 'Date < Date', we should remove this case //currently, if we rewrite expr to CAST(mydatetime AS DATE) < date '2019-06-01', BE returns 0 tuple. qt_1 "select count(1) from casttbl where CAST(CAST(mydatetime AS DATE) AS DATETIME) < date '2019-06-01';" -} \ No newline at end of file + + sql "insert into casttbl values(2, '', '', '', ''), (3, '2020', '2020', '2020', '2020')" + qt_2 "select * from casttbl" + + qt_3 "select '' > date '2019-06-01'" + qt_3 "select '' > date_sub('2019-06-01', -10)" + qt_4 "select '' > cast('2019-06-01 00:00:00' as datetime)" + qt_5 "select a, '' = mydate, '' = mydatev2, '' = mydatetime, '' = mydatetimev2 from casttbl" +} From 2c7fccf0f5a5a963f2c9a5ec911dd386f726fb6c Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 6 Mar 2025 19:14:29 +0800 Subject: [PATCH 4/5] add test --- .../nereids_syntax_p0/test_cast_datetime.out | 18 ++++++++++++------ .../test_cast_datetime.groovy | 10 ++++++---- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/regression-test/data/nereids_syntax_p0/test_cast_datetime.out b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out index a0d754fe52dcf8..30a569f6b66a4e 100644 --- a/regression-test/data/nereids_syntax_p0/test_cast_datetime.out +++ b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out @@ -8,16 +8,22 @@ 3 \N \N \N \N -- !3 -- +1 \N \N \N \N +2 \N \N \N \N +3 \N \N \N \N + +-- !4 -- \N --- !3 -- +-- !5 -- \N --- !4 -- +-- !7 -- \N --- !5 -- -1 \N \N \N \N -2 \N \N \N \N -3 \N \N \N \N +-- !8 -- +\N + +-- !9 -- +\N diff --git a/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy index 327fd8ef7662cf..74a4b6de975fb6 100644 --- a/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy +++ b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy @@ -47,9 +47,11 @@ suite("test_cast_datetime") { sql "insert into casttbl values(2, '', '', '', ''), (3, '2020', '2020', '2020', '2020')" qt_2 "select * from casttbl" + qt_3 "select a, '' = mydate, '' = mydatev2, '' = mydatetime, '' = mydatetimev2 from casttbl" - qt_3 "select '' > date '2019-06-01'" - qt_3 "select '' > date_sub('2019-06-01', -10)" - qt_4 "select '' > cast('2019-06-01 00:00:00' as datetime)" - qt_5 "select a, '' = mydate, '' = mydatev2, '' = mydatetime, '' = mydatetimev2 from casttbl" + qt_4 "select '' > date '2019-06-01'" + qt_5 "select '' > date_sub('2019-06-01', -10)" + qt_7 "select '' > cast('2019-06-01 00:00:00' as datetime)" + qt_8 "select date_add('', 10)" + qt_9 "select date_add('2020', 10)" } From 0df43dcd1087ed2a65160d5008aadba5fe7295d2 Mon Sep 17 00:00:00 2001 From: yujun Date: Mon, 10 Mar 2025 19:33:19 +0800 Subject: [PATCH 5/5] fix test --- .../apache/doris/nereids/stats/ExpressionEstimation.java | 3 ++- .../nereids/trees/expressions/literal/DateLiteral.java | 2 +- .../trees/expressions/literal/DateLiteralTest.java | 8 +++++--- .../data/nereids_syntax_p0/test_cast_datetime.out | 6 ++++++ .../suites/nereids_syntax_p0/test_cast_datetime.groovy | 2 ++ 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java index 780c5922c6acdf..39656b6636bcb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java @@ -100,6 +100,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.collections.CollectionUtils; +import java.time.DateTimeException; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -222,7 +223,7 @@ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType long max = dateMaxLiteral.getValue(); builder.setMaxValue(max); builder.setMaxExpr(dateMaxLiteral.toLegacyLiteral()); - } catch (AnalysisException e) { + } catch (DateTimeException | AnalysisException e) { convertSuccess = false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index b8eba0b2499cc0..54a74944a2229a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -342,7 +342,7 @@ static Result normalize(String s) { // if Year is not present, throw exception if (!dateTime.isSupported(ChronoField.YEAR)) { return Result.err( - () -> new AnalysisException("date/datetime literal [" + originalString + "] is invalid") + () -> new DateTimeException("date/datetime literal [" + originalString + "] is invalid") ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java index 8db1c9446d0c6d..1919238ac7ea41 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.time.DateTimeException; import java.util.function.Consumer; class DateLiteralTest { @@ -67,7 +68,8 @@ void testDate() { new DateLiteral("2022-1-1"); new DateLiteral("20220101"); - Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("-01-01")); + Assertions.assertThrows(DateTimeException.class, () -> new DateLiteral("-01-01")); + Assertions.assertThrows(DateTimeException.class, () -> new DateLiteral("01-01")); } @Test @@ -128,8 +130,8 @@ void testIrregularDate() { @Test void testWrongPunctuationDate() { - Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020€02€01")); - Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020【02】01")); + Assertions.assertThrows(DateTimeException.class, () -> new DateTimeV2Literal("2020€02€01")); + Assertions.assertThrows(DateTimeException.class, () -> new DateTimeV2Literal("2020【02】01")); } @Test diff --git a/regression-test/data/nereids_syntax_p0/test_cast_datetime.out b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out index 30a569f6b66a4e..a2a359c2c43376 100644 --- a/regression-test/data/nereids_syntax_p0/test_cast_datetime.out +++ b/regression-test/data/nereids_syntax_p0/test_cast_datetime.out @@ -27,3 +27,9 @@ -- !9 -- \N +-- !10 -- +\N + +-- !12 -- +\N + diff --git a/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy index 74a4b6de975fb6..5e00ecd07895ce 100644 --- a/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy +++ b/regression-test/suites/nereids_syntax_p0/test_cast_datetime.groovy @@ -54,4 +54,6 @@ suite("test_cast_datetime") { qt_7 "select '' > cast('2019-06-01 00:00:00' as datetime)" qt_8 "select date_add('', 10)" qt_9 "select date_add('2020', 10)" + qt_10 "select date_add('08-09', 10)" + qt_12 "select date_add('abcd', 10)" }