From 50813fcd2d4e02bf5eca9cd0b236814be8d1d50a Mon Sep 17 00:00:00 2001 From: starocean999 <12095047@qq.com> Date: Fri, 16 Aug 2024 16:03:37 +0800 Subject: [PATCH 1/2] [fix](nereids)prevent null pointer exception if datetime value overflows --- .../rules/SimplifyComparisonPredicate.java | 5 +-- .../functions/executable/TimeRoundSeries.java | 35 ++++++++++-------- .../expressions/literal/DateLiteral.java | 2 +- .../datatype/test_datetime_overflow.groovy | 36 +++++++++++++++++++ 4 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 regression-test/suites/nereids_p0/datatype/test_datetime_overflow.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java index d26b5a53036897..522a539e4a7913 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java @@ -346,9 +346,10 @@ private static boolean cannotAdjust(DateTimeLiteral l, ComparisonPredicate cp) { private static Expression migrateToDateV2(DateTimeLiteral l, AdjustType type) { DateV2Literal d = new DateV2Literal(l.getYear(), l.getMonth(), l.getDay()); if (type == AdjustType.UPPER && (l.getHour() != 0 || l.getMinute() != 0 || l.getSecond() != 0)) { - d = ((DateV2Literal) d.plusDays(1)); + return d.plusDays(1); + } else { + return d; } - return d; } private static Expression migrateToDate(DateV2Literal l) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java index a9337f05370ce6..8189ea93896f58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java @@ -104,22 +104,27 @@ private static LocalDateTime getDateCeilOrFloor(DATE tag, LocalDateTime date, in if (getCeil) { step = step + (deltaInsidePeriod == 0 ? 0 : period); } - switch (tag) { - case YEAR: - return ((DateTimeLiteral) start.plusYears(step)).toJavaDateType(); - case MONTH: - return ((DateTimeLiteral) start.plusMonths(step)).toJavaDateType(); - case DAY: - return ((DateTimeLiteral) start.plusDays(step)).toJavaDateType(); - case HOUR: - return ((DateTimeLiteral) start.plusHours(step)).toJavaDateType(); - case MINUTE: - return ((DateTimeLiteral) start.plusMinutes(step)).toJavaDateType(); - case SECOND: - return ((DateTimeLiteral) start.plusSeconds(step)).toJavaDateType(); - default: - break; + try { + switch (tag) { + case YEAR: + return ((DateTimeLiteral) start.plusYears(step)).toJavaDateType(); + case MONTH: + return ((DateTimeLiteral) start.plusMonths(step)).toJavaDateType(); + case DAY: + return ((DateTimeLiteral) start.plusDays(step)).toJavaDateType(); + case HOUR: + return ((DateTimeLiteral) start.plusHours(step)).toJavaDateType(); + case MINUTE: + return ((DateTimeLiteral) start.plusMinutes(step)).toJavaDateType(); + case SECOND: + return ((DateTimeLiteral) start.plusSeconds(step)).toJavaDateType(); + default: + break; + } + } catch (Exception ex) { + // do nothing } + return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 6171707b87f3c7..9b25904b898f0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -336,7 +336,7 @@ protected boolean checkDate() { } protected static boolean isDateOutOfRange(LocalDateTime dateTime) { - return dateTime.isBefore(START_OF_A_DAY) || dateTime.isAfter(END_OF_A_DAY); + return dateTime.isBefore(START_OF_A_DAY) || dateTime.isAfter(END_OF_A_DAY) || dateTime == null; } private boolean checkDatetime(TemporalAccessor dateTime) { diff --git a/regression-test/suites/nereids_p0/datatype/test_datetime_overflow.groovy b/regression-test/suites/nereids_p0/datatype/test_datetime_overflow.groovy new file mode 100644 index 00000000000000..47109b26634f6e --- /dev/null +++ b/regression-test/suites/nereids_p0/datatype/test_datetime_overflow.groovy @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_datetime_overflow") { + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql """drop table if exists datetime_overflow_t""" + sql """CREATE TABLE datetime_overflow_t ( + `id` bigint NULL, + `c` datetime NULL, + `d` date NULL, + INDEX idx_c (`c`) USING INVERTED + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY RANDOM BUCKETS AUTO + PROPERTIES ( + "replication_num" = "1" + );""" + + sql """select * from datetime_overflow_t where d between "9999-12-31 00:00:01" and "9999-12-31 10:00:01";""" + sql """select * from datetime_overflow_t where d > "9999-12-31 00:00:01";""" +} From b27f49841ea16081e543182d82fa01ed73e5b4b8 Mon Sep 17 00:00:00 2001 From: starocean999 <12095047@qq.com> Date: Tue, 20 Aug 2024 16:58:10 +0800 Subject: [PATCH 2/2] modified based on comments --- .../functions/executable/TimeRoundSeries.java | 48 +++++++++++-------- .../expressions/literal/DateLiteral.java | 2 +- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java index 8189ea93896f58..3a98ee6252791a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java @@ -104,28 +104,34 @@ private static LocalDateTime getDateCeilOrFloor(DATE tag, LocalDateTime date, in if (getCeil) { step = step + (deltaInsidePeriod == 0 ? 0 : period); } - try { - switch (tag) { - case YEAR: - return ((DateTimeLiteral) start.plusYears(step)).toJavaDateType(); - case MONTH: - return ((DateTimeLiteral) start.plusMonths(step)).toJavaDateType(); - case DAY: - return ((DateTimeLiteral) start.plusDays(step)).toJavaDateType(); - case HOUR: - return ((DateTimeLiteral) start.plusHours(step)).toJavaDateType(); - case MINUTE: - return ((DateTimeLiteral) start.plusMinutes(step)).toJavaDateType(); - case SECOND: - return ((DateTimeLiteral) start.plusSeconds(step)).toJavaDateType(); - default: - break; - } - } catch (Exception ex) { - // do nothing + Expression result = null; + switch (tag) { + case YEAR: + result = start.plusYears(step); + break; + case MONTH: + result = start.plusMonths(step); + break; + case DAY: + result = start.plusDays(step); + break; + case HOUR: + result = start.plusHours(step); + break; + case MINUTE: + result = start.plusMinutes(step); + break; + case SECOND: + result = start.plusSeconds(step); + break; + default: + break; + } + if (result != null && result instanceof DateTimeLiteral) { + return ((DateTimeLiteral) result).toJavaDateType(); + } else { + return null; } - - return null; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 9b25904b898f0f..f1946463aa71eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -336,7 +336,7 @@ protected boolean checkDate() { } protected static boolean isDateOutOfRange(LocalDateTime dateTime) { - return dateTime.isBefore(START_OF_A_DAY) || dateTime.isAfter(END_OF_A_DAY) || dateTime == null; + return dateTime == null || dateTime.isBefore(START_OF_A_DAY) || dateTime.isAfter(END_OF_A_DAY); } private boolean checkDatetime(TemporalAccessor dateTime) {