apache · jackwener · Feb 2, 2024 · Feb 2, 2024
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
@@ -183,9 +183,9 @@ static String normalize(String s) {
         // normalize leading 0 for date and time
         // date and time contains 6 number part at most, so we just need normal 6 number part
         int partNumber = 0;
-        while (i < s.length()) {
+        while (i < s.length() && partNumber < 6) {
             char c = s.charAt(i);
-            if (Character.isDigit(c) && partNumber < 6) {
+            if (Character.isDigit(c)) {
                 // find consecutive digit
                 int j = i + 1;
                 while (j < s.length() && Character.isDigit(s.charAt(j))) {
@@ -234,11 +234,14 @@ static String normalize(String s) {
         }
 
         // parse MicroSecond
+        // Keep up to 7 digits at most, 7th digit is use for overflow.
         if (partNumber == 6 && i < s.length() && s.charAt(i) == '.') {
             sb.append(s.charAt(i));
             i += 1;
             while (i < s.length() && Character.isDigit(s.charAt(i))) {
-                sb.append(s.charAt(i));
+                if (i - 19 <= 7) {
+                    sb.append(s.charAt(i));
+                }
                 i += 1;
             }
         }
@@ -266,11 +269,12 @@ protected static TemporalAccessor parse(String s) {
         try {
             TemporalAccessor dateTime;
 
-            // remove suffix ' '
+            // remove suffix/prefix ' '
             s = s.trim();
             // parse condition without '-' and ':'
             boolean containsPunctuation = false;
-            for (int i = 0; i < s.length(); i++) {
+            int len = Math.min(s.length(), 11);
+            for (int i = 0; i < len; i++) {
                 if (isPunctuation(s.charAt(i))) {
                     containsPunctuation = true;
                     break;

diff --git a/...ore/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/...ore/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
@@ -136,7 +136,6 @@ protected void init(String s) throws AnalysisException {
         hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY);
         minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR);
         second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE);
-        microSecond = DateUtils.getOrDefault(temporal, ChronoField.MICRO_OF_SECOND);
 
         ZoneId zoneId = temporal.query(TemporalQueries.zone());
         if (zoneId != null) {
@@ -153,6 +152,21 @@ protected void init(String s) throws AnalysisException {
             }
         }
 
+        microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L;
+        // Microseconds have 7 digits.
+        long sevenDigit = microSecond % 10;
+        microSecond = microSecond / 10;
+        if (sevenDigit >= 5 && this instanceof DateTimeV2Literal) {
+            DateTimeV2Literal result = (DateTimeV2Literal) ((DateTimeV2Literal) this).plusMicroSeconds(1);
+            this.second = result.second;
+            this.minute = result.minute;
+            this.hour = result.hour;
+            this.day = result.day;
+            this.month = result.month;
+            this.year = result.year;
+            this.microSecond = result.microSecond;
+        }
+
         if (checkRange() || checkDate()) {
             throw new AnalysisException("datetime literal [" + s + "] is out of range");
         }

diff --git a/...e/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/...e/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
@@ -66,7 +66,7 @@ private void roundMicroSecond(int scale) {
 
         if (this.microSecond >= 1000000) {
             LocalDateTime localDateTime = DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND,
-                            getStringValue()).plusSeconds(1);
+                    getStringValue()).plusSeconds(1);
             this.year = localDateTime.getYear();
             this.month = localDateTime.getMonthValue();
             this.day = localDateTime.getDayOfMonth();
@@ -77,6 +77,11 @@ private void roundMicroSecond(int scale) {
         }
     }
 
+    public String getFullMicroSecondValue() {
+        return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d",
+                year, month, day, hour, minute, second, microSecond);
+    }
+
     @Override
     public DateTimeV2Type getDataType() throws UnboundException {
         return (DateTimeV2Type) super.getDataType();
@@ -165,7 +170,7 @@ public Expression plusSeconds(long seconds) {
 
     public Expression plusMicroSeconds(long microSeconds) {
         return fromJavaDateType(
-                DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
+                DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getFullMicroSecondValue())
                         .plusNanos(microSeconds * 1000L), getDataType().getScale());
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
@@ -84,10 +84,13 @@ public static DateTimeV2Type forType(DataType dataType) {
 
     /**
      * return proper type of datetimev2 for String
-     * may be we need to check for validity?
+     * maybe we need to check for validity?
      */
     public static DateTimeV2Type forTypeFromString(String s) {
         int scale = DateTimeLiteral.determineScale(s);
+        if (scale > MAX_SCALE) {
+            scale = MAX_SCALE;
+        }
         return DateTimeV2Type.of(scale);
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
@@ -56,16 +56,17 @@ public class DateTimeFormatterUtils {
             .appendValue(ChronoField.HOUR_OF_DAY, 2)
             .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2)
             .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+            // microsecond maxWidth is 7, we may need 7th digit to judge overflow
             .appendOptional(new DateTimeFormatterBuilder()
-                    .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter())
+                    .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter())
             .toFormatter().withResolverStyle(ResolverStyle.STRICT);
     // Time without delimiter: HHmmss[.microsecond]
     private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder()
             .appendValue(ChronoField.HOUR_OF_DAY, 2)
             .appendValue(ChronoField.MINUTE_OF_HOUR, 2)
             .appendValue(ChronoField.SECOND_OF_MINUTE, 2)
             .appendOptional(new DateTimeFormatterBuilder()
-                    .appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter())
+                    .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter())
             .toFormatter().withResolverStyle(ResolverStyle.STRICT);
     // yyyymmdd
     private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder()

diff --git a/...a/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java b/...a/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java
@@ -116,15 +116,6 @@ void dateLikeOverflow() {
                         )
                 );
 
-        PlanChecker.from(connectContext)
-                .analyze("select CONVERT('2021-01-30 00:00:00.0000001', DATETIME(6))")
-                .rewrite()
-                .matches(
-                        logicalResultSink(
-                                logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).equals(new NullLiteral(DateTimeV2Type.of(6))))
-                        )
-                );
-
         PlanChecker.from(connectContext)
                 .analyze("select CONVERT_TZ('2021-01-32 00:00:00', '+08:00', 'America/London') = '2021-01-30'")
                 .rewrite()

diff --git a/...ore/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/...ore/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java
@@ -28,14 +28,6 @@
 class DateLiteralTest {
     @Test
     void reject() {
-        // TODO: reject them.
-        // Now parse them as date + offset.
-        // PG parse them as date + offset, MySQL parse them as date + time (rubbish behavior!)
-        // So strange! reject these strange case.
-        // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-01"));
-        // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-1"));
-        // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01+01"));
-        // Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01+1"));
         Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 01:00:00.000000"));
         Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:01:00.000000"));
         Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:00:01.000000"));
@@ -212,7 +204,7 @@ void testPoint() {
         new DateLiteral("2020.02.01 00.00.00");
         new DateTimeV2Literal("2020.02.01 00.00.00.1");
         new DateTimeV2Literal("2020.02.01 00.00.00.000001");
-        Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020.02.01 00.00.00.0000001"));
+        new DateTimeV2Literal("2020.02.01 00.00.00.0000001");
     }
 
     @Test

diff --git a/...src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/...src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java
@@ -41,13 +41,14 @@ void testBasic() {
             Assertions.assertEquals(1, datetime.day);
             Assertions.assertEquals(1, datetime.hour);
             Assertions.assertEquals(1, datetime.minute);
-            Assertions.assertEquals(1, datetime.second);
+            Assertions.assertEquals(2, datetime.second);
         };
 
-        assertFunc.accept(new DateTimeV2Literal("20220801010101"));
-        assertFunc.accept(new DateTimeV2Literal("20220801T010101"));
-        assertFunc.accept(new DateTimeV2Literal("220801010101"));
-        assertFunc.accept(new DateTimeV2Literal("220801T010101"));
+        assertFunc.accept(new DateTimeV2Literal("20220801010102"));
+        assertFunc.accept(new DateTimeV2Literal("20220801T010102"));
+        assertFunc.accept(new DateTimeV2Literal("220801010102"));
+        assertFunc.accept(new DateTimeV2Literal("220801T010102"));
+        assertFunc.accept(new DateTimeV2Literal("20220801010101.9999999"));
     }
 
     @Test
@@ -386,6 +387,29 @@ void testIrregularDateTimeHourMinuteSecondMicrosecond() {
         // Testing with microsecond of length 6
         new DateTimeV2Literal("2016-07-02 01:01:01.123456");
         new DateTimeV2Literal("2016-7-02 01:01:01.123456");
+
+        // Testing with microsecond of length 7
+        DateTimeV2Literal literal = new DateTimeV2Literal("2016-07-02 01:01:01.12345678");
+        Assertions.assertEquals(123457, literal.microSecond);
+
+        literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444444");
+        Assertions.assertEquals(444444, literal.microSecond);
+
+        literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444445");
+        Assertions.assertEquals(444444, literal.microSecond);
+
+        literal = new DateTimeV2Literal("2016-07-02 01:01:01.4444445");
+        Assertions.assertEquals(444445, literal.microSecond);
+
+        literal = new DateTimeV2Literal("2016-07-02 01:01:01.9999995");
+        Assertions.assertEquals(0, literal.microSecond);
+        Assertions.assertEquals(2, literal.second);
+
+        literal = new DateTimeV2Literal("2021-01-01 23:59:59.9999995");
+        Assertions.assertEquals(0, literal.microSecond);
+        Assertions.assertEquals(0, literal.second);
+        Assertions.assertEquals(0, literal.minute);
+        Assertions.assertEquals(0, literal.hour);
     }
 
     @Test

diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java
@@ -70,18 +70,18 @@ void testBasicDateTimeFormatter() {
         assertDatePart(dateTime);
         dateTime = formatter.parse("20200219T010101.1");
         assertDatePart(dateTime);
+        dateTime = formatter.parse("20200219T010101.0000001");
+        assertDatePart(dateTime);
 
         // failed case
         DateTimeFormatter withT = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER;
         Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219 010101"));
         Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101."));
         Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101.0000001"));
         Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101."));
-        Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101.0000001"));
         DateTimeFormatter withoutT = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T;
         Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219 010101"));
         Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101."));
-        Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101.0000001"));
         Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101."));
         Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101.0000001"));
     }

diff --git a/regression-test/data/nereids_syntax_p0/test_literal.out b/regression-test/data/nereids_syntax_p0/test_literal.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !date1 --
+2016-07-03
+
+-- !date2 --
+2016-07-03
+
+-- !datetime1 --
+2016-07-02T23:59:59.990
+
+-- !datetime2 --
+2016-07-03T00:00
+
+-- !datetime3 --
+2016-07-03T00:00
+
+-- !datetime4 --
+2020-02-19T01:01:01
+
+-- !datetime5 --
+2021-01-30T00:00
+
diff --git a/regression-test/suites/nereids_syntax_p0/test_literal.groovy b/regression-test/suites/nereids_syntax_p0/test_literal.groovy
@@ -24,4 +24,32 @@ suite("test_literal") {
     sql """
         select {}, [], [[[null]], [[1, 2, 3]]], {1:[null], 3:[3]};
     """
+
+    qt_date1 """
+        select cast('20160702235959.9999999' as date);
+    """
+
+    qt_date2 """
+        select cast('2016-07-02 23:59:59.9999999' as date);
+    """
+
+    qt_datetime1 """
+        select timestamp'2016-07-02 23:59:59.99';
+    """
+
+    qt_datetime2 """
+        select cast('20160702235959.9999999' as datetime);
+    """
+
+    qt_datetime3 """
+        select cast('2016-07-02 23:59:59.9999999' as datetime);
+    """
+
+    qt_datetime4 """
+        select timestamp'20200219010101.0000001';
+    """
+
+    qt_datetime5 """
+        select CONVERT('2021-01-30 00:00:00.0000001', DATETIME(6))
+    """
 }