From d098699e6694c4cbbd1ed02fe6b62112a2451ad8 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Mon, 18 Dec 2023 19:31:04 +0800 Subject: [PATCH] [feature](Nereids) support datev1 and datetimev1 --- .../org/apache/doris/nereids/DorisParser.g4 | 2 +- .../apache/doris/nereids/types/DataType.java | 14 +++ .../doris/nereids/types/DateTimeType.java | 10 ++- .../apache/doris/nereids/types/DateType.java | 10 ++- .../nereids/parser/NereidsParserTest.java | 25 +++++- .../data/nereids_p0/datatype/test_datev1.out | 52 +++++++++++ .../nereids_p0/datatype/test_datev1.groovy | 89 +++++++++++++++++++ 7 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/nereids_p0/datatype/test_datev1.out create mode 100644 regression-test/suites/nereids_p0/datatype/test_datev1.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index be683f5ee4b400..77e8188131f908 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -753,7 +753,7 @@ specifiedPartition constant : NULL #nullLiteral - | type=(DATE | DATEV1 | DATEV2 | TIMESTAMP) STRING_LITERAL #typeConstructor + | type=(DATE | DATEV1 | DATEV2 | TIMESTAMP) STRING_LITERAL #typeConstructor | number #numericLiteral | booleanValue #booleanLiteral | STRING_LITERAL #stringLiteral diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index 8f887326216fc4..b22c6093f9a75c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -216,6 +216,9 @@ public static DataType convertPrimitiveFromStrings(List types, boolean u case "date": dataType = DateType.INSTANCE; break; + case "datev1": + dataType = DateType.NOT_CONVERSION; + break; case "datev2": dataType = DateV2Type.INSTANCE; break; @@ -234,6 +237,17 @@ public static DataType convertPrimitiveFromStrings(List types, boolean u throw new AnalysisException("Nereids do not support type: " + type); } break; + case "datetimev1": + switch (types.size()) { + case 1: + dataType = DateTimeType.NOT_CONVERSION; + break; + case 2: + throw new AnalysisException("Nereids do not support datetimev1 type with precision"); + default: + throw new AnalysisException("Nereids do not support type: " + type); + } + break; case "datetimev2": switch (types.size()) { case 1: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java index 250a69cbd37ce4..8a0250d7b447c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java @@ -31,15 +31,23 @@ public class DateTimeType extends DateLikeType { public static final DateTimeType INSTANCE = new DateTimeType(); + public static final DateTimeType NOT_CONVERSION = new DateTimeType(false); private static final int WIDTH = 16; + private final boolean shouldConversion; + private DateTimeType() { + this.shouldConversion = true; + } + + private DateTimeType(boolean shouldConversion) { + this.shouldConversion = shouldConversion; } @Override public DataType conversion() { - if (Config.enable_date_conversion) { + if (Config.enable_date_conversion && shouldConversion) { return DateTimeV2Type.SYSTEM_DEFAULT; } return this; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java index 69572895ddddfa..854c50d7f1ec4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java @@ -31,15 +31,23 @@ public class DateType extends DateLikeType { public static final DateType INSTANCE = new DateType(); + public static final DateType NOT_CONVERSION = new DateType(false); private static final int WIDTH = 16; + private final boolean shouldConversion; + private DateType() { + this.shouldConversion = true; + } + + private DateType(boolean shouldConversion) { + this.shouldConversion = shouldConversion; } @Override public DataType conversion() { - if (Config.enable_date_conversion) { + if (Config.enable_date_conversion && shouldConversion) { return DateV2Type.INSTANCE; } return this; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 9d0ca7176a436f..bae4d7459ce9f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -22,6 +22,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundResultSink; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.expressions.Cast; @@ -37,6 +38,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.qe.SessionVariable; @@ -305,7 +308,27 @@ public void testParseDecimal() { .stream() .mapToLong(e -> e.>collect(DecimalLiteral.class::isInstance).size()) .sum(); - Assertions.assertEquals(doubleCount, Config.enable_decimal_conversion ? 0 : 1); + Assertions.assertEquals(Config.enable_decimal_conversion ? 0 : 1, doubleCount); + } + + @Test + public void testDatev1() { + String dv1 = "SELECT CAST('2023-12-18' AS DATEV1)"; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = (LogicalPlan) nereidsParser.parseSingle(dv1).child(0); + Assertions.assertEquals(DateType.INSTANCE, logicalPlan.getExpressions().get(0).getDataType()); + } + + @Test + public void testDatetimev1() { + String dtv1 = "SELECT CAST('2023-12-18' AS DATETIMEV1)"; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = (LogicalPlan) nereidsParser.parseSingle(dtv1).child(0); + Assertions.assertEquals(DateTimeType.INSTANCE, logicalPlan.getExpressions().get(0).getDataType()); + + String wrongDtv1 = "SELECT CAST('2023-12-18' AS DATETIMEV1(2))"; + Assertions.assertThrows(AnalysisException.class, () -> nereidsParser.parseSingle(wrongDtv1).child(0)); + } @Test diff --git a/regression-test/data/nereids_p0/datatype/test_datev1.out b/regression-test/data/nereids_p0/datatype/test_datev1.out new file mode 100644 index 00000000000000..f81b8f1a698afd --- /dev/null +++ b/regression-test/data/nereids_p0/datatype/test_datev1.out @@ -0,0 +1,52 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +2016-11-04 + +-- !sql2 -- +2016-11-04 + +-- !sql1 -- +1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + +-- !sql2 -- +1 2000-01-01 2000-01-01T11:11:11 1 2000-01-01 2000-01-01T11:11:11 +2 2000-02-02 2000-02-02T11:11:11 2 2000-02-02 2000-02-02T11:11:11 +3 2000-03-02 2000-03-02T11:11:11 3 2000-03-02 2000-03-02T11:11:11 + diff --git a/regression-test/suites/nereids_p0/datatype/test_datev1.groovy b/regression-test/suites/nereids_p0/datatype/test_datev1.groovy new file mode 100644 index 00000000000000..3ea4d2561d5878 --- /dev/null +++ b/regression-test/suites/nereids_p0/datatype/test_datev1.groovy @@ -0,0 +1,89 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_datev1") { + def tbName = "test_datev1_exprs" + + sql """set enable_nereids_planner=true""" + sql """set enable_fallback_to_original_planner=false""" + sql """set enable_nereids_dml=true""" + + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + create table ${tbName}(k1 datetimev1, k2 int) distributed by hash(k1) buckets 1 properties("replication_num" = "1"); + """ + sql """ insert into ${tbName} values("2016-11-04 00:00:01", 1); """ + + qt_sql1 """ select dt + from + ( + select cast(k1 as datev1) as dt + from ${tbName} + ) r; """ + qt_sql2 """ select dt + from + ( + select cast(k1 as date) as dt + from ${tbName} + ) r; """ + sql "DROP TABLE ${tbName}" + + tbName = "test_datev1_runtime_filter" + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName} ( + c0 int, + c2 datev1, + c3 datetimev1 + ) + DISTRIBUTED BY HASH(c0) BUCKETS 5 properties("replication_num" = "1"); + """ + sql "insert into ${tbName} values(1, '2000-01-01', '2000-01-01 11:11:11')" + sql "insert into ${tbName} values(2, '2000-02-02', '2000-02-02 11:11:11')" + sql "insert into ${tbName} values(3, '2000-03-02', '2000-03-02 11:11:11')" + + qt_sql1 "select * from ${tbName} ORDER BY c2" + + sql " set runtime_filter_type = 1; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql " set runtime_filter_type = 2; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql " set runtime_filter_type = 4; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql " set runtime_filter_type = 8; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql " set runtime_filter_wait_time_ms = 0; " + + sql " set runtime_filter_type = 1; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql " set runtime_filter_type = 2; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql " set runtime_filter_type = 4; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql " set runtime_filter_type = 8; " + qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c3 = b.c3 ORDER BY a.c2" + + sql "DROP TABLE ${tbName}" +}