From 2851aa678ed0f760c31ba6bf1986cf3d283454ff Mon Sep 17 00:00:00 2001 From: liziyan Date: Sat, 23 Mar 2024 11:06:43 +0800 Subject: [PATCH 1/4] [fix] (query) fix date function rewrite on datetimev1 column bug (cherry picked from commit 9ae2e4bb78bfeae75ddd4b0b825bb9a343143ec2) --- .../doris/nereids/trees/expressions/literal/DateLiteral.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index b4f5869e46d0dd..c6e19bb96cbc27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -428,10 +428,10 @@ public DateTimeLiteral toBeginOfTheDay() { /** * 2020-01-01 - * @return 2020-01-01 24:00:00 + * @return 2020-01-01 23:59:59 */ public DateTimeLiteral toEndOfTheDay() { - return new DateTimeLiteral(year, month, day, 24, 0, 0); + return new DateTimeLiteral(year, month, day, 23, 59, 59); } /** From 5ff4add4fdb30adea123c0dba6f0f1323c176bca Mon Sep 17 00:00:00 2001 From: liziyan Date: Thu, 21 Mar 2024 01:31:24 +0800 Subject: [PATCH 2/4] [test](datetime) add datetimev1 column comparison regression test (cherry picked from commit 6df4c836cd3527383281a557501c064b119e1463) --- .../datetimev1/test_datetimev1_compare.groovy | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 regression-test/suites/datatype_p0/datetimev1/test_datetimev1_compare.groovy diff --git a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_compare.groovy b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_compare.groovy new file mode 100644 index 00000000000000..387c72798248db --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_compare.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_datetimev1_compare", "nonConcurrent") { + + /// legacy date/datetime format need to run in non concurrent mode. + sql """ + admin set frontend config("enable_date_conversion" = "false"); + """ + + def table_dup = "test_datetimev1_compare_dup_tbl" // duplicate key + + sql "drop table if exists ${table_dup};" + + sql """ + CREATE TABLE IF NOT EXISTS `${table_dup}` ( + `date_key1` datetimev1 NULL COMMENT "", + `date_key2` datetimev1 NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`date_key1`, `date_key2`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`date_key1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ) + """ + + def insert_data = { table_name -> + sql """ + insert into ${table_name} + values + ('2010-01-01 19:29:39', '2012-01-02 19:29:39'), + ('2010-01-02 18:28:38', '2012-02-02 19:29:39'), + ('2010-01-03 23:37:47', '2012-03-02 19:29:39'), + ('2010-01-04 21:42:43', '2012-04-02'), + ('2010-01-05 17:58:59', '2012-05-02'), + ('2010-01-06 21:02:03', '2012-06-02 00:00:00'), + ('2010-01-07 09:29:39', '2012-07-02 00:00:00'), + ('2010-01-08 11:33:55', '2012-08-02 00:00:00'), + ('2010-01-09 14:43:42', '2012-09-02 00:00:00'); + """ + } + + insert_data(table_dup) + + def run_compare_test = { table_name, col, col_value -> + def query1 = """ + select + * + from + `${table_name}` + where + `${col}` < ${col_value} + order by 1, 2, 3, 4; + """ + quickTest("compare_test_datetime_lt_${table_name}", query1) + + def query2 = """ + select + * + from + `${table_name}` + where + `${col}` <= ${col_value} + order by 1, 2, 3, 4; + """ + quickTest("compare_test_datetime_lte_${table_name}", query2) + + def query3 = """ + select + * + from + `${table_name}` + where + date(`${col}`) < ${col_value} + order by 1, 2, 3, 4; + """ + quickTest("compare_test_datetime_lt_${table_name}", query3) + + def query4 = """ + select + * + from + `${table_name}` + where + date(`${col}`) <= ${col_value} + order by 1, 2, 3, 4; + """ + quickTest("compare_test_datetime_lte_${table_name}", query4) + } + + run_compare_test(table_dup, "date_key1", "'2010-01-05 17:58:59'"); + run_compare_test(table_dup, "date_key2", "'2012-05-02 00:00:00'"); + run_compare_test(table_dup, "date_key1", "'2010-01-05 17:58:59'"); + run_compare_test(table_dup, "date_key2", "'2012-05-02 00:00:00'"); + + sql """ + admin set frontend config("enable_date_conversion" = "true"); + """ +} From e62da2e4a3d63485c79835f9b635fc02915c3d0a Mon Sep 17 00:00:00 2001 From: liziyan Date: Thu, 21 Mar 2024 02:25:34 +0800 Subject: [PATCH 3/4] [test](datetime) add datetimev1 column date function regression test (cherry picked from commit e58f840d46376ebd6808b71cc3ea385b23052c15) --- ...y => test_datetimev1_date_function.groovy} | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) rename regression-test/suites/datatype_p0/datetimev1/{test_datetimev1_compare.groovy => test_datetimev1_date_function.groovy} (64%) diff --git a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_compare.groovy b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy similarity index 64% rename from regression-test/suites/datatype_p0/datetimev1/test_datetimev1_compare.groovy rename to regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy index 387c72798248db..a17695e1940352 100644 --- a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_compare.groovy +++ b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy @@ -15,23 +15,22 @@ // specific language governing permissions and limitations // under the License. -suite("test_datetimev1_compare", "nonConcurrent") { +suite("test_datetimev1_date_function", "nonConcurrent") { /// legacy date/datetime format need to run in non concurrent mode. sql """ admin set frontend config("enable_date_conversion" = "false"); """ - def table_dup = "test_datetimev1_compare_dup_tbl" // duplicate key + def table_dup = "test_datetimev1_date_function_dup_tbl" // duplicate key sql "drop table if exists ${table_dup};" sql """ CREATE TABLE IF NOT EXISTS `${table_dup}` ( `date_key1` datetimev1 NULL COMMENT "", - `date_key2` datetimev1 NULL COMMENT "" ) ENGINE=OLAP - DUPLICATE KEY(`date_key1`, `date_key2`) + DUPLICATE KEY(`date_key1`) COMMENT "OLAP" DISTRIBUTED BY HASH(`date_key1`) BUCKETS 1 PROPERTIES ( @@ -45,15 +44,15 @@ suite("test_datetimev1_compare", "nonConcurrent") { sql """ insert into ${table_name} values - ('2010-01-01 19:29:39', '2012-01-02 19:29:39'), - ('2010-01-02 18:28:38', '2012-02-02 19:29:39'), - ('2010-01-03 23:37:47', '2012-03-02 19:29:39'), - ('2010-01-04 21:42:43', '2012-04-02'), - ('2010-01-05 17:58:59', '2012-05-02'), - ('2010-01-06 21:02:03', '2012-06-02 00:00:00'), - ('2010-01-07 09:29:39', '2012-07-02 00:00:00'), - ('2010-01-08 11:33:55', '2012-08-02 00:00:00'), - ('2010-01-09 14:43:42', '2012-09-02 00:00:00'); + ('2020-01-01 19:29:39'), + ('2020-01-02 18:28:38'), + ('2020-01-03 23:37:47'), + ('2020-01-04 21:42:43'), + ('2020-01-05 17:58:59'), + ('2020-01-06 21:02:03'), + ('2020-01-07 09:29:39'), + ('2020-01-08 11:33:55'), + ('2020-01-09 14:43:42'); """ } @@ -66,10 +65,10 @@ suite("test_datetimev1_compare", "nonConcurrent") { from `${table_name}` where - `${col}` < ${col_value} + date(`${col}`) > ${col_value} order by 1, 2, 3, 4; """ - quickTest("compare_test_datetime_lt_${table_name}", query1) + quickTest("date_function_test_datetimev1_gt_${table_name}", query1) def query2 = """ select @@ -77,10 +76,10 @@ suite("test_datetimev1_compare", "nonConcurrent") { from `${table_name}` where - `${col}` <= ${col_value} + date(`${col}`) >= ${col_value} order by 1, 2, 3, 4; """ - quickTest("compare_test_datetime_lte_${table_name}", query2) + quickTest("date_function_test_datetimev1_gte_${table_name}", query2) def query3 = """ select @@ -91,7 +90,7 @@ suite("test_datetimev1_compare", "nonConcurrent") { date(`${col}`) < ${col_value} order by 1, 2, 3, 4; """ - quickTest("compare_test_datetime_lt_${table_name}", query3) + quickTest("date_function_test_datetimev1_lt_${table_name}", query3) def query4 = """ select @@ -102,13 +101,13 @@ suite("test_datetimev1_compare", "nonConcurrent") { date(`${col}`) <= ${col_value} order by 1, 2, 3, 4; """ - quickTest("compare_test_datetime_lte_${table_name}", query4) + quickTest("date_function_test_datetimev1_lte_${table_name}", query4) } - run_compare_test(table_dup, "date_key1", "'2010-01-05 17:58:59'"); - run_compare_test(table_dup, "date_key2", "'2012-05-02 00:00:00'"); - run_compare_test(table_dup, "date_key1", "'2010-01-05 17:58:59'"); - run_compare_test(table_dup, "date_key2", "'2012-05-02 00:00:00'"); + run_compare_test(table_dup, "date_key1", "'2020-01-05 17:58:59'"); + run_compare_test(table_dup, "date_key1", "'2022-05-02 00:00:00'"); + run_compare_test(table_dup, "date_key1", "'2020-01-05 17:58:59'"); + run_compare_test(table_dup, "date_key1", "'2022-05-02 00:00:00'"); sql """ admin set frontend config("enable_date_conversion" = "true"); From a86844ee36782df308bdabc017f92adb9ce264ff Mon Sep 17 00:00:00 2001 From: liziyan Date: Thu, 21 Mar 2024 11:15:31 +0800 Subject: [PATCH 4/4] [test](datetime) opt datetimev1 date function regression test case and add test file (cherry picked from commit 912a6c0d9ed5f09b602a2d102011090864b530e2) --- .../test_datetimev1_date_function.out | 32 +++++++++++++++++ .../test_datetimev1_date_function.groovy | 35 ++++++++++++------- 2 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out diff --git a/regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out b/regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out new file mode 100644 index 00000000000000..81a89b2286f856 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev1/test_datetimev1_date_function.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !date_function_test_datetimev1_gt -- +2020-01-06T21:02:03 +2020-01-07T09:29:39 +2020-01-08T11:33:55 +2020-01-09T14:43:42 +2020-01-10T23:59:59 + +-- !date_function_test_datetimev1_gte -- +2020-01-05T17:58:59 +2020-01-06T21:02:03 +2020-01-07T09:29:39 +2020-01-08T11:33:55 +2020-01-09T14:43:42 +2020-01-10T23:59:59 + +-- !date_function_test_datetimev1_lt -- +2020-01-01T19:29:39 +2020-01-02T18:28:38 +2020-01-03T23:37:47 +2020-01-04T21:42:43 + +-- !date_function_test_datetimev1_lte -- +2020-01-01T19:29:39 +2020-01-02T18:28:38 +2020-01-03T23:37:47 +2020-01-04T21:42:43 +2020-01-05T17:58:59 + +-- !date_function_test_datetimev1_eq -- +2020-01-05T17:58:59 + diff --git a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy index a17695e1940352..ec473a820276c8 100644 --- a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy +++ b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_date_function.groovy @@ -52,7 +52,8 @@ suite("test_datetimev1_date_function", "nonConcurrent") { ('2020-01-06 21:02:03'), ('2020-01-07 09:29:39'), ('2020-01-08 11:33:55'), - ('2020-01-09 14:43:42'); + ('2020-01-09 14:43:42'), + ('2020-01-10 23:59:59'); """ } @@ -66,9 +67,9 @@ suite("test_datetimev1_date_function", "nonConcurrent") { `${table_name}` where date(`${col}`) > ${col_value} - order by 1, 2, 3, 4; + order by 1; """ - quickTest("date_function_test_datetimev1_gt_${table_name}", query1) + quickTest("date_function_test_datetimev1_gt", query1) def query2 = """ select @@ -77,9 +78,9 @@ suite("test_datetimev1_date_function", "nonConcurrent") { `${table_name}` where date(`${col}`) >= ${col_value} - order by 1, 2, 3, 4; + order by 1; """ - quickTest("date_function_test_datetimev1_gte_${table_name}", query2) + quickTest("date_function_test_datetimev1_gte", query2) def query3 = """ select @@ -88,9 +89,9 @@ suite("test_datetimev1_date_function", "nonConcurrent") { `${table_name}` where date(`${col}`) < ${col_value} - order by 1, 2, 3, 4; + order by 1; """ - quickTest("date_function_test_datetimev1_lt_${table_name}", query3) + quickTest("date_function_test_datetimev1_lt", query3) def query4 = """ select @@ -99,15 +100,23 @@ suite("test_datetimev1_date_function", "nonConcurrent") { `${table_name}` where date(`${col}`) <= ${col_value} - order by 1, 2, 3, 4; + order by 1; """ - quickTest("date_function_test_datetimev1_lte_${table_name}", query4) + quickTest("date_function_test_datetimev1_lte", query4) + + def query5 = """ + select + * + from + `${table_name}` + where + date(`${col}`) = ${col_value} + order by 1; + """ + quickTest("date_function_test_datetimev1_eq", query5) } - run_compare_test(table_dup, "date_key1", "'2020-01-05 17:58:59'"); - run_compare_test(table_dup, "date_key1", "'2022-05-02 00:00:00'"); - run_compare_test(table_dup, "date_key1", "'2020-01-05 17:58:59'"); - run_compare_test(table_dup, "date_key1", "'2022-05-02 00:00:00'"); + run_compare_test(table_dup, "date_key1", "'2020-01-05'") sql """ admin set frontend config("enable_date_conversion" = "true");