From 1a15f533098eec6708121004ef835d111dc374f3 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 13 Aug 2020 22:09:31 +0800 Subject: [PATCH 01/15] udf: replace function --- be/src/exprs/string_functions.cpp | 19 ++++++++ be/src/exprs/string_functions.h | 3 ++ be/test/exprs/string_functions_test.cpp | 13 ++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ gensrc/script/doris_builtins_functions.py | 2 + 6 files changed, 129 insertions(+) create mode 100644 docs/en/sql-reference/sql-functions/string-functions/str_replace.md create mode 100644 docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 998794c8392345..ecf7695f45ce12 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -998,4 +998,23 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& int len = (find[field.val - 1] == -1 ? content.len : find[field.val - 1]) - start_pos; return StringVal(content.ptr + start_pos, len); } + +StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { + if (origStr.is_null || oldStr.is_null || newStr.is_null) { + return origStr; + } + std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); + std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); + std::string new_str = std::string(reinterpret_cast(newStr.ptr), newStr.len); + std::string::size_type pos = 0; + std::string::size_type oldLen = old_str.size(); + std::string::size_type newLen = new_str.size(); + while(pos = orig_str.find(old_str, pos)) + { + if(pos == std::string::npos) break; + orig_str.replace(pos, oldLen, new_str); + pos += newLen; + } + return AnyValUtil::from_string_temp(context, orig_str); +} } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index 13b96bcf5feb61..e60969bde5aa7c 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -190,6 +190,9 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); + + static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 05d484c67cd1bd..339f89f4342cb7 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -494,6 +494,19 @@ TEST_F(StringFunctionsTest, rpad) { ASSERT_EQ(StringVal("呵呵hih"), StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } + +TEST_F(StringFunctionsTest, str_replace) { + ASSERT_EQ(StringVal("http://www.baidu.com:8080"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); +} + } // namespace doris int main(int argc, char** argv) { diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..f9a28cccece692 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +replace all old substring with new substring in str + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..cf0bd769badfe4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +将str字符串中的old子串全部替换为new串 + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 5534f1b2b0407f..c966b7d8f9a9cb 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,6 +605,8 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From 391158f52190755b5fc621c31bf703835c0a7b3b Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 10:11:25 +0800 Subject: [PATCH 02/15] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/test/exprs/string_functions_test.cpp | 22 +++++++++++++++++++ .../string-functions/str_replace.md | 2 +- .../string-functions/str_replace.md | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index ecf7695f45ce12..06c6d31aa75aa9 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1001,7 +1001,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { - return origStr; + return StringVal::null(); } std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 339f89f4342cb7..09185505859138 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -496,15 +496,37 @@ TEST_F(StringFunctionsTest, rpad) { } TEST_F(StringFunctionsTest, str_replace) { + //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + + //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + + //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + + //origin string is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + + //old substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + + //new substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + + //substring contains Chinese character + ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), + StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md index f9a28cccece692..dfbd8221b9f72a 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md index cf0bd769badfe4..2180cfa0670741 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE From 5eb52e1e5742d2e9f7aae5ac8da81545ae5f3df2 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 11:58:30 +0800 Subject: [PATCH 03/15] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 06c6d31aa75aa9..deca61f9c3bc87 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1009,7 +1009,7 @@ StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal std::string::size_type pos = 0; std::string::size_type oldLen = old_str.size(); std::string::size_type newLen = new_str.size(); - while(pos = orig_str.find(old_str, pos)) + while ((pos = orig_str.find(old_str, pos))) { if(pos == std::string::npos) break; orig_str.replace(pos, oldLen, new_str); From 86f841f29fa4ee19773d6c5922465194a2887cf6 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:56:14 +0800 Subject: [PATCH 04/15] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/src/exprs/string_functions.h | 2 +- be/test/exprs/string_functions_test.cpp | 18 +++++++++--------- fe/fe-core/src/main/cup/sql_parser.cup | 2 ++ gensrc/script/doris_builtins_functions.py | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index deca61f9c3bc87..1679eca54b9ddd 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -999,7 +999,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& return StringVal(content.ptr + start_pos, len); } -StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { +StringVal StringFunctions::replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { return StringVal::null(); } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index e60969bde5aa7c..f1a4df9e373f51 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -191,7 +191,7 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); - static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + static StringVal replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 09185505859138..3a6e351f60f990 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -495,38 +495,38 @@ TEST_F(StringFunctionsTest, rpad) { StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } -TEST_F(StringFunctionsTest, str_replace) { +TEST_F(StringFunctionsTest, replace) { //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); //origin string is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); //old substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); //new substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); //substring contains Chinese character ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), - StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); + StringFunctions::replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index a2d8294915cc21..7ba56810734001 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -4593,6 +4593,8 @@ keyword ::= {: RESULT = id; :} | KW_REPEATABLE:id {: RESULT = id; :} + | KW_REPLACE:id + {: RESULT = id; :} | KW_REPLACE_IF_NOT_NULL:id {: RESULT = id; :} | KW_REPOSITORY:id diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index c966b7d8f9a9cb..ed26fae15f87aa 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,7 +605,7 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], - [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' From ade1afa75c94a178a6801ba1324acf25ad8b16fe Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:59:59 +0800 Subject: [PATCH 05/15] udf: replace function --- .../string-functions/{str_replace.md => replace.md} | 10 +++++----- .../string-functions/{str_replace.md => replace.md} | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) rename docs/en/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (83%) rename docs/zh-CN/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (82%) diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md similarity index 83% rename from docs/en/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/en/sql-reference/sql-functions/string-functions/replace.md index dfbd8221b9f72a..6f86b18a679d3d 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,20 +24,20 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` replace all old substring with new substring in str ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md similarity index 82% rename from docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md index 2180cfa0670741..dda2625df07d90 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,23 +24,23 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` 将str字符串中的old子串全部替换为new串 ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE From 1d95a491115064c9753694ad4d45c2f72b8d2645 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 16:01:45 +0800 Subject: [PATCH 06/15] udf: replace function --- docs/en/sql-reference/sql-functions/string-functions/replace.md | 2 +- gensrc/script/doris_builtins_functions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/sql-functions/string-functions/replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md index 6f86b18a679d3d..d5111fa43c8077 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -43,4 +43,4 @@ mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index ed26fae15f87aa..b960ba6afbbba0 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -606,7 +606,7 @@ [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], + '_ZN5doris15StringFunctions7replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From 33af93fd282010043dac760ce09ddf70aacbc1c6 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Sat, 17 Oct 2020 20:21:32 +0800 Subject: [PATCH 07/15] check invalid date type --- .../doris/analysis/BinaryPredicate.java | 36 +++++++++ .../apache/doris/rewrite/ExprRewriter.java | 10 ++- .../apache/doris/planner/QueryPlanTest.java | 77 +++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) mode change 100644 => 100755 fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java mode change 100644 => 100755 fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index 30ae97435bac05..53970cccea698c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -428,6 +428,42 @@ public Expr getSlotBinding(SlotId id) { return null; } + /** + * If predicate is of the form " ", returns expr, + * otherwise returns null. Slotref may be wrapped in a CastExpr. + */ + public Expr getBinding() { + SlotRef slotRef = null; + // check left operand + if (getChild(0) instanceof SlotRef) { + slotRef = (SlotRef) getChild(0); + } else if (getChild(0) instanceof CastExpr && getChild(0).getChild(0) instanceof SlotRef) { + if (((CastExpr) getChild(0)).canHashPartition()) { + slotRef = (SlotRef) getChild(0).getChild(0); + } + } + if (slotRef != null) { + slotIsleft = true; + return getChild(1); + } + + // check right operand + if (getChild(1) instanceof SlotRef) { + slotRef = (SlotRef) getChild(1); + } else if (getChild(1) instanceof CastExpr && getChild(1).getChild(0) instanceof SlotRef) { + if (((CastExpr) getChild(1)).canHashPartition()) { + slotRef = (SlotRef) getChild(1).getChild(0); + } + } + + if (slotRef != null) { + slotIsleft = false; + return getChild(0); + } + + return null; + } + /** * If e is an equality predicate between two slots that only require implicit * casts, returns those two slots; otherwise returns null. diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java old mode 100644 new mode 100755 index d5207ff5960beb..39ca638a85c346 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java @@ -20,7 +20,10 @@ import java.util.List; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Lists; @@ -36,7 +39,6 @@ public class ExprRewriter { private int numChanges_ = 0; private final List rules_; - public ExprRewriter(List rules) { rules_ = rules; } @@ -55,6 +57,12 @@ public Expr rewrite(Expr expr, Analyzer analyzer) throws AnalysisException { rewrittenExpr = applyRuleRepeatedly(rewrittenExpr, rule, analyzer); } } while (oldNumChanges != numChanges_); + if (expr instanceof BinaryPredicate) { + Expr valueExpr = ((BinaryPredicate) expr).getBinding(); + if(valueExpr != null && valueExpr.getType() == Type.DATETIME && valueExpr instanceof CastExpr) { + throw new AnalysisException("invalid date type :" + valueExpr.toSql()); + } + } return rewrittenExpr; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java old mode 100644 new mode 100755 index f2b93fba6934dd..1eb3ed95cd4134 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -388,6 +388,20 @@ public static void beforeClass() throws Exception { "\"driver\" = \"Oracle Driver\",\n" + "\"odbc_type\" = \"mysql\"\n" + ");"); + + createTable("create table test.invalid_date (" + + "`date` datetime NULL," + + "`day` date NULL," + + "`site_id` int(11) NULL )" + + " ENGINE=OLAP " + + "DUPLICATE KEY(`date`, `day`, `site_id`)" + + "PARTITION BY RANGE(`day`) ()" + + "DISTRIBUTED BY HASH(`site_id`) BUCKETS 10 " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");"); } @AfterClass @@ -1310,6 +1324,69 @@ public void testAggregateSatisfyOlapTableDistribution() throws Exception { System.out.println(explainString); Assert.assertTrue(explainString.contains("AGGREGATE (update finalize)")); } + + @Test + public void testCheckInvalidDate() throws Exception { + FeConstants.runningUnitTest = true; + connectContext.setDatabase("default_cluster:test"); + //valid date + String sql = "select day from invalid_date where day = '2020-10-30'"; + String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + System.out.println(explainString); + //valid date + sql = "select day from invalid_date where day = 20201030"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + System.out.println(explainString); + //invalid date + sql = "select day from invalid_date where day = '2020-10-32'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + System.out.println(explainString); + Assert.assertTrue(explainString.contains("invalid date type")); + //invalid date + sql = "select day from invalid_date where day = 'hello'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("invalid date type")); + //invalid date + sql = "select day from invalid_date where day = 2020-10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + //invalid date + sql = "select day from invalid_date where day = 10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("invalid date type")); + + //valid datetime + sql = "select day from invalid_date where date = '2020-10-30 12:12:30'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + System.out.println(explainString); + //valid date + sql = "select day from invalid_date where day = 20201030"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + System.out.println(explainString); + //valid date + sql = "select day from invalid_date where day = '2020-10-30'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + System.out.println(explainString); + //invalid date + sql = "select day from invalid_date where day = '2020-10-32'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("invalid date type")); + //invalid date + sql = "select day from invalid_date where day = 'hello'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("invalid date type")); + //invalid date + sql = "select day from invalid_date where day = 2020-10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("invalid date type")); + //invalid date + sql = "select day from invalid_date where day = 10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("invalid date type")); + //invalid date + sql = "select day from invalid_date where day = '2020-10-12 12:23:76'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("invalid date type")); + } } From ab58cbd3dc2cbd0c7358c5847eef3f8aad8bfcf0 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Sat, 17 Oct 2020 20:55:07 +0800 Subject: [PATCH 08/15] Check date type to avoid scan all partitions --- .../src/main/java/org/apache/doris/rewrite/ExprRewriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java index 39ca638a85c346..59242f3922a181 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java @@ -39,6 +39,7 @@ public class ExprRewriter { private int numChanges_ = 0; private final List rules_; + public ExprRewriter(List rules) { rules_ = rules; } From 92ee1afa05c4c077ba17598780d970e6ef2970bb Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 22 Oct 2020 16:35:41 +0800 Subject: [PATCH 09/15] update --- .../org/apache/doris/analysis/Analyzer.java | 2 + .../doris/analysis/BinaryPredicate.java | 36 -------------- .../rewrite/BinaryPredicatesDateRule.java | 48 +++++++++++++++++++ .../apache/doris/rewrite/ExprRewriter.java | 9 ---- 4 files changed, 50 insertions(+), 45 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index 0956a6ef0a1722..9869d9c131bec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -46,6 +46,7 @@ import org.apache.doris.rewrite.mvrewrite.HLLHashToSlotRefRule; import org.apache.doris.rewrite.mvrewrite.NDVToHll; import org.apache.doris.rewrite.mvrewrite.ToBitmapToSlotRefRule; +import org.apache.doris.rewrite.BinaryPredicatesDateRule; import org.apache.doris.thrift.TQueryGlobals; import com.google.common.base.Joiner; @@ -255,6 +256,7 @@ public GlobalState(Catalog catalog, ConnectContext context) { // pushdown and Parquet row group pruning based on min/max statistics. rules.add(NormalizeBinaryPredicatesRule.INSTANCE); rules.add(FoldConstantsRule.INSTANCE); + rules.add(BinaryPredicatesDateRule.INSTANCE); exprRewriter_ = new ExprRewriter(rules); // init mv rewriter List mvRewriteRules = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index 53970cccea698c..30ae97435bac05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -428,42 +428,6 @@ public Expr getSlotBinding(SlotId id) { return null; } - /** - * If predicate is of the form " ", returns expr, - * otherwise returns null. Slotref may be wrapped in a CastExpr. - */ - public Expr getBinding() { - SlotRef slotRef = null; - // check left operand - if (getChild(0) instanceof SlotRef) { - slotRef = (SlotRef) getChild(0); - } else if (getChild(0) instanceof CastExpr && getChild(0).getChild(0) instanceof SlotRef) { - if (((CastExpr) getChild(0)).canHashPartition()) { - slotRef = (SlotRef) getChild(0).getChild(0); - } - } - if (slotRef != null) { - slotIsleft = true; - return getChild(1); - } - - // check right operand - if (getChild(1) instanceof SlotRef) { - slotRef = (SlotRef) getChild(1); - } else if (getChild(1) instanceof CastExpr && getChild(1).getChild(0) instanceof SlotRef) { - if (((CastExpr) getChild(1)).canHashPartition()) { - slotRef = (SlotRef) getChild(1).getChild(0); - } - } - - if (slotRef != null) { - slotIsleft = false; - return getChild(0); - } - - return null; - } - /** * If e is an equality predicate between two slots that only require implicit * casts, returns those two slots; otherwise returns null. diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java new file mode 100644 index 00000000000000..3d9212ca5400c3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.BoolLiteral; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; + +/** + * Normalizes binary predicates of the form so that the slot is + * on the left hand side. Predicates where is wrapped in a cast (implicit or + * explicit) are normalized, too. + * + * Examples: + * 5 > id -> id < 5 + */ +public class BinaryPredicatesDateRule implements ExprRewriteRule { + public static ExprRewriteRule INSTANCE = new BinaryPredicatesDateRule(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { + if (!(expr instanceof BinaryPredicate)) return expr; + Expr valueExpr = expr.getChild(1); + if(valueExpr.getType() == Type.DATETIME && valueExpr instanceof CastExpr) { + return new BoolLiteral(false); + } + return expr; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java index 59242f3922a181..d5207ff5960beb 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java @@ -20,10 +20,7 @@ import java.util.List; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.CastExpr; -import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Lists; @@ -58,12 +55,6 @@ public Expr rewrite(Expr expr, Analyzer analyzer) throws AnalysisException { rewrittenExpr = applyRuleRepeatedly(rewrittenExpr, rule, analyzer); } } while (oldNumChanges != numChanges_); - if (expr instanceof BinaryPredicate) { - Expr valueExpr = ((BinaryPredicate) expr).getBinding(); - if(valueExpr != null && valueExpr.getType() == Type.DATETIME && valueExpr instanceof CastExpr) { - throw new AnalysisException("invalid date type :" + valueExpr.toSql()); - } - } return rewrittenExpr; } From c5ef96ee72be91124694f0944169e3b14053fb1e Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 22 Oct 2020 17:38:58 +0800 Subject: [PATCH 10/15] update --- .../rewrite/BinaryPredicatesDateRule.java | 8 ++-- .../apache/doris/planner/QueryPlanTest.java | 47 +++++++++---------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java index 3d9212ca5400c3..859d2ed6db6ae4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java @@ -26,12 +26,10 @@ import org.apache.doris.common.AnalysisException; /** - * Normalizes binary predicates of the form so that the slot is - * on the left hand side. Predicates where is wrapped in a cast (implicit or - * explicit) are normalized, too. - * + * Binary predicaate date rule try to convert date expression, if date is invalid, it will be + * convert into bool literal to avoid to scan all partitions * Examples: - * 5 > id -> id < 5 + * date = "2020-10-32" => FALSE */ public class BinaryPredicatesDateRule implements ExprRewriteRule { public static ExprRewriteRule INSTANCE = new BinaryPredicatesDateRule(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 1068b1be7ebee6..b01d144e3bcfaf 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -389,13 +389,12 @@ public static void beforeClass() throws Exception { "\"odbc_type\" = \"mysql\"\n" + ");"); - createTable("create table test.invalid_date (" + + createTable("create table test.tbl_date (" + "`date` datetime NULL," + "`day` date NULL," + "`site_id` int(11) NULL )" + " ENGINE=OLAP " + "DUPLICATE KEY(`date`, `day`, `site_id`)" + - "PARTITION BY RANGE(`day`) ()" + "DISTRIBUTED BY HASH(`site_id`) BUCKETS 10 " + "PROPERTIES (\n" + "\"replication_num\" = \"1\",\n" + @@ -1337,62 +1336,62 @@ public void testCheckInvalidDate() throws Exception { FeConstants.runningUnitTest = true; connectContext.setDatabase("default_cluster:test"); //valid date - String sql = "select day from invalid_date where day = '2020-10-30'"; + String sql = "select day from tbl_date where day = '2020-10-30'"; String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); System.out.println(explainString); //valid date - sql = "select day from invalid_date where day = 20201030"; + sql = "select day from tbl_date where day = 20201030"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); System.out.println(explainString); //invalid date - sql = "select day from invalid_date where day = '2020-10-32'"; + sql = "select day from tbl_date where day = '2020-10-32'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); System.out.println(explainString); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); //invalid date - sql = "select day from invalid_date where day = 'hello'"; + sql = "select day from tbl_date where day = 'hello'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); //invalid date - sql = "select day from invalid_date where day = 2020-10-30"; + sql = "select day from tbl_date where day = 2020-10-30"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); //invalid date - sql = "select day from invalid_date where day = 10-30"; + sql = "select day from tbl_date where day = 10-30"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); //valid datetime - sql = "select day from invalid_date where date = '2020-10-30 12:12:30'"; + sql = "select day from tbl_date where date = '2020-10-30 12:12:30'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); System.out.println(explainString); //valid date - sql = "select day from invalid_date where day = 20201030"; + sql = "select day from tbl_date where day = 20201030"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); System.out.println(explainString); //valid date - sql = "select day from invalid_date where day = '2020-10-30'"; + sql = "select day from tbl_date where day = '2020-10-30'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); System.out.println(explainString); //invalid date - sql = "select day from invalid_date where day = '2020-10-32'"; + sql = "select day from tbl_date where day = '2020-10-32'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); //invalid date - sql = "select day from invalid_date where day = 'hello'"; + sql = "select day from tbl_date where day = 'hello'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); //invalid date - sql = "select day from invalid_date where day = 2020-10-30"; + sql = "select day from tbl_date where day = 2020-10-30"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); //invalid date - sql = "select day from invalid_date where day = 10-30"; + sql = "select day from tbl_date where day = 10-30"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); //invalid date - sql = "select day from invalid_date where day = '2020-10-12 12:23:76'"; + sql = "select day from tbl_date where day = '2020-10-12 12:23:76'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("invalid date type")); + Assert.assertTrue(explainString.contains("EMPTYSET")); } public void testLeadAndLagFunction() throws Exception { From 87378184aa355d7acffc6c250cc1d5a19f83b5ce Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 22 Oct 2020 17:40:44 +0800 Subject: [PATCH 11/15] update --- .../src/main/java/org/apache/doris/rewrite/ExprRewriter.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java old mode 100755 new mode 100644 From 9a302474f7b4f117e7c018698b393b89115ab4a9 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 23 Oct 2020 10:08:34 +0800 Subject: [PATCH 12/15] update --- .../apache/doris/rewrite/BinaryPredicatesDateRule.java | 4 ++-- .../java/org/apache/doris/planner/QueryPlanTest.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java index 859d2ed6db6ae4..1c7b1144a6afee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java @@ -26,8 +26,8 @@ import org.apache.doris.common.AnalysisException; /** - * Binary predicaate date rule try to convert date expression, if date is invalid, it will be - * convert into bool literal to avoid to scan all partitions + * Binary predicate date rule try to convert date expression, if date is invalid, it will be + * converted into bool literal to avoid to scan all partitions * Examples: * date = "2020-10-32" => FALSE */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index b01d144e3bcfaf..fffcfde821190d 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -1338,11 +1338,11 @@ public void testCheckInvalidDate() throws Exception { //valid date String sql = "select day from tbl_date where day = '2020-10-30'"; String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - System.out.println(explainString); + Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); //valid date sql = "select day from tbl_date where day = 20201030"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - System.out.println(explainString); + Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); //invalid date sql = "select day from tbl_date where day = '2020-10-32'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); @@ -1363,15 +1363,15 @@ public void testCheckInvalidDate() throws Exception { //valid datetime sql = "select day from tbl_date where date = '2020-10-30 12:12:30'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - System.out.println(explainString); + Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 12:12:30'")); //valid date sql = "select day from tbl_date where day = 20201030"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - System.out.println(explainString); + Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); //valid date sql = "select day from tbl_date where day = '2020-10-30'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - System.out.println(explainString); + Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); //invalid date sql = "select day from tbl_date where day = '2020-10-32'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); From a4bdbcdd677781efedfb5df8d487f6e876e407b2 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 23 Oct 2020 23:37:47 +0800 Subject: [PATCH 13/15] update --- .../apache/doris/planner/QueryPlanTest.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index fffcfde821190d..c8f09c2a175ab3 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -1364,32 +1364,32 @@ public void testCheckInvalidDate() throws Exception { sql = "select day from tbl_date where date = '2020-10-30 12:12:30'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 12:12:30'")); - //valid date - sql = "select day from tbl_date where day = 20201030"; + //valid datetime + sql = "select day from tbl_date where date = 20201030"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); - //valid date - sql = "select day from tbl_date where day = '2020-10-30'"; + Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 00:00:00'")); + //valid datetime + sql = "select day from tbl_date where date = '2020-10-30'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); - //invalid date - sql = "select day from tbl_date where day = '2020-10-32'"; + Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 00:00:00'")); + //invalid datetime + sql = "select day from tbl_date where date = '2020-10-32'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid date - sql = "select day from tbl_date where day = 'hello'"; + //invalid datetime + sql = "select day from tbl_date where date = 'hello'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid date - sql = "select day from tbl_date where day = 2020-10-30"; + //invalid datetime + sql = "select day from tbl_date where date = 2020-10-30"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid date - sql = "select day from tbl_date where day = 10-30"; + //invalid datetime + sql = "select day from tbl_date where date = 10-30"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid date - sql = "select day from tbl_date where day = '2020-10-12 12:23:76'"; + //invalid datetime + sql = "select day from tbl_date where date = '2020-10-12 12:23:76'"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains("EMPTYSET")); } From 685a7bf0ea0f884def809d8d8a664eeae96bd371 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Sun, 1 Nov 2020 19:39:24 +0800 Subject: [PATCH 14/15] update --- .../rewrite/BinaryPredicatesDateRule.java | 8 +- .../apache/doris/planner/QueryPlanTest.java | 127 +++++++++--------- 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java index 1c7b1144a6afee..200466cb994e11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/BinaryPredicatesDateRule.java @@ -22,7 +22,6 @@ import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.BoolLiteral; -import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; /** @@ -37,8 +36,13 @@ public class BinaryPredicatesDateRule implements ExprRewriteRule { @Override public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { if (!(expr instanceof BinaryPredicate)) return expr; + Expr lchild = expr.getChild(0); + if (!lchild.getType().isDateType()) { + return expr; + } Expr valueExpr = expr.getChild(1); - if(valueExpr.getType() == Type.DATETIME && valueExpr instanceof CastExpr) { + if(valueExpr.getType().isDateType() && valueExpr.isConstant() + && valueExpr instanceof CastExpr) { return new BoolLiteral(false); } return expr; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 153dd629ef27b1..1335179e466ab8 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -388,7 +388,7 @@ public static void beforeClass() throws Exception { "\"driver\" = \"Oracle Driver\",\n" + "\"odbc_type\" = \"mysql\"\n" + ");"); - + createTable("create table test.tbl_int_date (" + "`date` datetime NULL," + "`day` date NULL," + @@ -1331,69 +1331,6 @@ public void testAggregateSatisfyOlapTableDistribution() throws Exception { Assert.assertTrue(explainString.contains("AGGREGATE (update finalize)")); } - @Test - public void testCheckInvalidDate() throws Exception { - FeConstants.runningUnitTest = true; - connectContext.setDatabase("default_cluster:test"); - //valid date - String sql = "select day from tbl_date where day = '2020-10-30'"; - String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); - //valid date - sql = "select day from tbl_date where day = 20201030"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); - //invalid date - sql = "select day from tbl_date where day = '2020-10-32'"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - System.out.println(explainString); - Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid date - sql = "select day from tbl_date where day = 'hello'"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid date - sql = "select day from tbl_date where day = 2020-10-30"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - //invalid date - sql = "select day from tbl_date where day = 10-30"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("EMPTYSET")); - - //valid datetime - sql = "select day from tbl_date where date = '2020-10-30 12:12:30'"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 12:12:30'")); - //valid datetime - sql = "select day from tbl_date where date = 20201030"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 00:00:00'")); - //valid datetime - sql = "select day from tbl_date where date = '2020-10-30'"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 00:00:00'")); - //invalid datetime - sql = "select day from tbl_date where date = '2020-10-32'"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid datetime - sql = "select day from tbl_date where date = 'hello'"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid datetime - sql = "select day from tbl_date where date = 2020-10-30"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid datetime - sql = "select day from tbl_date where date = 10-30"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("EMPTYSET")); - //invalid datetime - sql = "select day from tbl_date where date = '2020-10-12 12:23:76'"; - explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("EMPTYSET")); - } - public void testLeadAndLagFunction() throws Exception { connectContext.setDatabase("default_cluster:test"); @@ -1444,6 +1381,68 @@ public void testIntDateTime() throws Exception { explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains("PREDICATES: `date` IN ('2020-10-30 00:00:00')")); } + + public void testCheckInvalidDate() throws Exception { + FeConstants.runningUnitTest = true; + connectContext.setDatabase("default_cluster:test"); + //valid date + String sql = "select day from tbl_int_date where day = '2020-10-30'"; + String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); + //valid date + sql = "select day from tbl_int_date where day = 20201030"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'")); + //invalid date + sql = "select day from tbl_int_date where day = '2020-10-32'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + System.out.println(explainString); + Assert.assertTrue(explainString.contains("EMPTYSET")); + //invalid date + sql = "select day from tbl_int_date where day = 'hello'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("EMPTYSET")); + //invalid date + sql = "select day from tbl_int_date where day = 2020-10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + //invalid date + sql = "select day from tbl_int_date where day = 10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("EMPTYSET")); + + //valid datetime + sql = "select day from tbl_int_date where date = '2020-10-30 12:12:30'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 12:12:30'")); + //valid datetime + sql = "select day from tbl_int_date where date = 20201030"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 00:00:00'")); + //valid datetime + sql = "select day from tbl_int_date where date = '2020-10-30'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 00:00:00'")); + //invalid datetime + sql = "select day from tbl_int_date where date = '2020-10-32'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("EMPTYSET")); + //invalid datetime + sql = "select day from tbl_int_date where date = 'hello'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("EMPTYSET")); + //invalid datetime + sql = "select day from tbl_int_date where date = 2020-10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("EMPTYSET")); + //invalid datetime + sql = "select day from tbl_int_date where date = 10-30"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("EMPTYSET")); + //invalid datetime + sql = "select day from tbl_int_date where date = '2020-10-12 12:23:76'"; + explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); + Assert.assertTrue(explainString.contains("EMPTYSET")); + } } From 7b2d36d78be8cf87d968e4a8df89dbc3cf03c1d3 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Sun, 1 Nov 2020 19:45:51 +0800 Subject: [PATCH 15/15] update --- .../src/test/java/org/apache/doris/planner/QueryPlanTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 1335179e466ab8..ac381077d4845f 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -388,7 +388,7 @@ public static void beforeClass() throws Exception { "\"driver\" = \"Oracle Driver\",\n" + "\"odbc_type\" = \"mysql\"\n" + ");"); - + createTable("create table test.tbl_int_date (" + "`date` datetime NULL," + "`day` date NULL," + @@ -1382,6 +1382,7 @@ public void testIntDateTime() throws Exception { Assert.assertTrue(explainString.contains("PREDICATES: `date` IN ('2020-10-30 00:00:00')")); } + @Test public void testCheckInvalidDate() throws Exception { FeConstants.runningUnitTest = true; connectContext.setDatabase("default_cluster:test");