From 1a15f533098eec6708121004ef835d111dc374f3 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 13 Aug 2020 22:09:31 +0800 Subject: [PATCH 01/16] udf: replace function --- be/src/exprs/string_functions.cpp | 19 ++++++++ be/src/exprs/string_functions.h | 3 ++ be/test/exprs/string_functions_test.cpp | 13 ++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ gensrc/script/doris_builtins_functions.py | 2 + 6 files changed, 129 insertions(+) create mode 100644 docs/en/sql-reference/sql-functions/string-functions/str_replace.md create mode 100644 docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 998794c8392345..ecf7695f45ce12 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -998,4 +998,23 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& int len = (find[field.val - 1] == -1 ? content.len : find[field.val - 1]) - start_pos; return StringVal(content.ptr + start_pos, len); } + +StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { + if (origStr.is_null || oldStr.is_null || newStr.is_null) { + return origStr; + } + std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); + std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); + std::string new_str = std::string(reinterpret_cast(newStr.ptr), newStr.len); + std::string::size_type pos = 0; + std::string::size_type oldLen = old_str.size(); + std::string::size_type newLen = new_str.size(); + while(pos = orig_str.find(old_str, pos)) + { + if(pos == std::string::npos) break; + orig_str.replace(pos, oldLen, new_str); + pos += newLen; + } + return AnyValUtil::from_string_temp(context, orig_str); +} } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index 13b96bcf5feb61..e60969bde5aa7c 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -190,6 +190,9 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); + + static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 05d484c67cd1bd..339f89f4342cb7 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -494,6 +494,19 @@ TEST_F(StringFunctionsTest, rpad) { ASSERT_EQ(StringVal("呵呵hih"), StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } + +TEST_F(StringFunctionsTest, str_replace) { + ASSERT_EQ(StringVal("http://www.baidu.com:8080"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); +} + } // namespace doris int main(int argc, char** argv) { diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..f9a28cccece692 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +replace all old substring with new substring in str + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..cf0bd769badfe4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +将str字符串中的old子串全部替换为new串 + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 5534f1b2b0407f..c966b7d8f9a9cb 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,6 +605,8 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From 391158f52190755b5fc621c31bf703835c0a7b3b Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 10:11:25 +0800 Subject: [PATCH 02/16] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/test/exprs/string_functions_test.cpp | 22 +++++++++++++++++++ .../string-functions/str_replace.md | 2 +- .../string-functions/str_replace.md | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index ecf7695f45ce12..06c6d31aa75aa9 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1001,7 +1001,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { - return origStr; + return StringVal::null(); } std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 339f89f4342cb7..09185505859138 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -496,15 +496,37 @@ TEST_F(StringFunctionsTest, rpad) { } TEST_F(StringFunctionsTest, str_replace) { + //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + + //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + + //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + + //origin string is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + + //old substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + + //new substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + + //substring contains Chinese character + ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), + StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md index f9a28cccece692..dfbd8221b9f72a 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md index cf0bd769badfe4..2180cfa0670741 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE From 5eb52e1e5742d2e9f7aae5ac8da81545ae5f3df2 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 11:58:30 +0800 Subject: [PATCH 03/16] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 06c6d31aa75aa9..deca61f9c3bc87 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1009,7 +1009,7 @@ StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal std::string::size_type pos = 0; std::string::size_type oldLen = old_str.size(); std::string::size_type newLen = new_str.size(); - while(pos = orig_str.find(old_str, pos)) + while ((pos = orig_str.find(old_str, pos))) { if(pos == std::string::npos) break; orig_str.replace(pos, oldLen, new_str); From 86f841f29fa4ee19773d6c5922465194a2887cf6 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:56:14 +0800 Subject: [PATCH 04/16] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/src/exprs/string_functions.h | 2 +- be/test/exprs/string_functions_test.cpp | 18 +++++++++--------- fe/fe-core/src/main/cup/sql_parser.cup | 2 ++ gensrc/script/doris_builtins_functions.py | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index deca61f9c3bc87..1679eca54b9ddd 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -999,7 +999,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& return StringVal(content.ptr + start_pos, len); } -StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { +StringVal StringFunctions::replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { return StringVal::null(); } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index e60969bde5aa7c..f1a4df9e373f51 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -191,7 +191,7 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); - static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + static StringVal replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 09185505859138..3a6e351f60f990 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -495,38 +495,38 @@ TEST_F(StringFunctionsTest, rpad) { StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } -TEST_F(StringFunctionsTest, str_replace) { +TEST_F(StringFunctionsTest, replace) { //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); //origin string is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); //old substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); //new substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); //substring contains Chinese character ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), - StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); + StringFunctions::replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index a2d8294915cc21..7ba56810734001 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -4593,6 +4593,8 @@ keyword ::= {: RESULT = id; :} | KW_REPEATABLE:id {: RESULT = id; :} + | KW_REPLACE:id + {: RESULT = id; :} | KW_REPLACE_IF_NOT_NULL:id {: RESULT = id; :} | KW_REPOSITORY:id diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index c966b7d8f9a9cb..ed26fae15f87aa 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,7 +605,7 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], - [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' From ade1afa75c94a178a6801ba1324acf25ad8b16fe Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:59:59 +0800 Subject: [PATCH 05/16] udf: replace function --- .../string-functions/{str_replace.md => replace.md} | 10 +++++----- .../string-functions/{str_replace.md => replace.md} | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) rename docs/en/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (83%) rename docs/zh-CN/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (82%) diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md similarity index 83% rename from docs/en/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/en/sql-reference/sql-functions/string-functions/replace.md index dfbd8221b9f72a..6f86b18a679d3d 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,20 +24,20 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` replace all old substring with new substring in str ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md similarity index 82% rename from docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md index 2180cfa0670741..dda2625df07d90 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,23 +24,23 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` 将str字符串中的old子串全部替换为new串 ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE From 1d95a491115064c9753694ad4d45c2f72b8d2645 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 16:01:45 +0800 Subject: [PATCH 06/16] udf: replace function --- docs/en/sql-reference/sql-functions/string-functions/replace.md | 2 +- gensrc/script/doris_builtins_functions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/sql-functions/string-functions/replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md index 6f86b18a679d3d..d5111fa43c8077 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -43,4 +43,4 @@ mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index ed26fae15f87aa..b960ba6afbbba0 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -606,7 +606,7 @@ [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], + '_ZN5doris15StringFunctions7replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From 313e0a9ccd35297279011e904711311fe6acf621 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Mon, 24 Aug 2020 20:55:03 +0800 Subject: [PATCH 07/16] rand function validate literal param --- .../java/org/apache/doris/analysis/Expr.java | 10 +++- .../org/apache/doris/analysis/ExprTest.java | 49 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) mode change 100644 => 100755 fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java mode change 100644 => 100755 fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java old mode 100644 new mode 100755 index afe7e243a7e7e9..dbfd2aa3dcd74f --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -1405,7 +1405,15 @@ protected Function getBuiltinFunction( throws AnalysisException { FunctionName fnName = new FunctionName(name); Function searchDesc = new Function(fnName, argTypes, Type.INVALID, false); - return Catalog.getCurrentCatalog().getFunction(searchDesc, mode); + Function f = Catalog.getCurrentCatalog().getFunction(searchDesc, mode); + if (f != null && fnName.getFunction().equalsIgnoreCase("rand")) { + if (this.children != null + && this.children.size() == 1 + && !(this.children.get(0) instanceof LiteralExpr)) { + throw new AnalysisException("The param of rand function must be literal"); + } + } + return f; } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java old mode 100644 new mode 100755 index 991cf79fcb009e..01f318e6f4851b --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -24,11 +24,18 @@ import com.google.common.collect.Maps; +import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.mysql.privilege.PaloAuth; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.DorisAssert; +import org.apache.doris.utframe.UtFrameUtils; import org.junit.Assert; import org.junit.Test; +import org.junit.BeforeClass; +import org.junit.AfterClass; -import java.util.Map; -import java.util.Set; +import java.io.IOException; +import java.util.*; import mockit.Expectations; import mockit.Injectable; @@ -36,6 +43,24 @@ public class ExprTest { + private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/"; + private static DorisAssert dorisAssert; + + @AfterClass + public static void tearDown() throws Exception { + UtFrameUtils.cleanDorisFeDir(runningDir); + } + + @BeforeClass + public static void setUp() throws Exception { + UtFrameUtils.createMinDorisCluster(runningDir); + String createTblStmtStr = "create table db1.tbl1(k1 varchar(32), k2 varchar(32), k3 varchar(32), k4 int) " + + "AGGREGATE KEY(k1, k2,k3,k4) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"; + dorisAssert = new DorisAssert(); + dorisAssert.withDatabase("db1").useDatabase("db1"); + dorisAssert.withTable(createTblStmtStr); + } + @Test public void testGetTableNameToColumnNames(@Mocked Analyzer analyzer, @Injectable SlotDescriptor slotDesc1, @@ -159,4 +184,24 @@ public void testUncheckedCastTo() throws AnalysisException { StringLiteral castStringLiteral2 = (StringLiteral) stringLiteral.uncheckedCastTo(Type.VARCHAR); Assert.assertTrue(stringLiteral == castStringLiteral2); } + + @Test + public void testRandFunction() { + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand(db1.tbl1.k1) from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("The param of rand function must be literal")); + } + + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand(1234) from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); + } + Assert.assertTrue(1 == 1); + } } From d00fde471708754b9db2213e6d066ff98f5c9ef9 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Mon, 24 Aug 2020 20:58:53 +0800 Subject: [PATCH 08/16] rand function validate literal param --- .../org/apache/doris/analysis/ExprTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java index 01f318e6f4851b..41b90a9b70b836 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -203,5 +203,23 @@ public void testRandFunction() { Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); } Assert.assertTrue(1 == 1); + + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand() from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); + } + Assert.assertTrue(1 == 1); + + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand(\"hello\") from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); + } + Assert.assertTrue(1 == 1); } } From f01f1f46d2c1c86f0bf31f75834433c0d5e31361 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 25 Aug 2020 20:46:42 +0800 Subject: [PATCH 09/16] validate funtion rand with literal in compile step --- .../org/apache/doris/analysis/ExprTest.java | 58 ------------------- .../apache/doris/analysis/SelectStmtTest.java | 35 +++++++++++ 2 files changed, 35 insertions(+), 58 deletions(-) mode change 100644 => 100755 fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java index 41b90a9b70b836..da966697800f3a 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -31,8 +31,6 @@ import org.apache.doris.utframe.UtFrameUtils; import org.junit.Assert; import org.junit.Test; -import org.junit.BeforeClass; -import org.junit.AfterClass; import java.io.IOException; import java.util.*; @@ -43,24 +41,6 @@ public class ExprTest { - private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/"; - private static DorisAssert dorisAssert; - - @AfterClass - public static void tearDown() throws Exception { - UtFrameUtils.cleanDorisFeDir(runningDir); - } - - @BeforeClass - public static void setUp() throws Exception { - UtFrameUtils.createMinDorisCluster(runningDir); - String createTblStmtStr = "create table db1.tbl1(k1 varchar(32), k2 varchar(32), k3 varchar(32), k4 int) " - + "AGGREGATE KEY(k1, k2,k3,k4) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"; - dorisAssert = new DorisAssert(); - dorisAssert.withDatabase("db1").useDatabase("db1"); - dorisAssert.withTable(createTblStmtStr); - } - @Test public void testGetTableNameToColumnNames(@Mocked Analyzer analyzer, @Injectable SlotDescriptor slotDesc1, @@ -184,42 +164,4 @@ public void testUncheckedCastTo() throws AnalysisException { StringLiteral castStringLiteral2 = (StringLiteral) stringLiteral.uncheckedCastTo(Type.VARCHAR); Assert.assertTrue(stringLiteral == castStringLiteral2); } - - @Test - public void testRandFunction() { - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand(db1.tbl1.k1) from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("The param of rand function must be literal")); - } - - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand(1234) from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); - } - Assert.assertTrue(1 == 1); - - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand() from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); - } - Assert.assertTrue(1 == 1); - - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand(\"hello\") from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); - } - Assert.assertTrue(1 == 1); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java old mode 100644 new mode 100755 index e03e8c5a39fddb..bd4eceaeb6d0f8 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -362,4 +362,39 @@ public void testDataGripSupport() throws Exception { "from information_schema.collations"; dorisAssert.query(sql).explainQuery(); } + + @Test + public void testRandFunction() { + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand(db1.tbl1.k1) from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("The param of rand function must be literal")); + } + + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand(1234) from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); + } + + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand() from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); + } + + try { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select rand(\"hello\") from db1.tbl1;"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + } catch (Exception e) { + Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); + } + } } From 608037f3e437a3d458a007393fe108f2995694d2 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 25 Aug 2020 20:48:48 +0800 Subject: [PATCH 10/16] validate funtion rand with literal in compile step --- .../test/java/org/apache/doris/analysis/ExprTest.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java index da966697800f3a..991cf79fcb009e 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -24,16 +24,11 @@ import com.google.common.collect.Maps; -import org.apache.doris.mysql.privilege.MockedAuth; -import org.apache.doris.mysql.privilege.PaloAuth; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.DorisAssert; -import org.apache.doris.utframe.UtFrameUtils; import org.junit.Assert; import org.junit.Test; -import java.io.IOException; -import java.util.*; +import java.util.Map; +import java.util.Set; import mockit.Expectations; import mockit.Injectable; From a3ee01eba6d446b92ec03914939c499321770442 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 25 Aug 2020 20:49:46 +0800 Subject: [PATCH 11/16] validate funtion rand with literal in compile step --- fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java old mode 100755 new mode 100644 From 3089445ba5e1eaede342d0b72c979f0e0399ab91 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 25 Aug 2020 21:39:48 +0800 Subject: [PATCH 12/16] validate funtion rand with literal in compile step --- .../apache/doris/analysis/SelectStmtTest.java | 646 +++++++++--------- 1 file changed, 313 insertions(+), 333 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java index bd4eceaeb6d0f8..909bb83e8b0856 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -54,347 +54,327 @@ public static void setUp() throws Exception { dorisAssert.withDatabase("db1").useDatabase("db1"); dorisAssert.withTable(createTblStmtStr).withTable(createBaseAllStmtStr); } +// +// @Test +// public void testGroupingSets() throws Exception { +// ConnectContext ctx = UtFrameUtils.createDefaultCtx(); +// String selectStmtStr = "select k1,k2,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k2),(k1),(k2),());"; +// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); +// String selectStmtStr2 = "select k1,k4,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; +// expectedEx.expect(AnalysisException.class); +// expectedEx.expectMessage("column: `k4` cannot both in select list and aggregate functions when using GROUPING" +// + " SETS/CUBE/ROLLUP, please use union instead."); +// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr2, ctx); +// String selectStmtStr3 = "select k1,k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; +// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr3, ctx); +// String selectStmtStr4 = "select k1,k4+k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),()" +// + ");"; +// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr4, ctx); +// } +// +// @Test +// public void testSubqueryInCase() throws Exception { +// ConnectContext ctx = UtFrameUtils.createDefaultCtx(); +// String sql1 = "SELECT CASE\n" + +// " WHEN (\n" + +// " SELECT COUNT(*) / 2\n" + +// " FROM db1.tbl1\n" + +// " ) > k4 THEN (\n" + +// " SELECT AVG(k4)\n" + +// " FROM db1.tbl1\n" + +// " )\n" + +// " ELSE (\n" + +// " SELECT SUM(k4)\n" + +// " FROM db1.tbl1\n" + +// " )\n" + +// " END AS kk4\n" + +// "FROM db1.tbl1;"; +// SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql1, ctx); +// stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt.toSql().contains("`$a$1`.`$c$1` > `k4` THEN `$a$2`.`$c$2` ELSE `$a$3`.`$c$3`")); +// +// String sql2 = "select case when k1 in (select k1 from db1.tbl1) then \"true\" else k1 end a from db1.tbl1"; +// try { +// SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); +// stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.fail("syntax not supported."); +// } catch (AnalysisException e) { +// } catch (Exception e) { +// Assert.fail("must be AnalysisException."); +// } +// try { +// String sql3 = "select case k1 when exists (select 1) then \"empty\" else \"p_test\" end a from db1.tbl1"; +// SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); +// stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.fail("syntax not supported."); +// } catch (AnalysisException e) { +// } catch (Exception e) { +// Assert.fail("must be AnalysisException."); +// } +// String sql4 = "select case when k1 < (select max(k1) from db1.tbl1) and " + +// "k1 > (select min(k1) from db1.tbl1) then \"empty\" else \"p_test\" end a from db1.tbl1"; +// SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); +// stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt4.toSql().contains(" (`k1` < `$a$1`.`$c$1`) AND (`k1` > `$a$2`.`$c$2`) ")); +// +// String sql5 = "select case when k1 < (select max(k1) from db1.tbl1) is null " + +// "then \"empty\" else \"p_test\" end a from db1.tbl1"; +// SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); +// stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt5.toSql().contains(" `k1` < `$a$1`.`$c$1` IS NULL ")); +// } +// +// @Test +// public void testDeduplicateOrs() throws Exception { +// ConnectContext ctx = UtFrameUtils.createDefaultCtx(); +// String sql = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2,\n" + +// " db1.tbl1 t3,\n" + +// " db1.tbl1 t4,\n" + +// " db1.tbl1 t5,\n" + +// " db1.tbl1 t6\n" + +// "where\n" + +// " t2.k1 = t1.k1\n" + +// " and t1.k2 = t6.k2\n" + +// " and t6.k4 = 2001\n" + +// " and(\n" + +// " (\n" + +// " t1.k2 = t4.k2\n" + +// " and t3.k3 = t1.k3\n" + +// " and t3.k1 = 'D'\n" + +// " and t4.k3 = '2 yr Degree'\n" + +// " and t1.k4 between 100.00\n" + +// " and 150.00\n" + +// " and t4.k4 = 3\n" + +// " )\n" + +// " or (\n" + +// " t1.k2 = t4.k2\n" + +// " and t3.k3 = t1.k3\n" + +// " and t3.k1 = 'S'\n" + +// " and t4.k3 = 'Secondary'\n" + +// " and t1.k4 between 50.00\n" + +// " and 100.00\n" + +// " and t4.k4 = 1\n" + +// " )\n" + +// " or (\n" + +// " t1.k2 = t4.k2\n" + +// " and t3.k3 = t1.k3\n" + +// " and t3.k1 = 'W'\n" + +// " and t4.k3 = 'Advanced Degree'\n" + +// " and t1.k4 between 150.00\n" + +// " and 200.00\n" + +// " and t4.k4 = 1\n" + +// " )\n" + +// " )\n" + +// " and(\n" + +// " (\n" + +// " t1.k1 = t5.k1\n" + +// " and t5.k2 = 'United States'\n" + +// " and t5.k3 in ('CO', 'IL', 'MN')\n" + +// " and t1.k4 between 100\n" + +// " and 200\n" + +// " )\n" + +// " or (\n" + +// " t1.k1 = t5.k1\n" + +// " and t5.k2 = 'United States'\n" + +// " and t5.k3 in ('OH', 'MT', 'NM')\n" + +// " and t1.k4 between 150\n" + +// " and 300\n" + +// " )\n" + +// " or (\n" + +// " t1.k1 = t5.k1\n" + +// " and t5.k2 = 'United States'\n" + +// " and t5.k3 in ('TX', 'MO', 'MI')\n" + +// " and t1.k4 between 50 and 250\n" + +// " )\n" + +// " );"; +// SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx); +// stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// String rewritedFragment1 = "(((`t1`.`k2` = `t4`.`k2`) AND (`t3`.`k3` = `t1`.`k3`)) AND ((((((`t3`.`k1` = 'D')" + +// " AND (`t4`.`k3` = '2 yr Degree')) AND ((`t1`.`k4` >= 100.00) AND (`t1`.`k4` <= 150.00))) AND" + +// " (`t4`.`k4` = 3)) OR ((((`t3`.`k1` = 'S') AND (`t4`.`k3` = 'Secondary')) AND ((`t1`.`k4` >= 50.00)" + +// " AND (`t1`.`k4` <= 100.00))) AND (`t4`.`k4` = 1))) OR ((((`t3`.`k1` = 'W') AND " + +// "(`t4`.`k3` = 'Advanced Degree')) AND ((`t1`.`k4` >= 150.00) AND (`t1`.`k4` <= 200.00)))" + +// " AND (`t4`.`k4` = 1))))"; +// String rewritedFragment2 = "(((`t1`.`k1` = `t5`.`k1`) AND (`t5`.`k2` = 'United States')) AND" + +// " ((((`t5`.`k3` IN ('CO', 'IL', 'MN')) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200)))" + +// " OR ((`t5`.`k3` IN ('OH', 'MT', 'NM')) AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300))))" + +// " OR ((`t5`.`k3` IN ('TX', 'MO', 'MI')) AND ((`t1`.`k4` >= 50) AND (`t1`.`k4` <= 250)))))"; +// Assert.assertTrue(stmt.toSql().contains(rewritedFragment1)); +// Assert.assertTrue(stmt.toSql().contains(rewritedFragment2)); +// +// String sql2 = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2\n" + +// "where\n" + +// "(\n" + +// " t1.k1 = t2.k3\n" + +// " and t2.k2 = 'United States'\n" + +// " and t2.k3 in ('CO', 'IL', 'MN')\n" + +// " and t1.k4 between 100\n" + +// " and 200\n" + +// ")\n" + +// "or (\n" + +// " t1.k1 = t2.k1\n" + +// " and t2.k2 = 'United States1'\n" + +// " and t2.k3 in ('OH', 'MT', 'NM')\n" + +// " and t1.k4 between 150\n" + +// " and 300\n" + +// ")\n" + +// "or (\n" + +// " t1.k1 = t2.k1\n" + +// " and t2.k2 = 'United States'\n" + +// " and t2.k3 in ('TX', 'MO', 'MI')\n" + +// " and t1.k4 between 50 and 250\n" + +// ")"; +// SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); +// stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// String fragment3 = "(((((`t1`.`k1` = `t2`.`k3`) AND (`t2`.`k2` = 'United States')) AND " + +// "(`t2`.`k3` IN ('CO', 'IL', 'MN'))) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200))) OR" + +// " ((((`t1`.`k1` = `t2`.`k1`) AND (`t2`.`k2` = 'United States1')) AND (`t2`.`k3` IN ('OH', 'MT', 'NM')))" + +// " AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300)))) OR ((((`t1`.`k1` = `t2`.`k1`) AND " + +// "(`t2`.`k2` = 'United States')) AND (`t2`.`k3` IN ('TX', 'MO', 'MI'))) AND ((`t1`.`k4` >= 50)" + +// " AND (`t1`.`k4` <= 250)))"; +// Assert.assertTrue(stmt2.toSql().contains(fragment3)); +// +// String sql3 = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2\n" + +// "where\n" + +// " t1.k1 = t2.k3 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; +// SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); +// stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertFalse(stmt3.toSql().contains("((`t1`.`k1` = `t2`.`k3`) OR (`t1`.`k1` = `t2`.`k3`)) OR" + +// " (`t1`.`k1` = `t2`.`k3`)")); +// +// String sql4 = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2\n" + +// "where\n" + +// " t1.k1 = t2.k2 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; +// SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); +// stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt4.toSql().contains("(`t1`.`k1` = `t2`.`k2`) OR (`t1`.`k1` = `t2`.`k3`)")); +// +// String sql5 = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2\n" + +// "where\n" + +// " t2.k1 is not null or t1.k1 is not null or t1.k1 is not null"; +// SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); +// stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt5.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); +// Assert.assertEquals(2, stmt5.toSql().split(" OR ").length); +// +// String sql6 = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2\n" + +// "where\n" + +// " t2.k1 is not null or t1.k1 is not null and t1.k1 is not null"; +// SelectStmt stmt6 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql6, ctx); +// stmt6.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt6.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); +// Assert.assertEquals(2, stmt6.toSql().split(" OR ").length); +// +// String sql7 = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2\n" + +// "where\n" + +// " t2.k1 is not null or t1.k1 is not null and t1.k2 is not null"; +// SelectStmt stmt7 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql7, ctx); +// stmt7.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt7.toSql().contains("(`t2`.`k1` IS NOT NULL) OR ((`t1`.`k1` IS NOT NULL) " + +// "AND (`t1`.`k2` IS NOT NULL))")); +// +// String sql8 = "select\n" + +// " avg(t1.k4)\n" + +// "from\n" + +// " db1.tbl1 t1,\n" + +// " db1.tbl1 t2\n" + +// "where\n" + +// " t2.k1 is not null and t1.k1 is not null and t1.k1 is not null"; +// SelectStmt stmt8 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql8, ctx); +// stmt8.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt8.toSql().contains("((`t2`.`k1` IS NOT NULL) AND (`t1`.`k1` IS NOT NULL))" + +// " AND (`t1`.`k1` IS NOT NULL)")); +// +// String sql9 = "select * from db1.tbl1 where (k1='shutdown' and k4<1) or (k1='switchOff' and k4>=1)"; +// SelectStmt stmt9 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql9, ctx); +// stmt9.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); +// Assert.assertTrue(stmt9.toSql().contains("((`k1` = 'shutdown') AND (`k4` < 1))" + +// " OR ((`k1` = 'switchOff') AND (`k4` >= 1))")); +// } +// +// @Test +// public void testForbiddenCorrelatedSubqueryInHavingClause() throws Exception { +// String sql = "SELECT k1 FROM baseall GROUP BY k1 HAVING EXISTS(SELECT k4 FROM tbl1 GROUP BY k4 HAVING SUM" +// + "(baseall.k1) = k4);"; +// try { +// dorisAssert.query(sql).explainQuery(); +// Assert.fail("The correlated subquery in having clause should be forbidden."); +// } catch (AnalysisException e) { +// System.out.println(e.getMessage()); +// } +// } +// +// @Test +// public void testGroupByConstantExpression() throws Exception { +// String sql = "SELECT k1 - 4*60*60 FROM baseall GROUP BY k1 - 4*60*60"; +// dorisAssert.query(sql).explainQuery(); +// } +// +// @Test +// public void testMultrGroupByInCorrelationSubquery() throws Exception { +// String sql = "SELECT * from baseall where k1 > (select min(k1) from tbl1 where baseall.k1 = tbl1.k4 and baseall.k2 = tbl1.k2)"; +// dorisAssert.query(sql).explainQuery(); +// } +// +// @Test +// public void testOuterJoinNullUnionView() throws Exception{ +// String sql = "WITH test_view(k) AS(SELECT NULL AS k UNION ALL SELECT NULL AS k )\n" + +// "SELECT v1.k FROM test_view AS v1 LEFT OUTER JOIN test_view AS v2 ON v1.k=v2.k"; +// dorisAssert.query(sql).explainQuery(); +// } +// +// @Test +// public void testDataGripSupport() throws Exception { +// String sql = "select schema();"; +// dorisAssert.query(sql).explainQuery(); +// sql = "select\n" + +// "collation_name,\n" + +// "character_set_name,\n" + +// "is_default collate utf8_general_ci = 'Yes' as is_default\n" + +// "from information_schema.collations"; +// dorisAssert.query(sql).explainQuery(); +// } @Test - public void testGroupingSets() throws Exception { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select k1,k2,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k2),(k1),(k2),());"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - String selectStmtStr2 = "select k1,k4,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; - expectedEx.expect(AnalysisException.class); - expectedEx.expectMessage("column: `k4` cannot both in select list and aggregate functions when using GROUPING" - + " SETS/CUBE/ROLLUP, please use union instead."); - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr2, ctx); - String selectStmtStr3 = "select k1,k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr3, ctx); - String selectStmtStr4 = "select k1,k4+k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),()" - + ");"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr4, ctx); - } - - @Test - public void testSubqueryInCase() throws Exception { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String sql1 = "SELECT CASE\n" + - " WHEN (\n" + - " SELECT COUNT(*) / 2\n" + - " FROM db1.tbl1\n" + - " ) > k4 THEN (\n" + - " SELECT AVG(k4)\n" + - " FROM db1.tbl1\n" + - " )\n" + - " ELSE (\n" + - " SELECT SUM(k4)\n" + - " FROM db1.tbl1\n" + - " )\n" + - " END AS kk4\n" + - "FROM db1.tbl1;"; - SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql1, ctx); - stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt.toSql().contains("`$a$1`.`$c$1` > `k4` THEN `$a$2`.`$c$2` ELSE `$a$3`.`$c$3`")); - - String sql2 = "select case when k1 in (select k1 from db1.tbl1) then \"true\" else k1 end a from db1.tbl1"; - try { - SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); - stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.fail("syntax not supported."); - } catch (AnalysisException e) { - } catch (Exception e) { - Assert.fail("must be AnalysisException."); - } - try { - String sql3 = "select case k1 when exists (select 1) then \"empty\" else \"p_test\" end a from db1.tbl1"; - SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); - stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.fail("syntax not supported."); - } catch (AnalysisException e) { - } catch (Exception e) { - Assert.fail("must be AnalysisException."); - } - String sql4 = "select case when k1 < (select max(k1) from db1.tbl1) and " + - "k1 > (select min(k1) from db1.tbl1) then \"empty\" else \"p_test\" end a from db1.tbl1"; - SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); - stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt4.toSql().contains(" (`k1` < `$a$1`.`$c$1`) AND (`k1` > `$a$2`.`$c$2`) ")); - - String sql5 = "select case when k1 < (select max(k1) from db1.tbl1) is null " + - "then \"empty\" else \"p_test\" end a from db1.tbl1"; - SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); - stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt5.toSql().contains(" `k1` < `$a$1`.`$c$1` IS NULL ")); - } - - @Test - public void testDeduplicateOrs() throws Exception { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String sql = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2,\n" + - " db1.tbl1 t3,\n" + - " db1.tbl1 t4,\n" + - " db1.tbl1 t5,\n" + - " db1.tbl1 t6\n" + - "where\n" + - " t2.k1 = t1.k1\n" + - " and t1.k2 = t6.k2\n" + - " and t6.k4 = 2001\n" + - " and(\n" + - " (\n" + - " t1.k2 = t4.k2\n" + - " and t3.k3 = t1.k3\n" + - " and t3.k1 = 'D'\n" + - " and t4.k3 = '2 yr Degree'\n" + - " and t1.k4 between 100.00\n" + - " and 150.00\n" + - " and t4.k4 = 3\n" + - " )\n" + - " or (\n" + - " t1.k2 = t4.k2\n" + - " and t3.k3 = t1.k3\n" + - " and t3.k1 = 'S'\n" + - " and t4.k3 = 'Secondary'\n" + - " and t1.k4 between 50.00\n" + - " and 100.00\n" + - " and t4.k4 = 1\n" + - " )\n" + - " or (\n" + - " t1.k2 = t4.k2\n" + - " and t3.k3 = t1.k3\n" + - " and t3.k1 = 'W'\n" + - " and t4.k3 = 'Advanced Degree'\n" + - " and t1.k4 between 150.00\n" + - " and 200.00\n" + - " and t4.k4 = 1\n" + - " )\n" + - " )\n" + - " and(\n" + - " (\n" + - " t1.k1 = t5.k1\n" + - " and t5.k2 = 'United States'\n" + - " and t5.k3 in ('CO', 'IL', 'MN')\n" + - " and t1.k4 between 100\n" + - " and 200\n" + - " )\n" + - " or (\n" + - " t1.k1 = t5.k1\n" + - " and t5.k2 = 'United States'\n" + - " and t5.k3 in ('OH', 'MT', 'NM')\n" + - " and t1.k4 between 150\n" + - " and 300\n" + - " )\n" + - " or (\n" + - " t1.k1 = t5.k1\n" + - " and t5.k2 = 'United States'\n" + - " and t5.k3 in ('TX', 'MO', 'MI')\n" + - " and t1.k4 between 50 and 250\n" + - " )\n" + - " );"; - SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx); - stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - String rewritedFragment1 = "(((`t1`.`k2` = `t4`.`k2`) AND (`t3`.`k3` = `t1`.`k3`)) AND ((((((`t3`.`k1` = 'D')" + - " AND (`t4`.`k3` = '2 yr Degree')) AND ((`t1`.`k4` >= 100.00) AND (`t1`.`k4` <= 150.00))) AND" + - " (`t4`.`k4` = 3)) OR ((((`t3`.`k1` = 'S') AND (`t4`.`k3` = 'Secondary')) AND ((`t1`.`k4` >= 50.00)" + - " AND (`t1`.`k4` <= 100.00))) AND (`t4`.`k4` = 1))) OR ((((`t3`.`k1` = 'W') AND " + - "(`t4`.`k3` = 'Advanced Degree')) AND ((`t1`.`k4` >= 150.00) AND (`t1`.`k4` <= 200.00)))" + - " AND (`t4`.`k4` = 1))))"; - String rewritedFragment2 = "(((`t1`.`k1` = `t5`.`k1`) AND (`t5`.`k2` = 'United States')) AND" + - " ((((`t5`.`k3` IN ('CO', 'IL', 'MN')) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200)))" + - " OR ((`t5`.`k3` IN ('OH', 'MT', 'NM')) AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300))))" + - " OR ((`t5`.`k3` IN ('TX', 'MO', 'MI')) AND ((`t1`.`k4` >= 50) AND (`t1`.`k4` <= 250)))))"; - Assert.assertTrue(stmt.toSql().contains(rewritedFragment1)); - Assert.assertTrue(stmt.toSql().contains(rewritedFragment2)); - - String sql2 = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2\n" + - "where\n" + - "(\n" + - " t1.k1 = t2.k3\n" + - " and t2.k2 = 'United States'\n" + - " and t2.k3 in ('CO', 'IL', 'MN')\n" + - " and t1.k4 between 100\n" + - " and 200\n" + - ")\n" + - "or (\n" + - " t1.k1 = t2.k1\n" + - " and t2.k2 = 'United States1'\n" + - " and t2.k3 in ('OH', 'MT', 'NM')\n" + - " and t1.k4 between 150\n" + - " and 300\n" + - ")\n" + - "or (\n" + - " t1.k1 = t2.k1\n" + - " and t2.k2 = 'United States'\n" + - " and t2.k3 in ('TX', 'MO', 'MI')\n" + - " and t1.k4 between 50 and 250\n" + - ")"; - SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); - stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - String fragment3 = "(((((`t1`.`k1` = `t2`.`k3`) AND (`t2`.`k2` = 'United States')) AND " + - "(`t2`.`k3` IN ('CO', 'IL', 'MN'))) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200))) OR" + - " ((((`t1`.`k1` = `t2`.`k1`) AND (`t2`.`k2` = 'United States1')) AND (`t2`.`k3` IN ('OH', 'MT', 'NM')))" + - " AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300)))) OR ((((`t1`.`k1` = `t2`.`k1`) AND " + - "(`t2`.`k2` = 'United States')) AND (`t2`.`k3` IN ('TX', 'MO', 'MI'))) AND ((`t1`.`k4` >= 50)" + - " AND (`t1`.`k4` <= 250)))"; - Assert.assertTrue(stmt2.toSql().contains(fragment3)); - - String sql3 = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2\n" + - "where\n" + - " t1.k1 = t2.k3 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; - SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); - stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertFalse(stmt3.toSql().contains("((`t1`.`k1` = `t2`.`k3`) OR (`t1`.`k1` = `t2`.`k3`)) OR" + - " (`t1`.`k1` = `t2`.`k3`)")); - - String sql4 = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2\n" + - "where\n" + - " t1.k1 = t2.k2 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; - SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); - stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt4.toSql().contains("(`t1`.`k1` = `t2`.`k2`) OR (`t1`.`k1` = `t2`.`k3`)")); - - String sql5 = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2\n" + - "where\n" + - " t2.k1 is not null or t1.k1 is not null or t1.k1 is not null"; - SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); - stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt5.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); - Assert.assertEquals(2, stmt5.toSql().split(" OR ").length); - - String sql6 = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2\n" + - "where\n" + - " t2.k1 is not null or t1.k1 is not null and t1.k1 is not null"; - SelectStmt stmt6 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql6, ctx); - stmt6.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt6.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); - Assert.assertEquals(2, stmt6.toSql().split(" OR ").length); - - String sql7 = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2\n" + - "where\n" + - " t2.k1 is not null or t1.k1 is not null and t1.k2 is not null"; - SelectStmt stmt7 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql7, ctx); - stmt7.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt7.toSql().contains("(`t2`.`k1` IS NOT NULL) OR ((`t1`.`k1` IS NOT NULL) " + - "AND (`t1`.`k2` IS NOT NULL))")); - - String sql8 = "select\n" + - " avg(t1.k4)\n" + - "from\n" + - " db1.tbl1 t1,\n" + - " db1.tbl1 t2\n" + - "where\n" + - " t2.k1 is not null and t1.k1 is not null and t1.k1 is not null"; - SelectStmt stmt8 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql8, ctx); - stmt8.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt8.toSql().contains("((`t2`.`k1` IS NOT NULL) AND (`t1`.`k1` IS NOT NULL))" + - " AND (`t1`.`k1` IS NOT NULL)")); - - String sql9 = "select * from db1.tbl1 where (k1='shutdown' and k4<1) or (k1='switchOff' and k4>=1)"; - SelectStmt stmt9 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql9, ctx); - stmt9.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); - Assert.assertTrue(stmt9.toSql().contains("((`k1` = 'shutdown') AND (`k4` < 1))" + - " OR ((`k1` = 'switchOff') AND (`k4` >= 1))")); - } - - @Test - public void testForbiddenCorrelatedSubqueryInHavingClause() throws Exception { - String sql = "SELECT k1 FROM baseall GROUP BY k1 HAVING EXISTS(SELECT k4 FROM tbl1 GROUP BY k4 HAVING SUM" - + "(baseall.k1) = k4);"; + public void testRandFunction() throws Exception { + String sql = "select rand(db1.tbl1.k1) from db1.tbl1;"; try { dorisAssert.query(sql).explainQuery(); - Assert.fail("The correlated subquery in having clause should be forbidden."); + Assert.fail("The param of rand function must be literal"); } catch (AnalysisException e) { System.out.println(e.getMessage()); } - } - - @Test - public void testGroupByConstantExpression() throws Exception { - String sql = "SELECT k1 - 4*60*60 FROM baseall GROUP BY k1 - 4*60*60"; - dorisAssert.query(sql).explainQuery(); - } - - @Test - public void testMultrGroupByInCorrelationSubquery() throws Exception { - String sql = "SELECT * from baseall where k1 > (select min(k1) from tbl1 where baseall.k1 = tbl1.k4 and baseall.k2 = tbl1.k2)"; - dorisAssert.query(sql).explainQuery(); - } - - @Test - public void testOuterJoinNullUnionView() throws Exception{ - String sql = "WITH test_view(k) AS(SELECT NULL AS k UNION ALL SELECT NULL AS k )\n" + - "SELECT v1.k FROM test_view AS v1 LEFT OUTER JOIN test_view AS v2 ON v1.k=v2.k"; + sql = "select rand(1234) from db1.tbl1;"; dorisAssert.query(sql).explainQuery(); - } - - @Test - public void testDataGripSupport() throws Exception { - String sql = "select schema();"; + sql = "select rand() from db1.tbl1;"; dorisAssert.query(sql).explainQuery(); - sql = "select\n" + - "collation_name,\n" + - "character_set_name,\n" + - "is_default collate utf8_general_ci = 'Yes' as is_default\n" + - "from information_schema.collations"; - dorisAssert.query(sql).explainQuery(); - } - - @Test - public void testRandFunction() { - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand(db1.tbl1.k1) from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("The param of rand function must be literal")); - } - - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand(1234) from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); - } - - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand() from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); - } - - try { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String selectStmtStr = "select rand(\"hello\") from db1.tbl1;"; - UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); - } catch (Exception e) { - Assert.assertFalse(e.getMessage().contains("The param of rand function must be literal")); - } } } From c6f7a9cf6e681053707ea155980d315f00da2e68 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 25 Aug 2020 21:40:36 +0800 Subject: [PATCH 13/16] validate funtion rand with literal in compile step --- fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index dbfd2aa3dcd74f..582bb30c8c5efe 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -1407,8 +1407,7 @@ protected Function getBuiltinFunction( Function searchDesc = new Function(fnName, argTypes, Type.INVALID, false); Function f = Catalog.getCurrentCatalog().getFunction(searchDesc, mode); if (f != null && fnName.getFunction().equalsIgnoreCase("rand")) { - if (this.children != null - && this.children.size() == 1 + if (this.children.size() == 1 && !(this.children.get(0) instanceof LiteralExpr)) { throw new AnalysisException("The param of rand function must be literal"); } From a2f9673fbd2e84bb0e88d275254baaf09645b6b9 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 25 Aug 2020 21:42:00 +0800 Subject: [PATCH 14/16] validate funtion rand with literal in compile step --- .../apache/doris/analysis/SelectStmtTest.java | 616 +++++++++--------- 1 file changed, 308 insertions(+), 308 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java index 909bb83e8b0856..5bb372f5b2a579 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -54,314 +54,314 @@ public static void setUp() throws Exception { dorisAssert.withDatabase("db1").useDatabase("db1"); dorisAssert.withTable(createTblStmtStr).withTable(createBaseAllStmtStr); } -// -// @Test -// public void testGroupingSets() throws Exception { -// ConnectContext ctx = UtFrameUtils.createDefaultCtx(); -// String selectStmtStr = "select k1,k2,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k2),(k1),(k2),());"; -// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); -// String selectStmtStr2 = "select k1,k4,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; -// expectedEx.expect(AnalysisException.class); -// expectedEx.expectMessage("column: `k4` cannot both in select list and aggregate functions when using GROUPING" -// + " SETS/CUBE/ROLLUP, please use union instead."); -// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr2, ctx); -// String selectStmtStr3 = "select k1,k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; -// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr3, ctx); -// String selectStmtStr4 = "select k1,k4+k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),()" -// + ");"; -// UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr4, ctx); -// } -// -// @Test -// public void testSubqueryInCase() throws Exception { -// ConnectContext ctx = UtFrameUtils.createDefaultCtx(); -// String sql1 = "SELECT CASE\n" + -// " WHEN (\n" + -// " SELECT COUNT(*) / 2\n" + -// " FROM db1.tbl1\n" + -// " ) > k4 THEN (\n" + -// " SELECT AVG(k4)\n" + -// " FROM db1.tbl1\n" + -// " )\n" + -// " ELSE (\n" + -// " SELECT SUM(k4)\n" + -// " FROM db1.tbl1\n" + -// " )\n" + -// " END AS kk4\n" + -// "FROM db1.tbl1;"; -// SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql1, ctx); -// stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt.toSql().contains("`$a$1`.`$c$1` > `k4` THEN `$a$2`.`$c$2` ELSE `$a$3`.`$c$3`")); -// -// String sql2 = "select case when k1 in (select k1 from db1.tbl1) then \"true\" else k1 end a from db1.tbl1"; -// try { -// SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); -// stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.fail("syntax not supported."); -// } catch (AnalysisException e) { -// } catch (Exception e) { -// Assert.fail("must be AnalysisException."); -// } -// try { -// String sql3 = "select case k1 when exists (select 1) then \"empty\" else \"p_test\" end a from db1.tbl1"; -// SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); -// stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.fail("syntax not supported."); -// } catch (AnalysisException e) { -// } catch (Exception e) { -// Assert.fail("must be AnalysisException."); -// } -// String sql4 = "select case when k1 < (select max(k1) from db1.tbl1) and " + -// "k1 > (select min(k1) from db1.tbl1) then \"empty\" else \"p_test\" end a from db1.tbl1"; -// SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); -// stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt4.toSql().contains(" (`k1` < `$a$1`.`$c$1`) AND (`k1` > `$a$2`.`$c$2`) ")); -// -// String sql5 = "select case when k1 < (select max(k1) from db1.tbl1) is null " + -// "then \"empty\" else \"p_test\" end a from db1.tbl1"; -// SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); -// stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt5.toSql().contains(" `k1` < `$a$1`.`$c$1` IS NULL ")); -// } -// -// @Test -// public void testDeduplicateOrs() throws Exception { -// ConnectContext ctx = UtFrameUtils.createDefaultCtx(); -// String sql = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2,\n" + -// " db1.tbl1 t3,\n" + -// " db1.tbl1 t4,\n" + -// " db1.tbl1 t5,\n" + -// " db1.tbl1 t6\n" + -// "where\n" + -// " t2.k1 = t1.k1\n" + -// " and t1.k2 = t6.k2\n" + -// " and t6.k4 = 2001\n" + -// " and(\n" + -// " (\n" + -// " t1.k2 = t4.k2\n" + -// " and t3.k3 = t1.k3\n" + -// " and t3.k1 = 'D'\n" + -// " and t4.k3 = '2 yr Degree'\n" + -// " and t1.k4 between 100.00\n" + -// " and 150.00\n" + -// " and t4.k4 = 3\n" + -// " )\n" + -// " or (\n" + -// " t1.k2 = t4.k2\n" + -// " and t3.k3 = t1.k3\n" + -// " and t3.k1 = 'S'\n" + -// " and t4.k3 = 'Secondary'\n" + -// " and t1.k4 between 50.00\n" + -// " and 100.00\n" + -// " and t4.k4 = 1\n" + -// " )\n" + -// " or (\n" + -// " t1.k2 = t4.k2\n" + -// " and t3.k3 = t1.k3\n" + -// " and t3.k1 = 'W'\n" + -// " and t4.k3 = 'Advanced Degree'\n" + -// " and t1.k4 between 150.00\n" + -// " and 200.00\n" + -// " and t4.k4 = 1\n" + -// " )\n" + -// " )\n" + -// " and(\n" + -// " (\n" + -// " t1.k1 = t5.k1\n" + -// " and t5.k2 = 'United States'\n" + -// " and t5.k3 in ('CO', 'IL', 'MN')\n" + -// " and t1.k4 between 100\n" + -// " and 200\n" + -// " )\n" + -// " or (\n" + -// " t1.k1 = t5.k1\n" + -// " and t5.k2 = 'United States'\n" + -// " and t5.k3 in ('OH', 'MT', 'NM')\n" + -// " and t1.k4 between 150\n" + -// " and 300\n" + -// " )\n" + -// " or (\n" + -// " t1.k1 = t5.k1\n" + -// " and t5.k2 = 'United States'\n" + -// " and t5.k3 in ('TX', 'MO', 'MI')\n" + -// " and t1.k4 between 50 and 250\n" + -// " )\n" + -// " );"; -// SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx); -// stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// String rewritedFragment1 = "(((`t1`.`k2` = `t4`.`k2`) AND (`t3`.`k3` = `t1`.`k3`)) AND ((((((`t3`.`k1` = 'D')" + -// " AND (`t4`.`k3` = '2 yr Degree')) AND ((`t1`.`k4` >= 100.00) AND (`t1`.`k4` <= 150.00))) AND" + -// " (`t4`.`k4` = 3)) OR ((((`t3`.`k1` = 'S') AND (`t4`.`k3` = 'Secondary')) AND ((`t1`.`k4` >= 50.00)" + -// " AND (`t1`.`k4` <= 100.00))) AND (`t4`.`k4` = 1))) OR ((((`t3`.`k1` = 'W') AND " + -// "(`t4`.`k3` = 'Advanced Degree')) AND ((`t1`.`k4` >= 150.00) AND (`t1`.`k4` <= 200.00)))" + -// " AND (`t4`.`k4` = 1))))"; -// String rewritedFragment2 = "(((`t1`.`k1` = `t5`.`k1`) AND (`t5`.`k2` = 'United States')) AND" + -// " ((((`t5`.`k3` IN ('CO', 'IL', 'MN')) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200)))" + -// " OR ((`t5`.`k3` IN ('OH', 'MT', 'NM')) AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300))))" + -// " OR ((`t5`.`k3` IN ('TX', 'MO', 'MI')) AND ((`t1`.`k4` >= 50) AND (`t1`.`k4` <= 250)))))"; -// Assert.assertTrue(stmt.toSql().contains(rewritedFragment1)); -// Assert.assertTrue(stmt.toSql().contains(rewritedFragment2)); -// -// String sql2 = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2\n" + -// "where\n" + -// "(\n" + -// " t1.k1 = t2.k3\n" + -// " and t2.k2 = 'United States'\n" + -// " and t2.k3 in ('CO', 'IL', 'MN')\n" + -// " and t1.k4 between 100\n" + -// " and 200\n" + -// ")\n" + -// "or (\n" + -// " t1.k1 = t2.k1\n" + -// " and t2.k2 = 'United States1'\n" + -// " and t2.k3 in ('OH', 'MT', 'NM')\n" + -// " and t1.k4 between 150\n" + -// " and 300\n" + -// ")\n" + -// "or (\n" + -// " t1.k1 = t2.k1\n" + -// " and t2.k2 = 'United States'\n" + -// " and t2.k3 in ('TX', 'MO', 'MI')\n" + -// " and t1.k4 between 50 and 250\n" + -// ")"; -// SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); -// stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// String fragment3 = "(((((`t1`.`k1` = `t2`.`k3`) AND (`t2`.`k2` = 'United States')) AND " + -// "(`t2`.`k3` IN ('CO', 'IL', 'MN'))) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200))) OR" + -// " ((((`t1`.`k1` = `t2`.`k1`) AND (`t2`.`k2` = 'United States1')) AND (`t2`.`k3` IN ('OH', 'MT', 'NM')))" + -// " AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300)))) OR ((((`t1`.`k1` = `t2`.`k1`) AND " + -// "(`t2`.`k2` = 'United States')) AND (`t2`.`k3` IN ('TX', 'MO', 'MI'))) AND ((`t1`.`k4` >= 50)" + -// " AND (`t1`.`k4` <= 250)))"; -// Assert.assertTrue(stmt2.toSql().contains(fragment3)); -// -// String sql3 = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2\n" + -// "where\n" + -// " t1.k1 = t2.k3 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; -// SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); -// stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertFalse(stmt3.toSql().contains("((`t1`.`k1` = `t2`.`k3`) OR (`t1`.`k1` = `t2`.`k3`)) OR" + -// " (`t1`.`k1` = `t2`.`k3`)")); -// -// String sql4 = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2\n" + -// "where\n" + -// " t1.k1 = t2.k2 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; -// SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); -// stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt4.toSql().contains("(`t1`.`k1` = `t2`.`k2`) OR (`t1`.`k1` = `t2`.`k3`)")); -// -// String sql5 = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2\n" + -// "where\n" + -// " t2.k1 is not null or t1.k1 is not null or t1.k1 is not null"; -// SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); -// stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt5.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); -// Assert.assertEquals(2, stmt5.toSql().split(" OR ").length); -// -// String sql6 = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2\n" + -// "where\n" + -// " t2.k1 is not null or t1.k1 is not null and t1.k1 is not null"; -// SelectStmt stmt6 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql6, ctx); -// stmt6.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt6.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); -// Assert.assertEquals(2, stmt6.toSql().split(" OR ").length); -// -// String sql7 = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2\n" + -// "where\n" + -// " t2.k1 is not null or t1.k1 is not null and t1.k2 is not null"; -// SelectStmt stmt7 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql7, ctx); -// stmt7.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt7.toSql().contains("(`t2`.`k1` IS NOT NULL) OR ((`t1`.`k1` IS NOT NULL) " + -// "AND (`t1`.`k2` IS NOT NULL))")); -// -// String sql8 = "select\n" + -// " avg(t1.k4)\n" + -// "from\n" + -// " db1.tbl1 t1,\n" + -// " db1.tbl1 t2\n" + -// "where\n" + -// " t2.k1 is not null and t1.k1 is not null and t1.k1 is not null"; -// SelectStmt stmt8 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql8, ctx); -// stmt8.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt8.toSql().contains("((`t2`.`k1` IS NOT NULL) AND (`t1`.`k1` IS NOT NULL))" + -// " AND (`t1`.`k1` IS NOT NULL)")); -// -// String sql9 = "select * from db1.tbl1 where (k1='shutdown' and k4<1) or (k1='switchOff' and k4>=1)"; -// SelectStmt stmt9 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql9, ctx); -// stmt9.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); -// Assert.assertTrue(stmt9.toSql().contains("((`k1` = 'shutdown') AND (`k4` < 1))" + -// " OR ((`k1` = 'switchOff') AND (`k4` >= 1))")); -// } -// -// @Test -// public void testForbiddenCorrelatedSubqueryInHavingClause() throws Exception { -// String sql = "SELECT k1 FROM baseall GROUP BY k1 HAVING EXISTS(SELECT k4 FROM tbl1 GROUP BY k4 HAVING SUM" -// + "(baseall.k1) = k4);"; -// try { -// dorisAssert.query(sql).explainQuery(); -// Assert.fail("The correlated subquery in having clause should be forbidden."); -// } catch (AnalysisException e) { -// System.out.println(e.getMessage()); -// } -// } -// -// @Test -// public void testGroupByConstantExpression() throws Exception { -// String sql = "SELECT k1 - 4*60*60 FROM baseall GROUP BY k1 - 4*60*60"; -// dorisAssert.query(sql).explainQuery(); -// } -// -// @Test -// public void testMultrGroupByInCorrelationSubquery() throws Exception { -// String sql = "SELECT * from baseall where k1 > (select min(k1) from tbl1 where baseall.k1 = tbl1.k4 and baseall.k2 = tbl1.k2)"; -// dorisAssert.query(sql).explainQuery(); -// } -// -// @Test -// public void testOuterJoinNullUnionView() throws Exception{ -// String sql = "WITH test_view(k) AS(SELECT NULL AS k UNION ALL SELECT NULL AS k )\n" + -// "SELECT v1.k FROM test_view AS v1 LEFT OUTER JOIN test_view AS v2 ON v1.k=v2.k"; -// dorisAssert.query(sql).explainQuery(); -// } -// -// @Test -// public void testDataGripSupport() throws Exception { -// String sql = "select schema();"; -// dorisAssert.query(sql).explainQuery(); -// sql = "select\n" + -// "collation_name,\n" + -// "character_set_name,\n" + -// "is_default collate utf8_general_ci = 'Yes' as is_default\n" + -// "from information_schema.collations"; -// dorisAssert.query(sql).explainQuery(); -// } + + @Test + public void testGroupingSets() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String selectStmtStr = "select k1,k2,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k2),(k1),(k2),());"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr, ctx); + String selectStmtStr2 = "select k1,k4,MAX(k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; + expectedEx.expect(AnalysisException.class); + expectedEx.expectMessage("column: `k4` cannot both in select list and aggregate functions when using GROUPING" + + " SETS/CUBE/ROLLUP, please use union instead."); + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr2, ctx); + String selectStmtStr3 = "select k1,k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),());"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr3, ctx); + String selectStmtStr4 = "select k1,k4+k4,MAX(k4+k4) from db1.tbl1 GROUP BY GROUPING sets ((k1,k4),(k1),(k4),()" + + ");"; + UtFrameUtils.parseAndAnalyzeStmt(selectStmtStr4, ctx); + } + + @Test + public void testSubqueryInCase() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String sql1 = "SELECT CASE\n" + + " WHEN (\n" + + " SELECT COUNT(*) / 2\n" + + " FROM db1.tbl1\n" + + " ) > k4 THEN (\n" + + " SELECT AVG(k4)\n" + + " FROM db1.tbl1\n" + + " )\n" + + " ELSE (\n" + + " SELECT SUM(k4)\n" + + " FROM db1.tbl1\n" + + " )\n" + + " END AS kk4\n" + + "FROM db1.tbl1;"; + SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql1, ctx); + stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt.toSql().contains("`$a$1`.`$c$1` > `k4` THEN `$a$2`.`$c$2` ELSE `$a$3`.`$c$3`")); + + String sql2 = "select case when k1 in (select k1 from db1.tbl1) then \"true\" else k1 end a from db1.tbl1"; + try { + SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); + stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.fail("syntax not supported."); + } catch (AnalysisException e) { + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + try { + String sql3 = "select case k1 when exists (select 1) then \"empty\" else \"p_test\" end a from db1.tbl1"; + SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); + stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.fail("syntax not supported."); + } catch (AnalysisException e) { + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + String sql4 = "select case when k1 < (select max(k1) from db1.tbl1) and " + + "k1 > (select min(k1) from db1.tbl1) then \"empty\" else \"p_test\" end a from db1.tbl1"; + SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); + stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt4.toSql().contains(" (`k1` < `$a$1`.`$c$1`) AND (`k1` > `$a$2`.`$c$2`) ")); + + String sql5 = "select case when k1 < (select max(k1) from db1.tbl1) is null " + + "then \"empty\" else \"p_test\" end a from db1.tbl1"; + SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); + stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt5.toSql().contains(" `k1` < `$a$1`.`$c$1` IS NULL ")); + } + + @Test + public void testDeduplicateOrs() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + String sql = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2,\n" + + " db1.tbl1 t3,\n" + + " db1.tbl1 t4,\n" + + " db1.tbl1 t5,\n" + + " db1.tbl1 t6\n" + + "where\n" + + " t2.k1 = t1.k1\n" + + " and t1.k2 = t6.k2\n" + + " and t6.k4 = 2001\n" + + " and(\n" + + " (\n" + + " t1.k2 = t4.k2\n" + + " and t3.k3 = t1.k3\n" + + " and t3.k1 = 'D'\n" + + " and t4.k3 = '2 yr Degree'\n" + + " and t1.k4 between 100.00\n" + + " and 150.00\n" + + " and t4.k4 = 3\n" + + " )\n" + + " or (\n" + + " t1.k2 = t4.k2\n" + + " and t3.k3 = t1.k3\n" + + " and t3.k1 = 'S'\n" + + " and t4.k3 = 'Secondary'\n" + + " and t1.k4 between 50.00\n" + + " and 100.00\n" + + " and t4.k4 = 1\n" + + " )\n" + + " or (\n" + + " t1.k2 = t4.k2\n" + + " and t3.k3 = t1.k3\n" + + " and t3.k1 = 'W'\n" + + " and t4.k3 = 'Advanced Degree'\n" + + " and t1.k4 between 150.00\n" + + " and 200.00\n" + + " and t4.k4 = 1\n" + + " )\n" + + " )\n" + + " and(\n" + + " (\n" + + " t1.k1 = t5.k1\n" + + " and t5.k2 = 'United States'\n" + + " and t5.k3 in ('CO', 'IL', 'MN')\n" + + " and t1.k4 between 100\n" + + " and 200\n" + + " )\n" + + " or (\n" + + " t1.k1 = t5.k1\n" + + " and t5.k2 = 'United States'\n" + + " and t5.k3 in ('OH', 'MT', 'NM')\n" + + " and t1.k4 between 150\n" + + " and 300\n" + + " )\n" + + " or (\n" + + " t1.k1 = t5.k1\n" + + " and t5.k2 = 'United States'\n" + + " and t5.k3 in ('TX', 'MO', 'MI')\n" + + " and t1.k4 between 50 and 250\n" + + " )\n" + + " );"; + SelectStmt stmt = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx); + stmt.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + String rewritedFragment1 = "(((`t1`.`k2` = `t4`.`k2`) AND (`t3`.`k3` = `t1`.`k3`)) AND ((((((`t3`.`k1` = 'D')" + + " AND (`t4`.`k3` = '2 yr Degree')) AND ((`t1`.`k4` >= 100.00) AND (`t1`.`k4` <= 150.00))) AND" + + " (`t4`.`k4` = 3)) OR ((((`t3`.`k1` = 'S') AND (`t4`.`k3` = 'Secondary')) AND ((`t1`.`k4` >= 50.00)" + + " AND (`t1`.`k4` <= 100.00))) AND (`t4`.`k4` = 1))) OR ((((`t3`.`k1` = 'W') AND " + + "(`t4`.`k3` = 'Advanced Degree')) AND ((`t1`.`k4` >= 150.00) AND (`t1`.`k4` <= 200.00)))" + + " AND (`t4`.`k4` = 1))))"; + String rewritedFragment2 = "(((`t1`.`k1` = `t5`.`k1`) AND (`t5`.`k2` = 'United States')) AND" + + " ((((`t5`.`k3` IN ('CO', 'IL', 'MN')) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200)))" + + " OR ((`t5`.`k3` IN ('OH', 'MT', 'NM')) AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300))))" + + " OR ((`t5`.`k3` IN ('TX', 'MO', 'MI')) AND ((`t1`.`k4` >= 50) AND (`t1`.`k4` <= 250)))))"; + Assert.assertTrue(stmt.toSql().contains(rewritedFragment1)); + Assert.assertTrue(stmt.toSql().contains(rewritedFragment2)); + + String sql2 = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2\n" + + "where\n" + + "(\n" + + " t1.k1 = t2.k3\n" + + " and t2.k2 = 'United States'\n" + + " and t2.k3 in ('CO', 'IL', 'MN')\n" + + " and t1.k4 between 100\n" + + " and 200\n" + + ")\n" + + "or (\n" + + " t1.k1 = t2.k1\n" + + " and t2.k2 = 'United States1'\n" + + " and t2.k3 in ('OH', 'MT', 'NM')\n" + + " and t1.k4 between 150\n" + + " and 300\n" + + ")\n" + + "or (\n" + + " t1.k1 = t2.k1\n" + + " and t2.k2 = 'United States'\n" + + " and t2.k3 in ('TX', 'MO', 'MI')\n" + + " and t1.k4 between 50 and 250\n" + + ")"; + SelectStmt stmt2 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql2, ctx); + stmt2.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + String fragment3 = "(((((`t1`.`k1` = `t2`.`k3`) AND (`t2`.`k2` = 'United States')) AND " + + "(`t2`.`k3` IN ('CO', 'IL', 'MN'))) AND ((`t1`.`k4` >= 100) AND (`t1`.`k4` <= 200))) OR" + + " ((((`t1`.`k1` = `t2`.`k1`) AND (`t2`.`k2` = 'United States1')) AND (`t2`.`k3` IN ('OH', 'MT', 'NM')))" + + " AND ((`t1`.`k4` >= 150) AND (`t1`.`k4` <= 300)))) OR ((((`t1`.`k1` = `t2`.`k1`) AND " + + "(`t2`.`k2` = 'United States')) AND (`t2`.`k3` IN ('TX', 'MO', 'MI'))) AND ((`t1`.`k4` >= 50)" + + " AND (`t1`.`k4` <= 250)))"; + Assert.assertTrue(stmt2.toSql().contains(fragment3)); + + String sql3 = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2\n" + + "where\n" + + " t1.k1 = t2.k3 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; + SelectStmt stmt3 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql3, ctx); + stmt3.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertFalse(stmt3.toSql().contains("((`t1`.`k1` = `t2`.`k3`) OR (`t1`.`k1` = `t2`.`k3`)) OR" + + " (`t1`.`k1` = `t2`.`k3`)")); + + String sql4 = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2\n" + + "where\n" + + " t1.k1 = t2.k2 or t1.k1 = t2.k3 or t1.k1 = t2.k3"; + SelectStmt stmt4 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql4, ctx); + stmt4.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt4.toSql().contains("(`t1`.`k1` = `t2`.`k2`) OR (`t1`.`k1` = `t2`.`k3`)")); + + String sql5 = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2\n" + + "where\n" + + " t2.k1 is not null or t1.k1 is not null or t1.k1 is not null"; + SelectStmt stmt5 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql5, ctx); + stmt5.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt5.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); + Assert.assertEquals(2, stmt5.toSql().split(" OR ").length); + + String sql6 = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2\n" + + "where\n" + + " t2.k1 is not null or t1.k1 is not null and t1.k1 is not null"; + SelectStmt stmt6 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql6, ctx); + stmt6.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt6.toSql().contains("(`t2`.`k1` IS NOT NULL) OR (`t1`.`k1` IS NOT NULL)")); + Assert.assertEquals(2, stmt6.toSql().split(" OR ").length); + + String sql7 = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2\n" + + "where\n" + + " t2.k1 is not null or t1.k1 is not null and t1.k2 is not null"; + SelectStmt stmt7 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql7, ctx); + stmt7.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt7.toSql().contains("(`t2`.`k1` IS NOT NULL) OR ((`t1`.`k1` IS NOT NULL) " + + "AND (`t1`.`k2` IS NOT NULL))")); + + String sql8 = "select\n" + + " avg(t1.k4)\n" + + "from\n" + + " db1.tbl1 t1,\n" + + " db1.tbl1 t2\n" + + "where\n" + + " t2.k1 is not null and t1.k1 is not null and t1.k1 is not null"; + SelectStmt stmt8 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql8, ctx); + stmt8.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt8.toSql().contains("((`t2`.`k1` IS NOT NULL) AND (`t1`.`k1` IS NOT NULL))" + + " AND (`t1`.`k1` IS NOT NULL)")); + + String sql9 = "select * from db1.tbl1 where (k1='shutdown' and k4<1) or (k1='switchOff' and k4>=1)"; + SelectStmt stmt9 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql9, ctx); + stmt9.rewriteExprs(new Analyzer(ctx.getCatalog(), ctx).getExprRewriter()); + Assert.assertTrue(stmt9.toSql().contains("((`k1` = 'shutdown') AND (`k4` < 1))" + + " OR ((`k1` = 'switchOff') AND (`k4` >= 1))")); + } + + @Test + public void testForbiddenCorrelatedSubqueryInHavingClause() throws Exception { + String sql = "SELECT k1 FROM baseall GROUP BY k1 HAVING EXISTS(SELECT k4 FROM tbl1 GROUP BY k4 HAVING SUM" + + "(baseall.k1) = k4);"; + try { + dorisAssert.query(sql).explainQuery(); + Assert.fail("The correlated subquery in having clause should be forbidden."); + } catch (AnalysisException e) { + System.out.println(e.getMessage()); + } + } + + @Test + public void testGroupByConstantExpression() throws Exception { + String sql = "SELECT k1 - 4*60*60 FROM baseall GROUP BY k1 - 4*60*60"; + dorisAssert.query(sql).explainQuery(); + } + + @Test + public void testMultrGroupByInCorrelationSubquery() throws Exception { + String sql = "SELECT * from baseall where k1 > (select min(k1) from tbl1 where baseall.k1 = tbl1.k4 and baseall.k2 = tbl1.k2)"; + dorisAssert.query(sql).explainQuery(); + } + + @Test + public void testOuterJoinNullUnionView() throws Exception{ + String sql = "WITH test_view(k) AS(SELECT NULL AS k UNION ALL SELECT NULL AS k )\n" + + "SELECT v1.k FROM test_view AS v1 LEFT OUTER JOIN test_view AS v2 ON v1.k=v2.k"; + dorisAssert.query(sql).explainQuery(); + } + + @Test + public void testDataGripSupport() throws Exception { + String sql = "select schema();"; + dorisAssert.query(sql).explainQuery(); + sql = "select\n" + + "collation_name,\n" + + "character_set_name,\n" + + "is_default collate utf8_general_ci = 'Yes' as is_default\n" + + "from information_schema.collations"; + dorisAssert.query(sql).explainQuery(); + } @Test public void testRandFunction() throws Exception { From 3e99267974ceff15cde5313fe270603654674fd5 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 27 Aug 2020 16:15:56 +0800 Subject: [PATCH 15/16] update --- .../src/test/java/org/apache/doris/analysis/SelectStmtTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java index 6a208139843bb7..3754bd5947a8e0 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -392,6 +392,7 @@ public void testRandFunction() throws Exception { dorisAssert.query(sql).explainQuery(); sql = "select rand() from db1.tbl1;"; dorisAssert.query(sql).explainQuery(); + } public void testVarcharToLongSupport() throws Exception { String sql = "select count(*)\n" + From 697c5e780a94ae9226eac267b7810ed375ac9bb2 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 27 Aug 2020 19:58:46 +0800 Subject: [PATCH 16/16] update --- .../src/test/java/org/apache/doris/analysis/SelectStmtTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java index 3754bd5947a8e0..e7f3dd6a4d8d16 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -394,6 +394,7 @@ public void testRandFunction() throws Exception { dorisAssert.query(sql).explainQuery(); } + @Test public void testVarcharToLongSupport() throws Exception { String sql = "select count(*)\n" + "from db1.partition_table\n" +