From 1a15f533098eec6708121004ef835d111dc374f3 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 13 Aug 2020 22:09:31 +0800 Subject: [PATCH 1/8] udf: replace function --- be/src/exprs/string_functions.cpp | 19 ++++++++ be/src/exprs/string_functions.h | 3 ++ be/test/exprs/string_functions_test.cpp | 13 ++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ gensrc/script/doris_builtins_functions.py | 2 + 6 files changed, 129 insertions(+) create mode 100644 docs/en/sql-reference/sql-functions/string-functions/str_replace.md create mode 100644 docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 998794c8392345..ecf7695f45ce12 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -998,4 +998,23 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& int len = (find[field.val - 1] == -1 ? content.len : find[field.val - 1]) - start_pos; return StringVal(content.ptr + start_pos, len); } + +StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { + if (origStr.is_null || oldStr.is_null || newStr.is_null) { + return origStr; + } + std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); + std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); + std::string new_str = std::string(reinterpret_cast(newStr.ptr), newStr.len); + std::string::size_type pos = 0; + std::string::size_type oldLen = old_str.size(); + std::string::size_type newLen = new_str.size(); + while(pos = orig_str.find(old_str, pos)) + { + if(pos == std::string::npos) break; + orig_str.replace(pos, oldLen, new_str); + pos += newLen; + } + return AnyValUtil::from_string_temp(context, orig_str); +} } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index 13b96bcf5feb61..e60969bde5aa7c 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -190,6 +190,9 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); + + static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 05d484c67cd1bd..339f89f4342cb7 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -494,6 +494,19 @@ TEST_F(StringFunctionsTest, rpad) { ASSERT_EQ(StringVal("呵呵hih"), StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } + +TEST_F(StringFunctionsTest, str_replace) { + ASSERT_EQ(StringVal("http://www.baidu.com:8080"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); +} + } // namespace doris int main(int argc, char** argv) { diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..f9a28cccece692 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +replace all old substring with new substring in str + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..cf0bd769badfe4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +将str字符串中的old子串全部替换为new串 + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 5534f1b2b0407f..c966b7d8f9a9cb 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,6 +605,8 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From 391158f52190755b5fc621c31bf703835c0a7b3b Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 10:11:25 +0800 Subject: [PATCH 2/8] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/test/exprs/string_functions_test.cpp | 22 +++++++++++++++++++ .../string-functions/str_replace.md | 2 +- .../string-functions/str_replace.md | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index ecf7695f45ce12..06c6d31aa75aa9 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1001,7 +1001,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { - return origStr; + return StringVal::null(); } std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 339f89f4342cb7..09185505859138 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -496,15 +496,37 @@ TEST_F(StringFunctionsTest, rpad) { } TEST_F(StringFunctionsTest, str_replace) { + //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + + //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + + //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + + //origin string is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + + //old substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + + //new substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + + //substring contains Chinese character + ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), + StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md index f9a28cccece692..dfbd8221b9f72a 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md index cf0bd769badfe4..2180cfa0670741 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE From 5eb52e1e5742d2e9f7aae5ac8da81545ae5f3df2 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 11:58:30 +0800 Subject: [PATCH 3/8] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 06c6d31aa75aa9..deca61f9c3bc87 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1009,7 +1009,7 @@ StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal std::string::size_type pos = 0; std::string::size_type oldLen = old_str.size(); std::string::size_type newLen = new_str.size(); - while(pos = orig_str.find(old_str, pos)) + while ((pos = orig_str.find(old_str, pos))) { if(pos == std::string::npos) break; orig_str.replace(pos, oldLen, new_str); From 86f841f29fa4ee19773d6c5922465194a2887cf6 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:56:14 +0800 Subject: [PATCH 4/8] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/src/exprs/string_functions.h | 2 +- be/test/exprs/string_functions_test.cpp | 18 +++++++++--------- fe/fe-core/src/main/cup/sql_parser.cup | 2 ++ gensrc/script/doris_builtins_functions.py | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index deca61f9c3bc87..1679eca54b9ddd 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -999,7 +999,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& return StringVal(content.ptr + start_pos, len); } -StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { +StringVal StringFunctions::replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { return StringVal::null(); } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index e60969bde5aa7c..f1a4df9e373f51 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -191,7 +191,7 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); - static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + static StringVal replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 09185505859138..3a6e351f60f990 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -495,38 +495,38 @@ TEST_F(StringFunctionsTest, rpad) { StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } -TEST_F(StringFunctionsTest, str_replace) { +TEST_F(StringFunctionsTest, replace) { //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); //origin string is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); //old substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); //new substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); //substring contains Chinese character ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), - StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); + StringFunctions::replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index a2d8294915cc21..7ba56810734001 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -4593,6 +4593,8 @@ keyword ::= {: RESULT = id; :} | KW_REPEATABLE:id {: RESULT = id; :} + | KW_REPLACE:id + {: RESULT = id; :} | KW_REPLACE_IF_NOT_NULL:id {: RESULT = id; :} | KW_REPOSITORY:id diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index c966b7d8f9a9cb..ed26fae15f87aa 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,7 +605,7 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], - [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' From ade1afa75c94a178a6801ba1324acf25ad8b16fe Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:59:59 +0800 Subject: [PATCH 5/8] udf: replace function --- .../string-functions/{str_replace.md => replace.md} | 10 +++++----- .../string-functions/{str_replace.md => replace.md} | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) rename docs/en/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (83%) rename docs/zh-CN/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (82%) diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md similarity index 83% rename from docs/en/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/en/sql-reference/sql-functions/string-functions/replace.md index dfbd8221b9f72a..6f86b18a679d3d 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,20 +24,20 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` replace all old substring with new substring in str ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md similarity index 82% rename from docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md index 2180cfa0670741..dda2625df07d90 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,23 +24,23 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` 将str字符串中的old子串全部替换为new串 ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE From 1d95a491115064c9753694ad4d45c2f72b8d2645 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 16:01:45 +0800 Subject: [PATCH 6/8] udf: replace function --- docs/en/sql-reference/sql-functions/string-functions/replace.md | 2 +- gensrc/script/doris_builtins_functions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/sql-functions/string-functions/replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md index 6f86b18a679d3d..d5111fa43c8077 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -43,4 +43,4 @@ mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index ed26fae15f87aa..b960ba6afbbba0 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -606,7 +606,7 @@ [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], + '_ZN5doris15StringFunctions7replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From 0c786c2545e0e41c9dee4754f265dac34fd6479d Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 17 Sep 2020 19:47:32 +0800 Subject: [PATCH 7/8] catch IllegalStateException --- .../org/apache/doris/load/loadv2/LoadingTaskPlanner.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) mode change 100644 => 100755 fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java old mode 100644 new mode 100755 index 803e4352b16315..65184543d3e507 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -127,7 +127,12 @@ public void plan(TUniqueId loadId, List> fileStatusesLis // 2. Olap table sink List partitionIds = getAllPartitionIds(); - OlapTableSink olapTableSink = new OlapTableSink(table, tupleDesc, partitionIds); + OlapTableSink olapTableSink; + try { + olapTableSink = new OlapTableSink(table, tupleDesc, partitionIds); + } catch (IllegalStateException e) { + throw new UserException(e.getMessage()); + } olapTableSink.init(loadId, txnId, dbId, timeoutS); olapTableSink.complete(); From cfc43437aa492d05264b498b80237924bc3e2862 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 22 Sep 2020 21:20:11 +0800 Subject: [PATCH 8/8] check partition before construct OlapTableSink --- .../org/apache/doris/analysis/InsertStmt.java | 3 ++ .../doris/load/loadv2/LoadingTaskPlanner.java | 8 ++- .../doris/planner/StreamLoadPlanner.java | 3 ++ .../doris/planner/StreamLoadPlannerTest.java | 54 +++++++++---------- 4 files changed, 33 insertions(+), 35 deletions(-) mode change 100644 => 100755 fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java mode change 100644 => 100755 fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java mode change 100644 => 100755 fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java old mode 100644 new mode 100755 index 36be17c9266445..1ec958df508060 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -740,6 +740,9 @@ private DataSink createDataSink() throws AnalysisException { return dataSink; } if (targetTable instanceof OlapTable) { + if (targetPartitionIds.size() <= 0) { + throw new AnalysisException("no target partition found."); + } dataSink = new OlapTableSink((OlapTable) targetTable, olapTuple, targetPartitionIds); dataPartition = dataSink.getOutputPartition(); } else if (targetTable instanceof BrokerTable) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index 65184543d3e507..9fb4ce8ab52909 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -127,12 +127,10 @@ public void plan(TUniqueId loadId, List> fileStatusesLis // 2. Olap table sink List partitionIds = getAllPartitionIds(); - OlapTableSink olapTableSink; - try { - olapTableSink = new OlapTableSink(table, tupleDesc, partitionIds); - } catch (IllegalStateException e) { - throw new UserException(e.getMessage()); + if (partitionIds.size() <= 0) { + throw new UserException("no partition found in file groups."); } + OlapTableSink olapTableSink = new OlapTableSink(table, tupleDesc, partitionIds); olapTableSink.init(loadId, txnId, dbId, timeoutS); olapTableSink.complete(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java old mode 100644 new mode 100755 index 9e779b6d674640..7abe9686c676d2 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -135,6 +135,9 @@ public TExecPlanFragmentParams plan(TUniqueId loadId) throws UserException { // create dest sink List partitionIds = getAllPartitionIds(); + if (partitionIds.size() <= 0) { + throw new UserException("no partition found in file groups."); + } OlapTableSink olapTableSink = new OlapTableSink(destTable, tupleDesc, partitionIds); olapTableSink.init(loadId, taskInfo.getTxnId(), db.getId(), taskInfo.getTimeout()); olapTableSink.complete(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java old mode 100644 new mode 100755 index 35b545c5af8196..7f5d9d8d34d6bf --- a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java @@ -26,7 +26,11 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.SinglePartitionInfo; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.RandomDistributionInfo; +import org.apache.doris.catalog.KeysType; import org.apache.doris.common.UserException; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.task.StreamLoadTask; @@ -41,52 +45,42 @@ import org.junit.Test; import java.io.StringReader; +import java.util.LinkedList; import java.util.List; import mockit.Expectations; import mockit.Injectable; import mockit.Mocked; +import org.apache.doris.thrift.TStorageType; public class StreamLoadPlannerTest { @Injectable Database db; - @Injectable - OlapTable destTable; - - @Mocked - StreamLoadScanNode scanNode; - - @Mocked - OlapTableSink sink; - @Test public void testNormalPlan() throws UserException { - List columns = Lists.newArrayList(); - Column c1 = new Column("c1", PrimitiveType.BIGINT, false); - columns.add(c1); - Column c2 = new Column("c2", PrimitiveType.BIGINT, true); - columns.add(c2); - new Expectations() { - { - destTable.getBaseSchema(); - minTimes = 0; - result = columns; - scanNode.init((Analyzer) any); - minTimes = 0; - scanNode.getChildren(); - minTimes = 0; - result = Lists.newArrayList(); - scanNode.getId(); - minTimes = 0; - result = new PlanNodeId(5); - } - }; + MaterializedIndex baseIndex = new MaterializedIndex(30001, MaterializedIndex.IndexState.NORMAL); + RandomDistributionInfo distributionInfo = new RandomDistributionInfo(10); + Partition partition = new Partition(20000L, "testTbl", baseIndex, distributionInfo); + List baseSchema = new LinkedList(); + Column column = new Column(); + baseSchema.add(column); + SinglePartitionInfo info = new SinglePartitionInfo(); + info.setReplicationNum(partition.getId(), (short) 3); + OlapTable destTable = new OlapTable(30000, "testTbl", baseSchema, + KeysType.AGG_KEYS, info, distributionInfo); + destTable.setIndexMeta(baseIndex.getId(), "testTbl", baseSchema, 0, 1, (short) 1, + TStorageType.COLUMN, KeysType.AGG_KEYS); + destTable.addPartition(partition); + destTable.setBaseIndexId(baseIndex.getId()); TStreamLoadPutRequest request = new TStreamLoadPutRequest(); request.setTxnId(1); + request.unsetIsTempPartition(); request.setLoadId(new TUniqueId(2, 3)); request.setFileType(TFileType.FILE_STREAM); request.setFormatType(TFileFormatType.FORMAT_CSV_PLAIN); + request.setPartitions("testTbl"); + StreamLoadTask streamLoadTask = StreamLoadTask.fromTStreamLoadPutRequest(request, db); StreamLoadPlanner planner = new StreamLoadPlanner(db, destTable, streamLoadTask); planner.plan(streamLoadTask.getId());