From 1a15f533098eec6708121004ef835d111dc374f3 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 13 Aug 2020 22:09:31 +0800 Subject: [PATCH 01/13] udf: replace function --- be/src/exprs/string_functions.cpp | 19 ++++++++ be/src/exprs/string_functions.h | 3 ++ be/test/exprs/string_functions_test.cpp | 13 ++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ .../string-functions/str_replace.md | 46 +++++++++++++++++++ gensrc/script/doris_builtins_functions.py | 2 + 6 files changed, 129 insertions(+) create mode 100644 docs/en/sql-reference/sql-functions/string-functions/str_replace.md create mode 100644 docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 998794c8392345..ecf7695f45ce12 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -998,4 +998,23 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& int len = (find[field.val - 1] == -1 ? content.len : find[field.val - 1]) - start_pos; return StringVal(content.ptr + start_pos, len); } + +StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { + if (origStr.is_null || oldStr.is_null || newStr.is_null) { + return origStr; + } + std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); + std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); + std::string new_str = std::string(reinterpret_cast(newStr.ptr), newStr.len); + std::string::size_type pos = 0; + std::string::size_type oldLen = old_str.size(); + std::string::size_type newLen = new_str.size(); + while(pos = orig_str.find(old_str, pos)) + { + if(pos == std::string::npos) break; + orig_str.replace(pos, oldLen, new_str); + pos += newLen; + } + return AnyValUtil::from_string_temp(context, orig_str); +} } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index 13b96bcf5feb61..e60969bde5aa7c 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -190,6 +190,9 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); + + static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 05d484c67cd1bd..339f89f4342cb7 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -494,6 +494,19 @@ TEST_F(StringFunctionsTest, rpad) { ASSERT_EQ(StringVal("呵呵hih"), StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } + +TEST_F(StringFunctionsTest, str_replace) { + ASSERT_EQ(StringVal("http://www.baidu.com:8080"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:9090"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + ASSERT_EQ(StringVal("http://www.baidu.com:"), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); +} + } // namespace doris int main(int argc, char** argv) { diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..f9a28cccece692 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +replace all old substring with new substring in str + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md new file mode 100644 index 00000000000000..cf0bd769badfe4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -0,0 +1,46 @@ +--- +{ + "title": "str_replace", + "language": "zh-CN" +} +--- + + + +# str_replace +## description +### Syntax + +`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` + +将str字符串中的old子串全部替换为new串 + +## example + +``` +mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); ++------------------------------------------------------+ +| str_replace('http://www.baidu.com:9090', '9090', '') | ++------------------------------------------------------+ +| http://www.baidu.com: | ++------------------------------------------------------+ +``` +##keyword +STR_REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 5534f1b2b0407f..c966b7d8f9a9cb 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,6 +605,8 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From 391158f52190755b5fc621c31bf703835c0a7b3b Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 10:11:25 +0800 Subject: [PATCH 02/13] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/test/exprs/string_functions_test.cpp | 22 +++++++++++++++++++ .../string-functions/str_replace.md | 2 +- .../string-functions/str_replace.md | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index ecf7695f45ce12..06c6d31aa75aa9 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1001,7 +1001,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { - return origStr; + return StringVal::null(); } std::string orig_str = std::string(reinterpret_cast(origStr.ptr), origStr.len); std::string old_str = std::string(reinterpret_cast(oldStr.ptr), oldStr.len); diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 339f89f4342cb7..09185505859138 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -496,15 +496,37 @@ TEST_F(StringFunctionsTest, rpad) { } TEST_F(StringFunctionsTest, str_replace) { + //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + + //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + + //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + + //origin string is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + + //old substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + + //new substring is null + ASSERT_EQ(StringVal::null(), + StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + + //substring contains Chinese character + ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), + StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md index f9a28cccece692..dfbd8221b9f72a 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md index cf0bd769badfe4..2180cfa0670741 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md @@ -42,5 +42,5 @@ mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); | http://www.baidu.com: | +------------------------------------------------------+ ``` -##keyword +## keyword STR_REPLACE From 5eb52e1e5742d2e9f7aae5ac8da81545ae5f3df2 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Fri, 14 Aug 2020 11:58:30 +0800 Subject: [PATCH 03/13] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 06c6d31aa75aa9..deca61f9c3bc87 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -1009,7 +1009,7 @@ StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal std::string::size_type pos = 0; std::string::size_type oldLen = old_str.size(); std::string::size_type newLen = new_str.size(); - while(pos = orig_str.find(old_str, pos)) + while ((pos = orig_str.find(old_str, pos))) { if(pos == std::string::npos) break; orig_str.replace(pos, oldLen, new_str); From 86f841f29fa4ee19773d6c5922465194a2887cf6 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:56:14 +0800 Subject: [PATCH 04/13] udf: replace function --- be/src/exprs/string_functions.cpp | 2 +- be/src/exprs/string_functions.h | 2 +- be/test/exprs/string_functions_test.cpp | 18 +++++++++--------- fe/fe-core/src/main/cup/sql_parser.cup | 2 ++ gensrc/script/doris_builtins_functions.py | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index deca61f9c3bc87..1679eca54b9ddd 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -999,7 +999,7 @@ StringVal StringFunctions::split_part(FunctionContext* context, const StringVal& return StringVal(content.ptr + start_pos, len); } -StringVal StringFunctions::str_replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { +StringVal StringFunctions::replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr) { if (origStr.is_null || oldStr.is_null || newStr.is_null) { return StringVal::null(); } diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index e60969bde5aa7c..f1a4df9e373f51 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -191,7 +191,7 @@ class StringFunctions { static StringVal split_part(FunctionContext* context, const StringVal& content, const StringVal& delimiter, const IntVal& field); - static StringVal str_replace(FunctionContext *context, const StringVal &origStr, + static StringVal replace(FunctionContext *context, const StringVal &origStr, const StringVal &oldStr, const StringVal &newStr); }; } diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 09185505859138..3a6e351f60f990 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -495,38 +495,38 @@ TEST_F(StringFunctionsTest, rpad) { StringFunctions::rpad(ctx, StringVal("呵呵"), IntVal(5), StringVal("hi"))); } -TEST_F(StringFunctionsTest, str_replace) { +TEST_F(StringFunctionsTest, replace) { //exist substring ASSERT_EQ(StringVal("http://www.baidu.com:8080"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal("8080"))); //not exist substring ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9070"), StringVal("8080"))); //old substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:9090"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal(""), StringVal("8080"))); //new substring is empty ASSERT_EQ(StringVal("http://www.baidu.com:"), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("9090"), StringVal(""))); //origin string is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal::null(), StringVal("hello"), StringVal("8080"))); //old substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal::null(), StringVal("8080"))); //new substring is null ASSERT_EQ(StringVal::null(), - StringFunctions::str_replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); + StringFunctions::replace(ctx, StringVal("http://www.baidu.com:9090"), StringVal("hello"), StringVal::null())); //substring contains Chinese character ASSERT_EQ(StringVal("http://华夏zhongguo:9090"), - StringFunctions::str_replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); + StringFunctions::replace(ctx, StringVal("http://中国hello:9090"), StringVal("中国hello"), StringVal("华夏zhongguo"))); } } // namespace doris diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index a2d8294915cc21..7ba56810734001 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -4593,6 +4593,8 @@ keyword ::= {: RESULT = id; :} | KW_REPEATABLE:id {: RESULT = id; :} + | KW_REPLACE:id + {: RESULT = id; :} | KW_REPLACE_IF_NOT_NULL:id {: RESULT = id; :} | KW_REPOSITORY:id diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index c966b7d8f9a9cb..ed26fae15f87aa 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -605,7 +605,7 @@ '15FunctionContextENS2_18FunctionStateScopeE'], [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], - [['str_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], + [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' From ade1afa75c94a178a6801ba1324acf25ad8b16fe Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 11:59:59 +0800 Subject: [PATCH 05/13] udf: replace function --- .../string-functions/{str_replace.md => replace.md} | 10 +++++----- .../string-functions/{str_replace.md => replace.md} | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) rename docs/en/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (83%) rename docs/zh-CN/sql-reference/sql-functions/string-functions/{str_replace.md => replace.md} (82%) diff --git a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md similarity index 83% rename from docs/en/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/en/sql-reference/sql-functions/string-functions/replace.md index dfbd8221b9f72a..6f86b18a679d3d 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,20 +24,20 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` replace all old substring with new substring in str ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ diff --git a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md similarity index 82% rename from docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md rename to docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md index 2180cfa0670741..dda2625df07d90 100644 --- a/docs/zh-CN/sql-reference/sql-functions/string-functions/str_replace.md +++ b/docs/zh-CN/sql-reference/sql-functions/string-functions/replace.md @@ -1,6 +1,6 @@ --- { - "title": "str_replace", + "title": "replace", "language": "zh-CN" } --- @@ -24,23 +24,23 @@ specific language governing permissions and limitations under the License. --> -# str_replace +# replace ## description ### Syntax -`VARCHAR STR_REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` +`VARCHAR REPLACE (VARCHAR str, VARCHAR old, VARCHAR new)` 将str字符串中的old子串全部替换为new串 ## example ``` -mysql> select str_replace("http://www.baidu.com:9090", "9090", ""); +mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ -| str_replace('http://www.baidu.com:9090', '9090', '') | +| replace('http://www.baidu.com:9090', '9090', '') | +------------------------------------------------------+ | http://www.baidu.com: | +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE From 1d95a491115064c9753694ad4d45c2f72b8d2645 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 18 Aug 2020 16:01:45 +0800 Subject: [PATCH 06/13] udf: replace function --- docs/en/sql-reference/sql-functions/string-functions/replace.md | 2 +- gensrc/script/doris_builtins_functions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/sql-functions/string-functions/replace.md b/docs/en/sql-reference/sql-functions/string-functions/replace.md index 6f86b18a679d3d..d5111fa43c8077 100644 --- a/docs/en/sql-reference/sql-functions/string-functions/replace.md +++ b/docs/en/sql-reference/sql-functions/string-functions/replace.md @@ -43,4 +43,4 @@ mysql> select replace("http://www.baidu.com:9090", "9090", ""); +------------------------------------------------------+ ``` ## keyword -STR_REPLACE +REPLACE diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index ed26fae15f87aa..b960ba6afbbba0 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -606,7 +606,7 @@ [['concat'], 'VARCHAR', ['VARCHAR', '...'], '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions11str_replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], + '_ZN5doris15StringFunctions7replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' '15FunctionContextERKNS1_9StringValEiPS5_'], From c1e58d5db9e462a0c0a0310feef7bbae81f75eaf Mon Sep 17 00:00:00 2001 From: wangxixu Date: Mon, 26 Oct 2020 20:19:52 +0800 Subject: [PATCH 07/13] unused importe --- .../main/java/org/apache/doris/service/FrontendServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index e29aa96a20e72e..f02802150d568c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -37,7 +37,6 @@ import org.apache.doris.common.ThriftServerContext; import org.apache.doris.common.ThriftServerEventProcessor; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.DebugUtil; import org.apache.doris.load.EtlStatus; import org.apache.doris.load.LoadJob; import org.apache.doris.load.MiniEtlTaskInfo; From e4405e4c777fda98c47bca926cb384bc85f2365c Mon Sep 17 00:00:00 2001 From: wangxixu Date: Mon, 26 Oct 2020 21:08:39 +0800 Subject: [PATCH 08/13] cat exception --- .../java/org/apache/doris/load/loadv2/BulkLoadJob.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index 04f3d938fc0d30..514e8b93ba0105 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -233,7 +233,12 @@ public void onTaskFailed(long taskId, FailMsg failMsg) { loadTask.updateRetryInfo(); idToTasks.put(loadTask.getSignature(), loadTask); // load id will be added to loadStatistic when executing this task - Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); + try { + Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); + } catch (Exception e) { + unprotectedExecuteCancel(failMsg, true); + logFinalOperation(); + } return; } } finally { From 9145509113efa219a9495aee8d564cb2420604a6 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Mon, 26 Oct 2020 21:25:08 +0800 Subject: [PATCH 09/13] update --- .../apache/doris/load/loadv2/BulkLoadJob.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index 514e8b93ba0105..9632281df52607 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -233,9 +233,29 @@ public void onTaskFailed(long taskId, FailMsg failMsg) { loadTask.updateRetryInfo(); idToTasks.put(loadTask.getSignature(), loadTask); // load id will be added to loadStatistic when executing this task - try { - Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); - } catch (Exception e) { + boolean retrySuccessFlag = false; + while (loadTask.getRetryTime() > 0) { + try { + Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); + retrySuccessFlag = true; + break; + } catch (Exception e) { + try { + Thread.sleep(100); + } catch (InterruptedException ex) { + LOG.warn("retry sublit task sleep failed. task_id:{}", id); + } + // retry task + idToTasks.remove(loadTask.getSignature()); + if (loadTask instanceof LoadLoadingTask) { + loadStatistic.removeLoad(((LoadLoadingTask) loadTask).getLoadId()); + } + loadTask.updateRetryInfo(); + idToTasks.put(loadTask.getSignature(), loadTask); + continue; + } + } + if (!retrySuccessFlag) { unprotectedExecuteCancel(failMsg, true); logFinalOperation(); } From 42c7c6d6e0cff8e4fe52f3e2b935de51795a76fc Mon Sep 17 00:00:00 2001 From: wangxixu Date: Mon, 26 Oct 2020 21:28:56 +0800 Subject: [PATCH 10/13] update --- .../main/java/org/apache/doris/service/FrontendServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index f02802150d568c..e29aa96a20e72e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -37,6 +37,7 @@ import org.apache.doris.common.ThriftServerContext; import org.apache.doris.common.ThriftServerEventProcessor; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.DebugUtil; import org.apache.doris.load.EtlStatus; import org.apache.doris.load.LoadJob; import org.apache.doris.load.MiniEtlTaskInfo; From a1a94cc2e097728e0bd62c6a141491b975b8e361 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Tue, 27 Oct 2020 13:10:11 +0800 Subject: [PATCH 11/13] update --- .../java/org/apache/doris/load/loadv2/BulkLoadJob.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index 9632281df52607..d938e04c322625 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -243,15 +243,9 @@ public void onTaskFailed(long taskId, FailMsg failMsg) { try { Thread.sleep(100); } catch (InterruptedException ex) { - LOG.warn("retry sublit task sleep failed. task_id:{}", id); - } - // retry task - idToTasks.remove(loadTask.getSignature()); - if (loadTask instanceof LoadLoadingTask) { - loadStatistic.removeLoad(((LoadLoadingTask) loadTask).getLoadId()); + LOG.warn("retry submit task sleep failed. task_id:{}", id); } loadTask.updateRetryInfo(); - idToTasks.put(loadTask.getSignature(), loadTask); continue; } } From 6f8e4074bb761d632d6ac7eb69b87b39fc307ca0 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 5 Nov 2020 18:01:49 +0800 Subject: [PATCH 12/13] update --- .../doris/load/loadv2/BrokerLoadJob.java | 8 +++++++ .../apache/doris/load/loadv2/BulkLoadJob.java | 22 ++++--------------- .../doris/load/loadv2/LoadJobScheduler.java | 7 ++++++ 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 9ca4c77303820b..a253dbe919c8ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -52,6 +52,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.RejectedExecutionException; /** * There are 3 steps in BrokerLoadJob: BrokerPendingTask, LoadLoadingTask, CommitAndPublishTxn. @@ -155,6 +156,13 @@ private void onPendingTaskFinished(BrokerPendingTaskAttachment attachment) { .build(), e); cancelJobWithoutCheck(new FailMsg(FailMsg.CancelType.ETL_RUN_FAIL, e.getMessage()), true, true); return; + } catch (RejectedExecutionException e) { + LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id) + .add("database_id", dbId) + .add("error_msg", "the task queque is full.") + .build(), e); + cancelJobWithoutCheck(new FailMsg(FailMsg.CancelType.ETL_RUN_FAIL, e.getMessage()), true, true); + return; } loadStartTimestamp = System.currentTimeMillis(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index d938e04c322625..ca326b8e10944e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -233,27 +233,13 @@ public void onTaskFailed(long taskId, FailMsg failMsg) { loadTask.updateRetryInfo(); idToTasks.put(loadTask.getSignature(), loadTask); // load id will be added to loadStatistic when executing this task - boolean retrySuccessFlag = false; - while (loadTask.getRetryTime() > 0) { - try { - Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); - retrySuccessFlag = true; - break; - } catch (Exception e) { - try { - Thread.sleep(100); - } catch (InterruptedException ex) { - LOG.warn("retry submit task sleep failed. task_id:{}", id); - } - loadTask.updateRetryInfo(); - continue; - } - } - if (!retrySuccessFlag) { + try { + Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); + } catch (Exception e) { unprotectedExecuteCancel(failMsg, true); logFinalOperation(); + return; } - return; } } finally { writeUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobScheduler.java index 9a2b691983963b..33d46773d35071 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobScheduler.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; /** * LoadScheduler will schedule the pending LoadJob which belongs to LoadManager. @@ -99,6 +100,12 @@ private void process() throws InterruptedException { .build(), e); needScheduleJobs.put(loadJob); return; + } catch (RejectedExecutionException e) { + LOG.warn(new LogBuilder(LogKey.LOAD_JOB, loadJob.getId()) + .add("error_msg", "Failed to submit etl job. Job queue is full.") + .build(), e); + loadJob.cancelJobWithoutCheck(new FailMsg(FailMsg.CancelType.ETL_SUBMIT_FAIL, e.getMessage()), + true, true); } } } From 6fbf63cdce0ef326dff397e01f1ed2ceb5611f10 Mon Sep 17 00:00:00 2001 From: wangxixu Date: Thu, 5 Nov 2020 18:03:30 +0800 Subject: [PATCH 13/13] update --- .../main/java/org/apache/doris/load/loadv2/BulkLoadJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index ca326b8e10944e..1d649c65627bc0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.RejectedExecutionException; /** * parent class of BrokerLoadJob and SparkLoadJob from load stmt @@ -235,7 +236,7 @@ public void onTaskFailed(long taskId, FailMsg failMsg) { // load id will be added to loadStatistic when executing this task try { Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); - } catch (Exception e) { + } catch (RejectedExecutionException e) { unprotectedExecuteCancel(failMsg, true); logFinalOperation(); return;