diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 2d78a430153341..79642952a832a8 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1199,6 +1199,10 @@ DEFINE_mInt32(max_s3_client_retry, "10"); DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/connectors"); +// the max package bytes be thrift server can receive +// avoid accepting error or too large package causing OOM,default 20000000(20M) +DEFINE_Int32(be_thrift_max_pkg_bytes, "20000000"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 8958e86c945120..204bbc72cf9d82 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1277,6 +1277,10 @@ DECLARE_String(tmp_file_dir); // the directory for storing the trino-connector plugins. DECLARE_String(trino_connector_plugin_dir); +// the max package bytes be thrift server can receive +// avoid accepting error or too large package causing OOM,default 20000000(20M) +DECLARE_Int32(be_thrift_max_pkg_bytes); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/util/thrift_server.cpp b/be/src/util/thrift_server.cpp index 99903a8a73db50..383f41bf23c650 100644 --- a/be/src/util/thrift_server.cpp +++ b/be/src/util/thrift_server.cpp @@ -308,6 +308,11 @@ Status ThriftServer::start() { DCHECK(!_started); std::shared_ptr protocol_factory( new apache::thrift::protocol::TBinaryProtocolFactory()); + // add binary_protocal_factory to call TBinaryProtocolFactory's member function:setStringSizeLimit + std::shared_ptr binary_protocal_factory = + std::dynamic_pointer_cast( + protocol_factory); + binary_protocal_factory->setStringSizeLimit(config::be_thrift_max_pkg_bytes); std::shared_ptr thread_mgr; std::shared_ptr thread_factory = std::make_shared(); diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md index 714ded467a8866..a047ed105e292d 100644 --- a/docs/zh-CN/docs/admin-manual/config/be-config.md +++ b/docs/zh-CN/docs/admin-manual/config/be-config.md @@ -1535,3 +1535,8 @@ load tablets from header failed, failed tablets size: xxx, path=xxx * 描述：是否直接使用 Doris 自带的时区数据库。开启后不再尝试查找系统目录。 * 默认值：false + +#### `be_thrift_max_pkg_bytes` + +* 描述: be节点thrift端口最大接收包大小(字节) +* 默认值: 20000000 diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index dbef19b59b6c93..33124695c485eb 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2772,3 +2772,13 @@ Doris 为了兼用 mysql 周边工具生态，会内置一个名为 mysql 的数 默认值：2000 对于自动分区表，防止用户意外创建大量分区，每个OLAP表允许的分区数量为`max_auto_partition_num`。默认2000。 + +#### `fe_thrift_max_pkg_bytes` + +默认值：20000000 + +是否可以动态配置：false + +是否为 Master FE 节点独有的配置项：false + +用于限制fe节点thrift端口可以接收的最大包长度，避免接收到过大或者错误的包导致OOM diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 0803ed20f2cd4e..0271cd6cbf9169 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2531,6 +2531,12 @@ public class Config extends ConfigBase { "Default storage format of inverted index, the default value is V1." }) public static String inverted_index_storage_format = "V1"; + + @ConfField(description = { + "限制fe节点thrift server可以接收的最大包大小,默认20M,设置为-1表示不限制", + "the max package size fe thrift server can receive,avoid accepting error" + + "or too large package causing OOM,default 20000000(20M),set -1 for unlimited. "}) + public static int fe_thrift_max_pkg_bytes = 20000000; //========================================================================== // begin of cloud config diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java index 2396dc95074cda..45c4bcb64f00c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServer.java @@ -92,14 +92,14 @@ public ThriftServerType getType() { private void createSimpleServer() throws TTransportException { TServer.Args args = new TServer.Args(new TServerSocket(port)).protocolFactory( - new TBinaryProtocol.Factory()).processor(processor); + new TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes, -1)).processor(processor); server = new TSimpleServer(args); } private void createThreadedServer() throws TTransportException { TThreadedSelectorServer.Args args = new TThreadedSelectorServer.Args( new TNonblockingServerSocket(port, Config.thrift_client_timeout_ms)).protocolFactory( - new TBinaryProtocol.Factory()).processor(processor); + new TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes, -1)).processor(processor); ThreadPoolExecutor threadPoolExecutor = ThreadPoolManager.newDaemonCacheThreadPool( Config.thrift_server_max_worker_threads, "thrift-server-pool", true); args.executorService(threadPoolExecutor); @@ -123,7 +123,7 @@ private void createThreadPoolServer() throws TTransportException { TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(new TServerSocket(socketTransportArgs)).protocolFactory( - new TBinaryProtocol.Factory()).processor(processor); + new TBinaryProtocol.Factory(Config.fe_thrift_max_pkg_bytes, -1)).processor(processor); ThreadPoolExecutor threadPoolExecutor = ThreadPoolManager.newDaemonCacheThreadPool( Config.thrift_server_max_worker_threads, "thrift-server-pool", true); serverArgs.executorService(threadPoolExecutor);