From 8c3aaad5a182beeee99d6fb994663ed238b88e13 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Thu, 2 Mar 2023 13:21:14 +0800 Subject: [PATCH 1/2] 1 --- be/src/common/config.h | 4 +++- be/src/util/mem_info.cpp | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 1b6786b7d92411..acb65dcbad6c11 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -60,7 +60,9 @@ CONF_String(memory_mode, "moderate"); // defaults to bytes if no unit is given" // must larger than 0. and if larger than physical memory size, // it will be set to physical memory size. -CONF_String(mem_limit, "80%"); +// `auto` means process mem limit is equal to max(physical mem * 0.9, 6.4G). +// 6.4G is the maximum memory reserved for the system. +CONF_String(mem_limit, "auto"); // Soft memory limit as a fraction of hard memory limit. CONF_Double(soft_mem_limit_frac, "0.9"); diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index 0bd547ea0e307e..18c8b36463f1b2 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -237,16 +237,21 @@ void MemInfo::init() { } bool is_percent = true; - _s_mem_limit = ParseUtil::parse_mem_spec(config::mem_limit, -1, _s_physical_mem, &is_percent); - if (_s_mem_limit <= 0) { - LOG(WARNING) << "Failed to parse mem limit from '" + config::mem_limit + "'."; - } - if (_s_mem_limit > _s_physical_mem) { - LOG(WARNING) << "Memory limit " << PrettyPrinter::print(_s_mem_limit, TUnit::BYTES) - << " exceeds physical memory of " - << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES) - << ". Using physical memory instead"; - _s_mem_limit = _s_physical_mem; + if (config::mem_limit == "auto") { + _s_mem_limit = std::max(_s_physical_mem * 0.9, _s_physical_mem - 6871947672); + } else { + _s_mem_limit = + ParseUtil::parse_mem_spec(config::mem_limit, -1, _s_physical_mem, &is_percent); + if (_s_mem_limit <= 0) { + LOG(WARNING) << "Failed to parse mem limit from '" + config::mem_limit + "'."; + } + if (_s_mem_limit > _s_physical_mem) { + LOG(WARNING) << "Memory limit " << PrettyPrinter::print(_s_mem_limit, TUnit::BYTES) + << " exceeds physical memory of " + << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES) + << ". Using physical memory instead"; + _s_mem_limit = _s_physical_mem; + } } _s_mem_limit_str = PrettyPrinter::print(_s_mem_limit, TUnit::BYTES); _s_soft_mem_limit = _s_mem_limit * config::soft_mem_limit_frac; From 3b93e465d2c20d9fce7a7d52817b8ae39ea1d85e Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Sat, 4 Mar 2023 23:57:41 +0800 Subject: [PATCH 2/2] 2 --- be/src/common/config.h | 4 ++-- docs/en/docs/admin-manual/config/be-config.md | 3 ++- docs/zh-CN/docs/admin-manual/config/be-config.md | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index acb65dcbad6c11..00278230ee4efd 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -60,8 +60,8 @@ CONF_String(memory_mode, "moderate"); // defaults to bytes if no unit is given" // must larger than 0. and if larger than physical memory size, // it will be set to physical memory size. -// `auto` means process mem limit is equal to max(physical mem * 0.9, 6.4G). -// 6.4G is the maximum memory reserved for the system. +// `auto` means process mem limit is equal to max(physical_mem * 0.9, physical_mem - 6.4G). +// 6.4G is the maximum memory reserved for the system by default. CONF_String(mem_limit, "auto"); // Soft memory limit as a fraction of hard memory limit. diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md index 8b466c749139a9..0887605c190d12 100644 --- a/docs/en/docs/admin-manual/config/be-config.md +++ b/docs/en/docs/admin-manual/config/be-config.md @@ -188,7 +188,8 @@ There are two ways to configure BE configuration items: * Type: string * Description: Limit the percentage of the server's maximum memory used by the BE process. It is used to prevent BE memory from occupying to many the machine's memory. This parameter must be greater than 0. When the percentage is greater than 100%, the value will default to 100%. -* Default value: 80% + - `auto` means process mem limit is equal to max(physical_mem * 0.9, physical_mem - 6.4G), 6.4G is the maximum memory reserved for the system by default. +* Default value: auto #### `cluster_id` diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md index aca0f71e3219c6..3907137985f73d 100644 --- a/docs/zh-CN/docs/admin-manual/config/be-config.md +++ b/docs/zh-CN/docs/admin-manual/config/be-config.md @@ -197,7 +197,8 @@ BE 重启后该配置将失效。如果想持久化修改结果，使用如下 * 类型：string * 描述：限制BE进程使用服务器最大内存百分比。用于防止BE内存挤占太多的机器内存，该参数必须大于0，当百分大于100%之后，该值会默认为100%。 -* 默认值：80% + - `auto` 等于 max(physical_mem * 0.9, physical_mem - 6.4G)，6.4G是默认为系统预留的最大内存。 +* 默认值：auto #### `cluster_id`