From e09dfdde375ca0b88b5e444541418c35e0069f71 Mon Sep 17 00:00:00 2001 From: shizhiqiang03 Date: Tue, 13 Sep 2022 15:57:04 +0800 Subject: [PATCH 1/5] fix fe oom because replica count too much when schema change --- .../org/apache/doris/alter/AlterHandler.java | 19 +++++++++++++++++++ .../doris/alter/MaterializedViewHandler.java | 1 + .../doris/alter/SchemaChangeHandler.java | 1 + .../java/org/apache/doris/common/Config.java | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java index c0d31cab9d1e2e..8274b2966ee209 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; @@ -253,4 +254,22 @@ public void replayAlterJobV2(AlterJobV2 alterJob) { existingJob.replay(alterJob); } } + + /** + * there will be OOM if there are too many replicas of the table when schema change. + */ + protected void checkReplicaCount(OlapTable olapTable) throws DdlException { + olapTable.readLock(); + try { + long replicaCount = olapTable.getReplicaCount(); + long maxReplicaCount = Config.mt_max_replica_count_when_schema_change; + if (replicaCount > maxReplicaCount) { + String msg = String.format("%s have %d replicas reach %d limit when schema change.", olapTable.getName(), replicaCount, maxReplicaCount); + LOG.warn(msg); + throw new DdlException(msg); + } + } finally { + olapTable.readUnlock(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 32ec16301b91c3..6a835d6ccbc8e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -233,6 +233,7 @@ public void processCreateMaterializedView(CreateMaterializedViewStmt addMVClause */ public void processBatchAddRollup(List alterClauses, Database db, OlapTable olapTable) throws DdlException, AnalysisException { + checkReplicaCount(olapTable); Map rollupNameJobMap = new LinkedHashMap<>(); // save job id for log Set logJobIdSet = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 85e12954c8beee..90b81cfb08f342 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1139,6 +1139,7 @@ private void createJob(long dbId, OlapTable olapTable, Map Date: Thu, 10 Nov 2022 10:09:29 +0800 Subject: [PATCH 2/5] add conf --- docs/zh-CN/docs/admin-manual/config/fe-config.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 302c5fbdbeea59..f79d621bf414d2 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2261,6 +2261,7 @@ load 标签清理器将每隔 `label_clean_interval_second` 运行一次以清 是否为 Master FE 节点独有的配置项：true +<<<<<<< HEAD ### `be_exec_version` 用于定义fragment之间传递block的序列化格式。 @@ -2309,6 +2310,13 @@ load 标签清理器将每隔 `label_clean_interval_second` 运行一次以清 该参数适用于回归测试环境，以减少偶发的心跳失败导致大量回归测试失败。 默认值：1 +======= +### max_replica_count_when_schema_change + +OlapTable在做schema change时，允许的最大副本数，副本数过大会导致FE OOM。 + +默认值：1000000 +>>>>>>> b011f89d49 (add conf) 是否可以动态配置：true From da913177d1eefa9a880c643877de103b97ac4ace Mon Sep 17 00:00:00 2001 From: shizhiqiang03 Date: Thu, 10 Nov 2022 10:12:27 +0800 Subject: [PATCH 3/5] fix some code --- docs/zh-CN/docs/admin-manual/config/fe-config.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index f79d621bf414d2..035715d1eadc95 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2261,7 +2261,6 @@ load 标签清理器将每隔 `label_clean_interval_second` 运行一次以清 是否为 Master FE 节点独有的配置项：true -<<<<<<< HEAD ### `be_exec_version` 用于定义fragment之间传递block的序列化格式。 @@ -2310,13 +2309,17 @@ load 标签清理器将每隔 `label_clean_interval_second` 运行一次以清 该参数适用于回归测试环境，以减少偶发的心跳失败导致大量回归测试失败。 默认值：1 -======= + +是否可以动态配置：true + +是否为 Master FE 节点独有的配置项：true + + ### max_replica_count_when_schema_change OlapTable在做schema change时，允许的最大副本数，副本数过大会导致FE OOM。 默认值：1000000 ->>>>>>> b011f89d49 (add conf) 是否可以动态配置：true From 5005c6744593359ee3a13f0a9459ad1b92e17ea7 Mon Sep 17 00:00:00 2001 From: shizhiqiang03 Date: Thu, 10 Nov 2022 10:58:57 +0800 Subject: [PATCH 4/5] fix some code --- docs/zh-CN/docs/admin-manual/config/fe-config.md | 2 +- fe/fe-core/src/main/java/org/apache/doris/common/Config.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 035715d1eadc95..957f87bcd6b947 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2319,7 +2319,7 @@ load 标签清理器将每隔 `label_clean_interval_second` 运行一次以清 OlapTable在做schema change时，允许的最大副本数，副本数过大会导致FE OOM。 -默认值：1000000 +默认值：100000 是否可以动态配置：true diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index ac34bb1ab35e94..396f813f4f548e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1834,9 +1834,9 @@ public class Config extends ConfigBase { /** * if table has too many replicas, Fe occur oom when schema change. - * 100W replicas is a reasonable value for testing. + * 10W replicas is a reasonable value for testing. */ @ConfField(mutable = true, masterOnly = true) - public static long max_replica_count_when_schema_change = 1000000; + public static long max_replica_count_when_schema_change = 100000; } From 6146fd610bf1a51b7740e36ce78a069d222ac94d Mon Sep 17 00:00:00 2001 From: shizhiqiang03 Date: Thu, 10 Nov 2022 11:09:00 +0800 Subject: [PATCH 5/5] fix some code --- .../src/main/java/org/apache/doris/alter/AlterHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java index 8274b2966ee209..c408f9690d4955 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -262,9 +262,10 @@ protected void checkReplicaCount(OlapTable olapTable) throws DdlException { olapTable.readLock(); try { long replicaCount = olapTable.getReplicaCount(); - long maxReplicaCount = Config.mt_max_replica_count_when_schema_change; + long maxReplicaCount = Config.max_replica_count_when_schema_change; if (replicaCount > maxReplicaCount) { - String msg = String.format("%s have %d replicas reach %d limit when schema change.", olapTable.getName(), replicaCount, maxReplicaCount); + String msg = String.format("%s have %d replicas reach %d limit when schema change.", + olapTable.getName(), replicaCount, maxReplicaCount); LOG.warn(msg); throw new DdlException(msg); }