From 6119e21ce5a7d27c25d6afc3841f31491b89c387 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 28 Mar 2024 09:37:54 +0800 Subject: [PATCH 1/6] add decommission check --- .../org/apache/doris/alter/SystemHandler.java | 81 ++++++++++++++++++- ..._decommission_with_replica_num_fail.groovy | 59 ++++++++++++++ .../suites/node_p0/test_backend.groovy | 39 +++++++++ 3 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java index d4aae2d7dc006b..bf04e54fba972c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java @@ -33,24 +33,34 @@ import org.apache.doris.analysis.ModifyFrontendHostNameClause; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MysqlCompatibleDatabase; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.ReplicaAllocation; +import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.NetUtils; import org.apache.doris.ha.FrontendNodeType; +import org.apache.doris.resource.Tag; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.system.SystemInfoService.HostInfo; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.lang3.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /* * SystemHandler is for @@ -220,12 +230,81 @@ public static List checkDecommission(List hostInfos) decommissionBackends.add(backend); } - // TODO(cmy): check if replication num can be met + checkDecommissionWithReplicaAllocation(decommissionBackends); + // TODO(cmy): check remaining space return decommissionBackends; } + private static void checkDecommissionWithReplicaAllocation(List decommissionBackends) + throws DdlException { + if (Config.isCloudMode() || decommissionBackends.isEmpty() + || DebugPointUtil.isEnable("SystemHandler.decommission_no_check_replica_num")) { + return; + } + + Set decommissionTags = decommissionBackends.stream().map(be -> be.getLocationTag()) + .collect(Collectors.toSet()); + Map tagAvailBackendNums = Maps.newHashMap(); + for (Backend backend : Env.getCurrentSystemInfo().getAllBackends()) { + long beId = backend.getId(); + if (!backend.isScheduleAvailable() + || decommissionBackends.stream().anyMatch(be -> be.getId() == beId)) { + continue; + } + + Tag tag = backend.getLocationTag(); + if (tag != null) { + tagAvailBackendNums.put(tag, tagAvailBackendNums.getOrDefault(tag, 0) + 1); + } + } + + Env env = Env.getCurrentEnv(); + List dbIds = env.getInternalCatalog().getDbIds(); + for (Long dbId : dbIds) { + Database db = env.getInternalCatalog().getDbNullable(dbId); + if (db == null) { + continue; + } + + if (db instanceof MysqlCompatibleDatabase) { + continue; + } + + for (Table table : db.getTables()) { + table.readLock(); + try { + if (!table.isManagedTable()) { + continue; + } + + OlapTable tbl = (OlapTable) table; + for (Partition partition : tbl.getAllPartitions()) { + ReplicaAllocation replicaAlloc = tbl.getPartitionInfo().getReplicaAllocation(partition.getId()); + for (Map.Entry entry : replicaAlloc.getAllocMap().entrySet()) { + Tag tag = entry.getKey(); + if (!decommissionTags.contains(tag)) { + continue; + } + int replicaNum = (int) entry.getValue(); + int backendNum = tagAvailBackendNums.getOrDefault(tag, 0); + if (replicaNum > backendNum) { + throw new DdlException("After decommission, partition " + partition.getName() + + " of table " + db.getName() + "." + tbl.getName() + + " 's replication allocation { " + replicaAlloc + + " } > available backend num " + backendNum + " on tag " + tag + + ", otherwise need to decrease the partition's replication num."); + } + } + } + } finally { + table.readUnlock(); + } + } + } + } + @Override public synchronized void cancel(CancelStmt stmt) throws DdlException { CancelAlterSystemStmt cancelAlterSystemStmt = (CancelAlterSystemStmt) stmt; diff --git a/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy b/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy new file mode 100644 index 00000000000000..ff19adae27dad2 --- /dev/null +++ b/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_decommission_with_replica_num_fail') { + if (isCloudMode()) { + return + } + + def tbl = 'test_decommission_with_replica_num_fail' + def backends = sql_return_maparray('show backends') + def replicaNum = 0 + def targetBackend = null + for (def be : backends) { + def alive = be.Alive.toBoolean() + def decommissioned = be.SystemDecommissioned.toBoolean() + if (alive && !decommissioned) { + replicaNum++ + targetBackend = be + } + } + assertTrue(replicaNum > 0) + + sql "DROP TABLE IF EXISTS ${tbl} FORCE" + sql """ + CREATE TABLE ${tbl} + ( + k1 int, + k2 int + ) + DISTRIBUTED BY HASH(k1) BUCKETS 6 + PROPERTIES + ( + "replication_num" = "${replicaNum}" + ); + """ + try { + test { + sql "ALTER SYSTEM DECOMMISSION BACKEND '${targetBackend.Host}:${targetBackend.HeartbeatPort}'" + exception "otherwise need to decrease the partition's replication num" + } + } finally { + sql "CANCEL DECOMMISSION BACKEND '${targetBackend.Host}:${targetBackend.HeartbeatPort}'" + } + sql "DROP TABLE IF EXISTS ${tbl} FORCE" +} diff --git a/regression-test/suites/node_p0/test_backend.groovy b/regression-test/suites/node_p0/test_backend.groovy index 5de31b1f964eba..fee0964d4318ea 100644 --- a/regression-test/suites/node_p0/test_backend.groovy +++ b/regression-test/suites/node_p0/test_backend.groovy @@ -39,4 +39,43 @@ suite("test_backend") { result = sql """SHOW BACKENDS;""" logger.info("result:${result}") } + + if (context.config.jdbcUser.equals("root")) { + def beId1 = null + try { + GetDebugPoint().enableDebugPointForAllFEs("SystemHandler.decommission_no_check_replica_num"); + try_sql """admin set frontend config("drop_backend_after_decommission" = "false")""" + def result = sql_return_maparray """SHOW BACKENDS;""" + logger.info("show backends result:${result}") + for (def res : result) { + beId1 = res.BackendId + break + } + result = sql """ALTER SYSTEM DECOMMISSION BACKEND "${beId1}" """ + logger.info("ALTER SYSTEM DECOMMISSION BACKEND ${result}") + result = sql_return_maparray """SHOW BACKENDS;""" + for (def res : result) { + if (res.BackendId == "${beId1}") { + assertTrue(res.SystemDecommissioned.toBoolean()) + } + } + } finally { + try { + if (beId1 != null) { + def result = sql """CANCEL DECOMMISSION BACKEND "${beId1}" """ + logger.info("CANCEL DECOMMISSION BACKEND ${result}") + + result = sql_return_maparray """SHOW BACKENDS;""" + for (def res : result) { + if (res.BackendId == "${beId1}") { + assertFalse(res.SystemDecommissioned.toBoolean()) + } + } + } + } finally { + GetDebugPoint().disableDebugPointForAllFEs('SystemHandler.decommission_no_check_replica_num'); + try_sql """admin set frontend config("drop_backend_after_decommission" = "true")""" + } + } + } } From ea0d88d08df9b29002e836ef3f1bb0bb886578dc Mon Sep 17 00:00:00 2001 From: yujun777 Date: Wed, 24 Apr 2024 14:53:28 +0800 Subject: [PATCH 2/6] fix compile --- .../src/main/java/org/apache/doris/alter/SystemHandler.java | 4 ++-- .../alter_p0/test_decommission_with_replica_num_fail.groovy | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java index bf04e54fba972c..8be8d4a215340f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java @@ -239,7 +239,7 @@ public static List checkDecommission(List hostInfos) private static void checkDecommissionWithReplicaAllocation(List decommissionBackends) throws DdlException { - if (Config.isCloudMode() || decommissionBackends.isEmpty() + if (decommissionBackends.isEmpty() || DebugPointUtil.isEnable("SystemHandler.decommission_no_check_replica_num")) { return; } @@ -291,7 +291,7 @@ private static void checkDecommissionWithReplicaAllocation(List decommi int backendNum = tagAvailBackendNums.getOrDefault(tag, 0); if (replicaNum > backendNum) { throw new DdlException("After decommission, partition " + partition.getName() - + " of table " + db.getName() + "." + tbl.getName() + + " of table " + db.getFullName() + "." + tbl.getName() + " 's replication allocation { " + replicaAlloc + " } > available backend num " + backendNum + " on tag " + tag + ", otherwise need to decrease the partition's replication num."); diff --git a/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy b/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy index ff19adae27dad2..d7941591eec273 100644 --- a/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy +++ b/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy @@ -16,10 +16,6 @@ // under the License. suite('test_decommission_with_replica_num_fail') { - if (isCloudMode()) { - return - } - def tbl = 'test_decommission_with_replica_num_fail' def backends = sql_return_maparray('show backends') def replicaNum = 0 From 6bb2496f790fdb42709e4edeafedf1832ce3d9ae Mon Sep 17 00:00:00 2001 From: yujun777 Date: Mon, 29 Apr 2024 17:06:21 +0800 Subject: [PATCH 3/6] fix test --- .../main/groovy/org/apache/doris/regression/suite/Suite.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 7575c185d54252..f0ca33486da88f 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -570,6 +570,9 @@ class Suite implements GroovyInterceptable { assert p.exitValue() == 0 } + List getFrontendIpHttpPort() { + return sql_return_maparray("show frontends").collect { it.Host + ":" + it.HttpPort }; + } void getBackendIpHttpPort(Map backendId_to_backendIP, Map backendId_to_backendHttpPort) { List> backends = sql("show backends"); From 48265ce1e0866dbcdcf31d75c0645d2a647d8b68 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Wed, 8 May 2024 14:09:52 +0800 Subject: [PATCH 4/6] update config --- regression-test/pipeline/p0/conf/fe.conf | 2 ++ regression-test/pipeline/p1/conf/be.conf | 1 + regression-test/pipeline/p1/conf/fe.conf | 2 ++ 3 files changed, 5 insertions(+) diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf index b301d04a88e9ca..1ec05be9fd9bf7 100644 --- a/regression-test/pipeline/p0/conf/fe.conf +++ b/regression-test/pipeline/p0/conf/fe.conf @@ -78,6 +78,8 @@ enable_map_type=true enable_struct_type=true enable_feature_binlog=true +enable_debug_points=true + # enable mtmv enable_mtmv = true diff --git a/regression-test/pipeline/p1/conf/be.conf b/regression-test/pipeline/p1/conf/be.conf index a97e528a6b89ca..ae95e4f65a8b8e 100644 --- a/regression-test/pipeline/p1/conf/be.conf +++ b/regression-test/pipeline/p1/conf/be.conf @@ -71,3 +71,4 @@ enable_set_in_bitmap_value=true enable_feature_binlog=true max_sys_mem_available_low_water_mark_bytes=69206016 enable_merge_on_write_correctness_check=false +enable_debug_points=true diff --git a/regression-test/pipeline/p1/conf/fe.conf b/regression-test/pipeline/p1/conf/fe.conf index adc042357ca572..10b13131e49052 100644 --- a/regression-test/pipeline/p1/conf/fe.conf +++ b/regression-test/pipeline/p1/conf/fe.conf @@ -75,6 +75,8 @@ remote_fragment_exec_timeout_ms=60000 fuzzy_test_type=p1 use_fuzzy_session_variable=true +enable_debug_points=true + # enable mtmv enable_mtmv = true From 3458926926ea809e3e8905879f1e163734a9c025 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Wed, 8 May 2024 16:54:33 +0800 Subject: [PATCH 5/6] update --- .../suites/node_p0/test_backend.groovy | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/regression-test/suites/node_p0/test_backend.groovy b/regression-test/suites/node_p0/test_backend.groovy index fee0964d4318ea..8fd5d9c15f9710 100644 --- a/regression-test/suites/node_p0/test_backend.groovy +++ b/regression-test/suites/node_p0/test_backend.groovy @@ -41,33 +41,34 @@ suite("test_backend") { } if (context.config.jdbcUser.equals("root")) { - def beId1 = null + def decommissionBe = null try { GetDebugPoint().enableDebugPointForAllFEs("SystemHandler.decommission_no_check_replica_num"); try_sql """admin set frontend config("drop_backend_after_decommission" = "false")""" def result = sql_return_maparray """SHOW BACKENDS;""" logger.info("show backends result:${result}") for (def res : result) { - beId1 = res.BackendId + decommissionBe = res break } - result = sql """ALTER SYSTEM DECOMMISSION BACKEND "${beId1}" """ + sql """CANCEL DECOMMISSION BACKEND "${decommissionBe.Host}:${decommissionBe.HeartbeatPort}" """ + result = sql """ALTER SYSTEM DECOMMISSION BACKEND "${decommissionBe.BackendId}" """ logger.info("ALTER SYSTEM DECOMMISSION BACKEND ${result}") result = sql_return_maparray """SHOW BACKENDS;""" for (def res : result) { - if (res.BackendId == "${beId1}") { + if (res.BackendId == "${decommissionBe.BackendId}") { assertTrue(res.SystemDecommissioned.toBoolean()) } } } finally { try { - if (beId1 != null) { - def result = sql """CANCEL DECOMMISSION BACKEND "${beId1}" """ + if (decommissionBe != null) { + def result = sql """CANCEL DECOMMISSION BACKEND "${decommissionBe.Host}:${decommissionBe.HeartbeatPort}" """ logger.info("CANCEL DECOMMISSION BACKEND ${result}") result = sql_return_maparray """SHOW BACKENDS;""" for (def res : result) { - if (res.BackendId == "${beId1}") { + if (res.BackendId == "${decommissionBe.BackendId}") { assertFalse(res.SystemDecommissioned.toBoolean()) } } From f9406e16f0e41c7c68d5bf8ddadef60ea475a573 Mon Sep 17 00:00:00 2001 From: yujun777 Date: Wed, 8 May 2024 17:46:26 +0800 Subject: [PATCH 6/6] update test --- regression-test/suites/node_p0/test_backend.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/node_p0/test_backend.groovy b/regression-test/suites/node_p0/test_backend.groovy index 8fd5d9c15f9710..4c85ff1b54abce 100644 --- a/regression-test/suites/node_p0/test_backend.groovy +++ b/regression-test/suites/node_p0/test_backend.groovy @@ -52,7 +52,7 @@ suite("test_backend") { break } sql """CANCEL DECOMMISSION BACKEND "${decommissionBe.Host}:${decommissionBe.HeartbeatPort}" """ - result = sql """ALTER SYSTEM DECOMMISSION BACKEND "${decommissionBe.BackendId}" """ + result = sql """ALTER SYSTEM DECOMMISSION BACKEND "${decommissionBe.Host}:${decommissionBe.HeartbeatPort}" """ logger.info("ALTER SYSTEM DECOMMISSION BACKEND ${result}") result = sql_return_maparray """SHOW BACKENDS;""" for (def res : result) {