diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java index 88ceb5a55406..021a34bce6b9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java @@ -62,6 +62,10 @@ final class BalancerConditionals implements Configurable { "hbase.master.balancer.stochastic.conditionals.isolateMetaTable"; public static final boolean ISOLATE_META_TABLE_DEFAULT = false; + public static final String ISOLATE_SYSTEM_TABLES_KEY = + "hbase.master.balancer.stochastic.conditionals.isolateSystemTables"; + public static final boolean ISOLATE_SYSTEM_TABLES_DEFAULT = false; + public static final String ADDITIONAL_CONDITIONALS_KEY = "hbase.master.balancer.stochastic.additionalConditionals"; @@ -96,6 +100,10 @@ boolean isReplicaDistributionEnabled() { } boolean isTableIsolationEnabled() { + return conditionalClasses.stream().anyMatch(TableIsolationConditional.class::isAssignableFrom); + } + + boolean isMetaTableIsolationEnabled() { return conditionalClasses.contains(MetaTableIsolationConditional.class); } @@ -208,6 +216,12 @@ public void setConf(Configuration conf) { conditionalClasses.add(MetaTableIsolationConditional.class); } + boolean isolateSystemTables = + conf.getBoolean(ISOLATE_SYSTEM_TABLES_KEY, ISOLATE_SYSTEM_TABLES_DEFAULT); + if (isolateSystemTables) { + conditionalClasses.add(SystemTableIsolationConditional.class); + } + Class[] classes = conf.getClasses(ADDITIONAL_CONDITIONALS_KEY); for (Class clazz : classes) { if (!RegionPlanConditional.class.isAssignableFrom(clazz)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java index 2cd27615e5fd..e99c0e93a159 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Set; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKey; @@ -41,12 +40,9 @@ public class DistributeReplicasConditional extends RegionPlanConditional { public DistributeReplicasConditional(BalancerConditionals balancerConditionals, BalancerClusterState cluster) { super(balancerConditionals.getConf(), cluster); - Configuration conf = balancerConditionals.getConf(); - float slop = - conf.getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, BaseLoadBalancer.REGIONS_SLOP_DEFAULT); this.candidateGenerators = ImmutableList.of(new DistributeReplicasCandidateGenerator(balancerConditionals), - new SlopFixingCandidateGenerator(balancerConditionals, slop)); + new SlopFixingCandidateGenerator(balancerConditionals)); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java index 732693c44f3e..5617468457c4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java @@ -27,7 +27,8 @@ class MetaTableIsolationConditional extends TableIsolationConditional { public MetaTableIsolationConditional(BalancerConditionals balancerConditionals, BalancerClusterState cluster) { - super(balancerConditionals, cluster); + super(new MetaTableIsolationCandidateGenerator(balancerConditionals), balancerConditionals, + cluster); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java index b1ea1de8d2b0..f78e1573b417 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java @@ -37,9 +37,10 @@ final class SlopFixingCandidateGenerator extends RegionPlanConditionalCandidateG private final float slop; - SlopFixingCandidateGenerator(BalancerConditionals balancerConditionals, float slop) { + SlopFixingCandidateGenerator(BalancerConditionals balancerConditionals) { super(balancerConditionals); - this.slop = slop; + this.slop = balancerConditionals.getConf().getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, + BaseLoadBalancer.REGIONS_SLOP_DEFAULT); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java new file mode 100644 index 000000000000..7ce8ff202965 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class SystemTableIsolationCandidateGenerator extends TableIsolationCandidateGenerator { + + private final BalancerConditionals balancerConditionals; + + SystemTableIsolationCandidateGenerator(BalancerConditionals balancerConditionals) { + super(balancerConditionals); + this.balancerConditionals = balancerConditionals; + } + + @Override + boolean shouldBeIsolated(RegionInfo regionInfo) { + if (balancerConditionals.isMetaTableIsolationEnabled() && regionInfo.isMetaRegion()) { + // If meta isolation is enabled, we can ignore meta regions here + return false; + } + return regionInfo.getTable().isSystemTable(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java new file mode 100644 index 000000000000..b5734b82faf7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class SystemTableIsolationConditional extends TableIsolationConditional { + + private final BalancerConditionals balancerConditionals; + + SystemTableIsolationConditional(BalancerConditionals balancerConditionals, + BalancerClusterState cluster) { + super(new SystemTableIsolationCandidateGenerator(balancerConditionals), balancerConditionals, + cluster); + this.balancerConditionals = balancerConditionals; + } + + @Override + boolean isRegionToIsolate(RegionInfo regionInfo) { + if (balancerConditionals.isMetaTableIsolationEnabled() && regionInfo.isMetaRegion()) { + // If meta isolation is enabled, we can ignore meta regions here + return false; + } + return regionInfo.getTable().isSystemTable(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java index cd3ce0b6fe18..24a6f519e8d8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java @@ -28,15 +28,12 @@ abstract class TableIsolationConditional extends RegionPlanConditional { private final List candidateGenerators; - TableIsolationConditional(BalancerConditionals balancerConditionals, - BalancerClusterState cluster) { + TableIsolationConditional(TableIsolationCandidateGenerator generator, + BalancerConditionals balancerConditionals, BalancerClusterState cluster) { super(balancerConditionals.getConf(), cluster); - float slop = balancerConditionals.getConf().getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, - BaseLoadBalancer.REGIONS_SLOP_DEFAULT); this.candidateGenerators = - ImmutableList.of(new MetaTableIsolationCandidateGenerator(balancerConditionals), - new SlopFixingCandidateGenerator(balancerConditionals, slop)); + ImmutableList.of(generator, new SlopFixingCandidateGenerator(balancerConditionals)); } abstract boolean isRegionToIsolate(RegionInfo regionInfo); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java index d2a2d432ff05..03bfcce8e150 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java @@ -54,9 +54,16 @@ private CandidateGeneratorTestUtil() { static void runBalancerToExhaustion(Configuration conf, Map> serverToRegions, Set> expectations, float targetMaxBalancerCost) { + runBalancerToExhaustion(conf, serverToRegions, expectations, targetMaxBalancerCost, 15000); + } + + static void runBalancerToExhaustion(Configuration conf, + Map> serverToRegions, + Set> expectations, float targetMaxBalancerCost, + long maxRunningTime) { // Do the full plan. We're testing with a lot of regions conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); - conf.setLong(MAX_RUNNING_TIME_KEY, 15000); + conf.setLong(MAX_RUNNING_TIME_KEY, maxRunningTime); conf.setFloat(MIN_COST_NEED_BALANCE_KEY, targetMaxBalancerCost); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java new file mode 100644 index 000000000000..ef26c548c209 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.isTableIsolated; +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category({ MediumTests.class, MasterTests.class }) +public class TestLargeClusterBalancingSystemTableIsolation { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestLargeClusterBalancingSystemTableIsolation.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestLargeClusterBalancingSystemTableIsolation.class); + + private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system"); + private static final TableName NON_SYSTEM_TABLE_NAME = TableName.valueOf("userTable"); + + private static final int NUM_SERVERS = 1000; + private static final int NUM_REGIONS = 20_000; + + private static final ServerName[] servers = new ServerName[NUM_SERVERS]; + private static final Map> serverToRegions = new HashMap<>(); + + @BeforeClass + public static void setup() { + // Initialize servers + for (int i = 0; i < NUM_SERVERS; i++) { + servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis()); + } + + // Create regions + List allRegions = new ArrayList<>(); + for (int i = 0; i < NUM_REGIONS; i++) { + TableName tableName = i < 3 ? SYSTEM_TABLE_NAME : NON_SYSTEM_TABLE_NAME; + byte[] startKey = new byte[1]; + startKey[0] = (byte) i; + byte[] endKey = new byte[1]; + endKey[0] = (byte) (i + 1); + + RegionInfo regionInfo = + RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey).build(); + allRegions.add(regionInfo); + } + + // Assign all regions to the first server + serverToRegions.put(servers[0], new ArrayList<>(allRegions)); + for (int i = 1; i < NUM_SERVERS; i++) { + serverToRegions.put(servers[i], new ArrayList<>()); + } + } + + @Test + public void testSystemTableIsolation() { + Configuration conf = new Configuration(false); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isSystemTableIsolated), + 10.0f); + LOG.info("Meta table regions are successfully isolated."); + } + + private boolean isSystemTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System"); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java index 5fbddf4878be..3a28ae801e4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; @@ -53,8 +54,8 @@ public class TestLargeClusterBalancingTableIsolationAndReplicaDistribution { private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system"); private static final TableName NON_ISOLATED_TABLE_NAME = TableName.valueOf("userTable"); - private static final int NUM_SERVERS = 1000; - private static final int NUM_REGIONS = 10_000; + private static final int NUM_SERVERS = 500; + private static final int NUM_REGIONS = 2_500; private static final int NUM_REPLICAS = 3; private static final ServerName[] servers = new ServerName[NUM_SERVERS]; @@ -69,7 +70,6 @@ public static void setup() { } // Create primary regions and their replicas - List allRegions = new ArrayList<>(); for (int i = 0; i < NUM_REGIONS; i++) { TableName tableName; if (i < 1) { @@ -86,29 +86,28 @@ public static void setup() { byte[] endKey = new byte[1]; endKey[0] = (byte) (i + 1); + Random random = new Random(); // Create 3 replicas for each primary region for (int replicaId = 0; replicaId < NUM_REPLICAS; replicaId++) { RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey) .setEndKey(endKey).setReplicaId(replicaId).build(); - allRegions.add(regionInfo); + // Assign region to random server + int randomServer = random.nextInt(servers.length); + serverToRegions.get(servers[randomServer]).add(regionInfo); } } - - // Assign all regions to one server - for (RegionInfo regionInfo : allRegions) { - serverToRegions.get(servers[0]).add(regionInfo); - } } @Test public void testTableIsolationAndReplicaDistribution() { - Configuration conf = new Configuration(false); conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); DistributeReplicasTestConditional.enableConditionalReplicaDistributionForTest(conf); runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isMetaTableIsolated, - CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f); + this::isSystemTableIsolated, CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f, + 60_000); LOG.info("Meta table regions are successfully isolated, " + "and region replicas are appropriately distributed."); } @@ -119,4 +118,11 @@ public void testTableIsolationAndReplicaDistribution() { private boolean isMetaTableIsolated(BalancerClusterState cluster) { return isTableIsolated(cluster, TableName.META_TABLE_NAME, "Meta"); } + + /** + * Validates whether all meta table regions are isolated. + */ + private boolean isSystemTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System"); + } }