HBASE-29075 Balancer conditionals should support system table isolation

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java

@@ -61,6 +61,10 @@ final class BalancerConditionals implements Configurable {
     "hbase.master.balancer.stochastic.conditionals.isolateMetaTable";
   public static final boolean ISOLATE_META_TABLE_DEFAULT = false;
 
+  public static final String ISOLATE_SYSTEM_TABLES_KEY =
+    "hbase.master.balancer.stochastic.conditionals.isolateSystemTables";
+  public static final boolean ISOLATE_SYSTEM_TABLES_DEFAULT = false;
+
   public static final String ADDITIONAL_CONDITIONALS_KEY =
     "hbase.master.balancer.stochastic.additionalConditionals";
 
@@ -95,6 +99,10 @@ boolean isReplicaDistributionEnabled() {
   }
 
   boolean isTableIsolationEnabled() {
+    return conditionalClasses.stream().anyMatch(TableIsolationConditional.class::isAssignableFrom);
+  }
+
+  boolean isMetaTableIsolationEnabled() {
     return conditionalClasses.contains(MetaTableIsolationConditional.class);
   }
 
@@ -207,6 +215,12 @@ public void setConf(Configuration conf) {
       conditionalClasses.add(MetaTableIsolationConditional.class);
     }
 
+    boolean isolateSystemTables =
+      conf.getBoolean(ISOLATE_SYSTEM_TABLES_KEY, ISOLATE_SYSTEM_TABLES_DEFAULT);
+    if (isolateSystemTables) {
+      conditionalClasses.add(SystemTableIsolationConditional.class);
+    }
+
     Class<?>[] classes = conf.getClasses(ADDITIONAL_CONDITIONALS_KEY);
     for (Class<?> clazz : classes) {
       if (!RegionPlanConditional.class.isAssignableFrom(clazz)) {

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java

@@ -19,7 +19,6 @@
 
 import java.util.List;
 import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKey;
@@ -41,12 +40,9 @@ public class DistributeReplicasConditional extends RegionPlanConditional {
   public DistributeReplicasConditional(BalancerConditionals balancerConditionals,
     BalancerClusterState cluster) {
     super(balancerConditionals.getConf(), cluster);
-    Configuration conf = balancerConditionals.getConf();
-    float slop =
-      conf.getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, BaseLoadBalancer.REGIONS_SLOP_DEFAULT);
     this.candidateGenerators =
       ImmutableList.of(new DistributeReplicasCandidateGenerator(balancerConditionals),
-        new SlopFixingCandidateGenerator(balancerConditionals, slop));
+        new SlopFixingCandidateGenerator(balancerConditionals));
   }
 
   @Override

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java

@@ -27,7 +27,8 @@ class MetaTableIsolationConditional extends TableIsolationConditional {
 
   public MetaTableIsolationConditional(BalancerConditionals balancerConditionals,
     BalancerClusterState cluster) {
-    super(balancerConditionals, cluster);
+    super(new MetaTableIsolationCandidateGenerator(balancerConditionals), balancerConditionals,
+      cluster);
   }
 
   @Override

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java

@@ -37,9 +37,10 @@ final class SlopFixingCandidateGenerator extends RegionPlanConditionalCandidateG
 
   private final float slop;
 
-  SlopFixingCandidateGenerator(BalancerConditionals balancerConditionals, float slop) {
+  SlopFixingCandidateGenerator(BalancerConditionals balancerConditionals) {
     super(balancerConditionals);
-    this.slop = slop;
+    this.slop = balancerConditionals.getConf().getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY,
+      BaseLoadBalancer.REGIONS_SLOP_DEFAULT);
   }
 
   @Override

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class SystemTableIsolationCandidateGenerator extends TableIsolationCandidateGenerator {
+
+  private final BalancerConditionals balancerConditionals;
+
+  SystemTableIsolationCandidateGenerator(BalancerConditionals balancerConditionals) {
+    super(balancerConditionals);
+    this.balancerConditionals = balancerConditionals;
+  }
+
+  @Override
+  boolean shouldBeIsolated(RegionInfo regionInfo) {
+    if (balancerConditionals.isMetaTableIsolationEnabled() && regionInfo.isMetaRegion()) {
+      // If meta isolation is enabled, we can ignore meta regions here
+      return false;
+    }
+    return regionInfo.getTable().isSystemTable();
+  }
+}

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class SystemTableIsolationConditional extends TableIsolationConditional {
+
+  private final BalancerConditionals balancerConditionals;
+
+  SystemTableIsolationConditional(BalancerConditionals balancerConditionals,
+    BalancerClusterState cluster) {
+    super(new SystemTableIsolationCandidateGenerator(balancerConditionals), balancerConditionals,
+      cluster);
+    this.balancerConditionals = balancerConditionals;
+  }
+
+  @Override
+  boolean isRegionToIsolate(RegionInfo regionInfo) {
+    if (balancerConditionals.isMetaTableIsolationEnabled() && regionInfo.isMetaRegion()) {
+      // If meta isolation is enabled, we can ignore meta regions here
+      return false;
+    }
+    return regionInfo.getTable().isSystemTable();
+  }
+}

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java

@@ -26,15 +26,12 @@ abstract class TableIsolationConditional extends RegionPlanConditional {
 
   private final List<RegionPlanConditionalCandidateGenerator> candidateGenerators;
 
-  TableIsolationConditional(BalancerConditionals balancerConditionals,
-    BalancerClusterState cluster) {
+  TableIsolationConditional(TableIsolationCandidateGenerator generator,
+    BalancerConditionals balancerConditionals, BalancerClusterState cluster) {
     super(balancerConditionals.getConf(), cluster);
 
-    float slop = balancerConditionals.getConf().getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY,
-      BaseLoadBalancer.REGIONS_SLOP_DEFAULT);
     this.candidateGenerators =
-      List.of(new MetaTableIsolationCandidateGenerator(balancerConditionals),
-        new SlopFixingCandidateGenerator(balancerConditionals, slop));
+      List.of(generator, new SlopFixingCandidateGenerator(balancerConditionals));
   }
 
   abstract boolean isRegionToIsolate(RegionInfo regionInfo);

hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java

@@ -49,9 +49,16 @@ private CandidateGeneratorTestUtil() {
   static void runBalancerToExhaustion(Configuration conf,
     Map<ServerName, List<RegionInfo>> serverToRegions,
     Set<Function<BalancerClusterState, Boolean>> expectations, float targetMaxBalancerCost) {
+    runBalancerToExhaustion(conf, serverToRegions, expectations, targetMaxBalancerCost, 15000);
+  }
+
+  static void runBalancerToExhaustion(Configuration conf,
+    Map<ServerName, List<RegionInfo>> serverToRegions,
+    Set<Function<BalancerClusterState, Boolean>> expectations, float targetMaxBalancerCost,
+    long maxRunningTime) {
     // Do the full plan. We're testing with a lot of regions
     conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true);
-    conf.setLong(MAX_RUNNING_TIME_KEY, 15000);
+    conf.setLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
 
     conf.setFloat(MIN_COST_NEED_BALANCE_KEY, targetMaxBalancerCost);
 

hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.isTableIsolated;
+import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ MediumTests.class, MasterTests.class })
+public class TestLargeClusterBalancingSystemTableIsolation {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+    HBaseClassTestRule.forClass(TestLargeClusterBalancingSystemTableIsolation.class);
+
+  private static final Logger LOG =
+    LoggerFactory.getLogger(TestLargeClusterBalancingSystemTableIsolation.class);
+
+  private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system");
+  private static final TableName NON_SYSTEM_TABLE_NAME = TableName.valueOf("userTable");
+
+  private static final int NUM_SERVERS = 1000;
+  private static final int NUM_REGIONS = 20_000;
+
+  private static final ServerName[] servers = new ServerName[NUM_SERVERS];
+  private static final Map<ServerName, List<RegionInfo>> serverToRegions = new HashMap<>();
+
+  @BeforeClass
+  public static void setup() {
+    // Initialize servers
+    for (int i = 0; i < NUM_SERVERS; i++) {
+      servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis());
+    }
+
+    // Create regions
+    List<RegionInfo> allRegions = new ArrayList<>();
+    for (int i = 0; i < NUM_REGIONS; i++) {
+      TableName tableName = i < 3 ? SYSTEM_TABLE_NAME : NON_SYSTEM_TABLE_NAME;
+      byte[] startKey = new byte[1];
+      startKey[0] = (byte) i;
+      byte[] endKey = new byte[1];
+      endKey[0] = (byte) (i + 1);
+
+      RegionInfo regionInfo =
+        RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey).build();
+      allRegions.add(regionInfo);
+    }
+
+    // Assign all regions to the first server
+    serverToRegions.put(servers[0], new ArrayList<>(allRegions));
+    for (int i = 1; i < NUM_SERVERS; i++) {
+      serverToRegions.put(servers[i], new ArrayList<>());
+    }
+  }
+
+  @Test
+  public void testSystemTableIsolation() {
+    Configuration conf = new Configuration(false);
+    conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true);
+    runBalancerToExhaustion(conf, serverToRegions, Set.of(this::isSystemTableIsolated), 10.0f);
+    LOG.info("Meta table regions are successfully isolated.");
+  }
+
+  private boolean isSystemTableIsolated(BalancerClusterState cluster) {
+    return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System");
+  }
+
+}

hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java

@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -52,8 +53,8 @@ public class TestLargeClusterBalancingTableIsolationAndReplicaDistribution {
   private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system");
   private static final TableName NON_ISOLATED_TABLE_NAME = TableName.valueOf("userTable");
 
-  private static final int NUM_SERVERS = 1000;
-  private static final int NUM_REGIONS = 10_000;
+  private static final int NUM_SERVERS = 500;
+  private static final int NUM_REGIONS = 2_500;
   private static final int NUM_REPLICAS = 3;
 
   private static final ServerName[] servers = new ServerName[NUM_SERVERS];
@@ -68,7 +69,6 @@ public static void setup() {
     }
 
     // Create primary regions and their replicas
-    List<RegionInfo> allRegions = new ArrayList<>();
     for (int i = 0; i < NUM_REGIONS; i++) {
       TableName tableName;
       if (i < 1) {
@@ -85,30 +85,28 @@ public static void setup() {
       byte[] endKey = new byte[1];
       endKey[0] = (byte) (i + 1);
 
+      Random random = new Random();
       // Create 3 replicas for each primary region
       for (int replicaId = 0; replicaId < NUM_REPLICAS; replicaId++) {
         RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey)
           .setEndKey(endKey).setReplicaId(replicaId).build();
-        allRegions.add(regionInfo);
+        // Assign region to random server
+        int randomServer = random.nextInt(servers.length);
+        serverToRegions.get(servers[randomServer]).add(regionInfo);
       }
     }
-
-    // Assign all regions to one server
-    for (RegionInfo regionInfo : allRegions) {
-      serverToRegions.get(servers[0]).add(regionInfo);
-    }
   }
 
   @Test
   public void testTableIsolationAndReplicaDistribution() {
-
     Configuration conf = new Configuration(false);
     conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true);
+    conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true);
     DistributeReplicasTestConditional.enableConditionalReplicaDistributionForTest(conf);
 
-    runBalancerToExhaustion(conf, serverToRegions,
-      Set.of(this::isMetaTableIsolated, CandidateGeneratorTestUtil::areAllReplicasDistributed),
-      10.0f);
+    runBalancerToExhaustion(conf, serverToRegions, Set.of(this::isMetaTableIsolated,
+      this::isSystemTableIsolated, CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f,
+      60_000);
     LOG.info("Meta table regions are successfully isolated, "
       + "and region replicas are appropriately distributed.");
   }
@@ -119,4 +117,11 @@ public void testTableIsolationAndReplicaDistribution() {
   private boolean isMetaTableIsolated(BalancerClusterState cluster) {
     return isTableIsolated(cluster, TableName.META_TABLE_NAME, "Meta");
   }
+
+  /**
+   * Validates whether all meta table regions are isolated.
+   */
+  private boolean isSystemTableIsolated(BalancerClusterState cluster) {
+    return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System");
+  }
 }