From 54d6d43f5bdb05b06dd4cfb9d1f7a839f01db682 Mon Sep 17 00:00:00 2001 From: Clara Xiong Date: Wed, 3 Mar 2021 17:27:31 -0800 Subject: [PATCH 1/3] HBASE-25625 StochasticBalancer CostFunctions needs a better way to evaluate resource distribution --- .../master/balancer/BaseLoadBalancer.java | 108 ++++++++++++++---- .../balancer/StochasticLoadBalancer.java | 52 ++------- .../master/balancer/TestBaseLoadBalancer.java | 2 +- 3 files changed, 93 insertions(+), 69 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 91215c7e265a..223964b78a27 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -168,7 +168,9 @@ protected static class Cluster { int[] initialRegionIndexToServerIndex; //regionIndex -> serverIndex (initial cluster state) int[] regionIndexToTableIndex; //regionIndex -> tableIndex int[][] numRegionsPerServerPerTable; //serverIndex -> tableIndex -> # regions - int[] numMaxRegionsPerTable; //tableIndex -> max number of regions in a single RS + int[] regionsPerTable; // count of regions per table + double[] regionStDevPerTable; //tableIndex -> standard deviation of region distribution + // on servers per table int[] regionIndexToPrimaryIndex; //regionIndex -> regionIndex of the primary boolean hasRegionReplicas = false; //whether there is regions with replicas @@ -297,7 +299,7 @@ protected Cluster( primariesOfRegionsPerHost = new int[numHosts][]; primariesOfRegionsPerRack = new int[numRacks][]; - int tableIndex = 0, regionIndex = 0, regionPerServerIndex = 0; + int regionIndex = 0, regionPerServerIndex = 0; for (Entry> entry : clusterState.entrySet()) { if (entry.getKey() == null) { @@ -383,20 +385,20 @@ protected Cluster( } } + regionsPerTable = new int[numTables]; + for (int i = 0; i < numTables; i++) { + regionsPerTable[i] = 0; + } + for (int i=0; i < regionIndexToServerIndex.length; i++) { if (regionIndexToServerIndex[i] >= 0) { numRegionsPerServerPerTable[regionIndexToServerIndex[i]][regionIndexToTableIndex[i]]++; + regionsPerTable[regionIndexToTableIndex[i]]++; } } - numMaxRegionsPerTable = new int[numTables]; - for (int[] aNumRegionsPerServerPerTable : numRegionsPerServerPerTable) { - for (tableIndex = 0; tableIndex < aNumRegionsPerServerPerTable.length; tableIndex++) { - if (aNumRegionsPerServerPerTable[tableIndex] > numMaxRegionsPerTable[tableIndex]) { - numMaxRegionsPerTable[tableIndex] = aNumRegionsPerServerPerTable[tableIndex]; - } - } - } + regionStDevPerTable = new double[numTables]; + reComputeRegionStDevPerTable(); for (int i = 0; i < regions.length; i ++) { RegionInfo info = regions[i]; @@ -513,6 +515,75 @@ private void registerRegion(RegionInfo region, int regionIndex, } } + private void reComputeRegionStDevPerTable() { + for (int i = 0; i < numTables; i++) { + regionStDevPerTable[i] = 0; + } + + for (int[] aNumRegionsPerServerPerTable : numRegionsPerServerPerTable) { + for (int tableIndex = 0; tableIndex < aNumRegionsPerServerPerTable.length; tableIndex++) { + double deviation = aNumRegionsPerServerPerTable[tableIndex] + - regionsPerTable[tableIndex] / numServers; + regionStDevPerTable[tableIndex] += deviation * deviation; + } + } + + for (int i = 0; i < numTables; i++) { + regionStDevPerTable[i] = scale(getMinStDev(regionsPerTable[i], numServers), + getMaxStDev(regionsPerTable[i], numServers), + Math.sqrt(regionStDevPerTable[i] / numServers)); + } + } + + /** + * Return the max standard deviation of distribution of regions + * Compute max as if all region servers had 0 and one had the sum of all costs. This must be + * a zero sum cost for this to make sense. + */ + public double getMaxStDev(double total, double numServers) { + double mean = total / numServers; + return Math.sqrt(((total - mean) * (total - mean) + + (numServers - 1) * mean * mean) + / numServers); + } + + /** + * Return the min standard deviation of distribution of regions + */ + public double getMinStDev(double total, double numServers) { + double mean = total / numServers; + // It's possible that there aren't enough regions to go around + double min; + if (numServers > total) { + min = Math.sqrt(((numServers - total) * mean * mean + + (1 - mean) * (1 - mean) * total) / numServers); + } else { + // Some will have 1 more than everything else. + int numHigh = (int) (total - (Math.floor(mean) * numServers)); + int numLow = (int) (numServers - numHigh); + min = Math.sqrt(((numHigh * (Math.ceil(mean) - mean) * (Math.ceil(mean) - mean)) + + (numLow * (mean - Math.floor(mean)) * (mean - Math.floor(mean)))) / numServers); + } + return min; + } + + /** + * Scale the value between 0 and 1. + * + * @param min Min value + * @param max The Max value + * @param value The value to be scaled. + * @return The scaled value. + */ + public double scale(double min, double max, double value) { + if (max <= min || value <= min) { + return 0; + } + if ((max - min) == 0) return 0; + + return Math.max(0d, Math.min(1d, (value - min) / (max - min))); + } + /** * Returns true iff a given server has less regions than the balanced amount */ @@ -837,19 +908,8 @@ void regionMoved(int region, int oldServer, int newServer) { } numRegionsPerServerPerTable[newServer][tableIndex]++; - //check whether this caused maxRegionsPerTable in the new Server to be updated - if (numRegionsPerServerPerTable[newServer][tableIndex] > numMaxRegionsPerTable[tableIndex]) { - numMaxRegionsPerTable[tableIndex] = numRegionsPerServerPerTable[newServer][tableIndex]; - } else if (oldServer >= 0 && (numRegionsPerServerPerTable[oldServer][tableIndex] + 1) - == numMaxRegionsPerTable[tableIndex]) { - //recompute maxRegionsPerTable since the previous value was coming from the old server - numMaxRegionsPerTable[tableIndex] = 0; - for (int[] aNumRegionsPerServerPerTable : numRegionsPerServerPerTable) { - if (aNumRegionsPerServerPerTable[tableIndex] > numMaxRegionsPerTable[tableIndex]) { - numMaxRegionsPerTable[tableIndex] = aNumRegionsPerServerPerTable[tableIndex]; - } - } - } + // recalculate stdev + reComputeRegionStDevPerTable(); // update for servers int primary = regionIndexToPrimaryIndex[region]; @@ -1019,7 +1079,7 @@ public String toString() { .append(Arrays.toString(serverIndicesSortedByRegionCount)) .append(", regionsPerServer=").append(Arrays.deepToString(regionsPerServer)); - desc.append(", numMaxRegionsPerTable=").append(Arrays.toString(numMaxRegionsPerTable)) + desc.append(", regionStDevPerTable=").append(Arrays.toString(regionStDevPerTable)) .append(", numRegions=").append(numRegions).append(", numServers=").append(numServers) .append(", numTables=").append(numTables).append(", numMovedRegions=") .append(numMovedRegions).append('}'); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 3f249b890133..2506fe7a6ae0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -817,31 +817,14 @@ protected double costFromArray(double[] stats) { double count = stats.length; double mean = total/count; - // Compute max as if all region servers had 0 and one had the sum of all costs. This must be - // a zero sum cost for this to make sense. - double max = ((count - 1) * mean) + (total - mean); - - // It's possible that there aren't enough regions to go around - double min; - if (count > total) { - min = ((count - total) * mean) + ((1 - mean) * total); - } else { - // Some will have 1 more than everything else. - int numHigh = (int) (total - (Math.floor(mean) * count)); - int numLow = (int) (count - numHigh); - - min = (numHigh * (Math.ceil(mean) - mean)) + (numLow * (mean - Math.floor(mean))); - - } - min = Math.max(0, min); for (int i=0; i Date: Wed, 3 Mar 2021 23:25:01 -0800 Subject: [PATCH 2/3] fix compile error test updates --- .../hadoop/hbase/master/balancer/TestBaseLoadBalancer.java | 2 +- .../hbase/master/balancer/TestStochasticLoadBalancer.java | 4 ++-- .../balancer/TestStochasticLoadBalancerLargeCluster.java | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java index 4164d16777bc..1d1ebd8189df 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java @@ -391,7 +391,7 @@ public void testRegionAvailabilityWithRegionMoves() throws Exception { // now move region1 from servers[0] to servers[2] cluster.doAction(new MoveRegionAction(0, 0, 2)); // check that the numMaxRegionsPerTable for "table" has increased to 2 - assertEquals(1, cluster.regionStDevPerTable[0]); + assertEquals(0.5773502691896257, cluster.regionStDevPerTable[0], 0.01); // now repeat check whether moving region1 from servers[1] to servers[2] // would lower availability assertTrue(cluster.wouldLowerAvailability(hri1, servers[2])); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index b1e81c482002..52eca5d9cbb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -409,7 +409,7 @@ public void testTableSkewCost() { BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster); costFunction.init(cluster); double cost = costFunction.cost(); - assertTrue(cost >= 0); + assertTrue(cost >= -0.01, "cost=" + cost); assertTrue(cost <= 1.01); } } @@ -470,7 +470,7 @@ public void testCostFromArray() { statThree[i] = (0); statThree[i+100] = 100; } - assertEquals(0.5, costFunction.costFromArray(statThree), 0.01); + assertEquals(0.0708881205008336, costFunction.costFromArray(statThree), 0.01); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java index da38187cce67..5b1f68f24ab9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java @@ -38,6 +38,8 @@ public void testLargeCluster() { int numRegionsPerServer = 80; // all servers except one int numTables = 100; int replication = 1; - testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true); + + // we need to capture the outlier and generate a move + testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, false); } } From b08e6b16c7cfb094534e32d281dde9c595bf9372 Mon Sep 17 00:00:00 2001 From: Clara Xiong Date: Fri, 5 Mar 2021 14:49:01 -0800 Subject: [PATCH 3/3] combine table skew across multiple tables as independent variables --- .../master/balancer/BaseLoadBalancer.java | 81 ++++++++++++++----- .../balancer/StochasticLoadBalancer.java | 8 +- .../master/balancer/TestBaseLoadBalancer.java | 4 +- .../balancer/TestStochasticLoadBalancer.java | 2 +- ...estStochasticLoadBalancerLargeCluster.java | 2 +- 5 files changed, 68 insertions(+), 29 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 223964b78a27..93c36ea166c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -169,8 +169,7 @@ protected static class Cluster { int[] regionIndexToTableIndex; //regionIndex -> tableIndex int[][] numRegionsPerServerPerTable; //serverIndex -> tableIndex -> # regions int[] regionsPerTable; // count of regions per table - double[] regionStDevPerTable; //tableIndex -> standard deviation of region distribution - // on servers per table + double regionStDevByTable; // standard deviation of region distribution by table int[] regionIndexToPrimaryIndex; //regionIndex -> regionIndex of the primary boolean hasRegionReplicas = false; //whether there is regions with replicas @@ -397,8 +396,7 @@ protected Cluster( } } - regionStDevPerTable = new double[numTables]; - reComputeRegionStDevPerTable(); + recomputeRegionStDevByTable(); for (int i = 0; i < regions.length; i ++) { RegionInfo info = regions[i]; @@ -515,28 +513,28 @@ private void registerRegion(RegionInfo region, int regionIndex, } } - private void reComputeRegionStDevPerTable() { - for (int i = 0; i < numTables; i++) { - regionStDevPerTable[i] = 0; - } + private void recomputeRegionStDevByTable() { + regionStDevByTable = 0; for (int[] aNumRegionsPerServerPerTable : numRegionsPerServerPerTable) { for (int tableIndex = 0; tableIndex < aNumRegionsPerServerPerTable.length; tableIndex++) { double deviation = aNumRegionsPerServerPerTable[tableIndex] - - regionsPerTable[tableIndex] / numServers; - regionStDevPerTable[tableIndex] += deviation * deviation; + - Double.valueOf(regionsPerTable[tableIndex]) / numServers; + regionStDevByTable += deviation * deviation; } } - for (int i = 0; i < numTables; i++) { - regionStDevPerTable[i] = scale(getMinStDev(regionsPerTable[i], numServers), - getMaxStDev(regionsPerTable[i], numServers), - Math.sqrt(regionStDevPerTable[i] / numServers)); + if (numServers == 0) { + regionStDevByTable = regionStDevByTable == 0? 0 : 1; } + + regionStDevByTable = scale(getMinStDevForMultiVar(regionsPerTable, numServers), + getMaxStDevForMultiVar(regionsPerTable, numServers), + Math.sqrt(regionStDevByTable / numServers)); } /** - * Return the max standard deviation of distribution of regions + * Return the max standard deviation of distribution * Compute max as if all region servers had 0 and one had the sum of all costs. This must be * a zero sum cost for this to make sense. */ @@ -548,7 +546,27 @@ public double getMaxStDev(double total, double numServers) { } /** - * Return the min standard deviation of distribution of regions + * Return the max standard deviation of distribution of multiple variables + * Compute max as if all region servers had 0 and one had the sum of all costs. This must be + * a zero sum cost for this to make sense. + */ + public double getMaxStDevForMultiVar(int[] total, double numServers) { + if (total == null || total.length == 0 || numServers == 0) + { + return 0; + } + + double variance = 0; + for (int i = 0; i < total.length; i++) { + double mean = total[i] / numServers; + variance += (total[i] - mean) * (total[i] - mean) + (numServers - 1) * mean * mean; + } + + return Math.sqrt(variance / numServers); + } + + /** + * Return the min standard deviation of distribution */ public double getMinStDev(double total, double numServers) { double mean = total / numServers; @@ -567,6 +585,33 @@ public double getMinStDev(double total, double numServers) { return min; } + /** + * Return the min standard deviation of distribution of multiple variables + */ + public double getMinStDevForMultiVar(int[] total, double numServers) { + if (total == null || total.length == 0 || numServers == 0) + { + return 0; + } + + double variance = 0; + for (int i = 0; i < total.length; i++) { + double mean = total[i] / numServers; + // It's possible that there aren't enough regions to go around + if (numServers > total[i]) { + variance += (numServers - total[i]) * mean * mean + (1 - mean) * (1 - mean) * total[i]; + } else { + // Some will have 1 more than everything else. + int numHigh = (int) (total[i] - (Math.floor(mean) * numServers)); + int numLow = (int) (numServers - numHigh); + variance += numHigh * (Math.ceil(mean) - mean) * (Math.ceil(mean) - mean) + + (numLow * (mean - Math.floor(mean)) * (mean - Math.floor(mean))); + } + } + + return Math.sqrt(variance / numServers); + } + /** * Scale the value between 0 and 1. * @@ -909,7 +954,7 @@ void regionMoved(int region, int oldServer, int newServer) { numRegionsPerServerPerTable[newServer][tableIndex]++; // recalculate stdev - reComputeRegionStDevPerTable(); + recomputeRegionStDevByTable(); // update for servers int primary = regionIndexToPrimaryIndex[region]; @@ -1079,7 +1124,7 @@ public String toString() { .append(Arrays.toString(serverIndicesSortedByRegionCount)) .append(", regionsPerServer=").append(Arrays.deepToString(regionsPerServer)); - desc.append(", regionStDevPerTable=").append(Arrays.toString(regionStDevPerTable)) + desc.append(", regionStDevByTable=").append(regionStDevByTable) .append(", numRegions=").append(numRegions).append(", numServers=").append(numServers) .append(", numTables=").append(numTables).append(", numMovedRegions=") .append(numMovedRegions).append('}'); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 2506fe7a6ae0..ffc77e7db586 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -990,13 +990,7 @@ static class TableSkewCostFunction extends CostFunction { @Override protected double cost() { - double value = 0; - - for (int i = 0; i < cluster.regionStDevPerTable.length; i++) { - value += cluster.regionStDevPerTable[i]; - } - - return value / cluster.regionStDevPerTable.length; + return cluster.regionStDevByTable; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java index 1d1ebd8189df..92c0ae39d2f0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java @@ -390,8 +390,8 @@ public void testRegionAvailabilityWithRegionMoves() throws Exception { // now move region1 from servers[0] to servers[2] cluster.doAction(new MoveRegionAction(0, 0, 2)); - // check that the numMaxRegionsPerTable for "table" has increased to 2 - assertEquals(0.5773502691896257, cluster.regionStDevPerTable[0], 0.01); + // check that the regionStDevByTable for "table" has increased + assertEquals(0.5773502691896257, cluster.regionStDevByTable, 0.01); // now repeat check whether moving region1 from servers[1] to servers[2] // would lower availability assertTrue(cluster.wouldLowerAvailability(hri1, servers[2])); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index 52eca5d9cbb4..bb1475373203 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -409,7 +409,7 @@ public void testTableSkewCost() { BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster); costFunction.init(cluster); double cost = costFunction.cost(); - assertTrue(cost >= -0.01, "cost=" + cost); + assertTrue(cost >= -0.01); assertTrue(cost <= 1.01); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java index 5b1f68f24ab9..9b43360b7ccc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java @@ -40,6 +40,6 @@ public void testLargeCluster() { int replication = 1; // we need to capture the outlier and generate a move - testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, false); + testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true); } }