diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 91215c7e265a..93c36ea166c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -168,7 +168,8 @@ protected static class Cluster { int[] initialRegionIndexToServerIndex; //regionIndex -> serverIndex (initial cluster state) int[] regionIndexToTableIndex; //regionIndex -> tableIndex int[][] numRegionsPerServerPerTable; //serverIndex -> tableIndex -> # regions - int[] numMaxRegionsPerTable; //tableIndex -> max number of regions in a single RS + int[] regionsPerTable; // count of regions per table + double regionStDevByTable; // standard deviation of region distribution by table int[] regionIndexToPrimaryIndex; //regionIndex -> regionIndex of the primary boolean hasRegionReplicas = false; //whether there is regions with replicas @@ -297,7 +298,7 @@ protected Cluster( primariesOfRegionsPerHost = new int[numHosts][]; primariesOfRegionsPerRack = new int[numRacks][]; - int tableIndex = 0, regionIndex = 0, regionPerServerIndex = 0; + int regionIndex = 0, regionPerServerIndex = 0; for (Entry> entry : clusterState.entrySet()) { if (entry.getKey() == null) { @@ -383,20 +384,19 @@ protected Cluster( } } + regionsPerTable = new int[numTables]; + for (int i = 0; i < numTables; i++) { + regionsPerTable[i] = 0; + } + for (int i=0; i < regionIndexToServerIndex.length; i++) { if (regionIndexToServerIndex[i] >= 0) { numRegionsPerServerPerTable[regionIndexToServerIndex[i]][regionIndexToTableIndex[i]]++; + regionsPerTable[regionIndexToTableIndex[i]]++; } } - numMaxRegionsPerTable = new int[numTables]; - for (int[] aNumRegionsPerServerPerTable : numRegionsPerServerPerTable) { - for (tableIndex = 0; tableIndex < aNumRegionsPerServerPerTable.length; tableIndex++) { - if (aNumRegionsPerServerPerTable[tableIndex] > numMaxRegionsPerTable[tableIndex]) { - numMaxRegionsPerTable[tableIndex] = aNumRegionsPerServerPerTable[tableIndex]; - } - } - } + recomputeRegionStDevByTable(); for (int i = 0; i < regions.length; i ++) { RegionInfo info = regions[i]; @@ -513,6 +513,122 @@ private void registerRegion(RegionInfo region, int regionIndex, } } + private void recomputeRegionStDevByTable() { + regionStDevByTable = 0; + + for (int[] aNumRegionsPerServerPerTable : numRegionsPerServerPerTable) { + for (int tableIndex = 0; tableIndex < aNumRegionsPerServerPerTable.length; tableIndex++) { + double deviation = aNumRegionsPerServerPerTable[tableIndex] + - Double.valueOf(regionsPerTable[tableIndex]) / numServers; + regionStDevByTable += deviation * deviation; + } + } + + if (numServers == 0) { + regionStDevByTable = regionStDevByTable == 0? 0 : 1; + } + + regionStDevByTable = scale(getMinStDevForMultiVar(regionsPerTable, numServers), + getMaxStDevForMultiVar(regionsPerTable, numServers), + Math.sqrt(regionStDevByTable / numServers)); + } + + /** + * Return the max standard deviation of distribution + * Compute max as if all region servers had 0 and one had the sum of all costs. This must be + * a zero sum cost for this to make sense. + */ + public double getMaxStDev(double total, double numServers) { + double mean = total / numServers; + return Math.sqrt(((total - mean) * (total - mean) + + (numServers - 1) * mean * mean) + / numServers); + } + + /** + * Return the max standard deviation of distribution of multiple variables + * Compute max as if all region servers had 0 and one had the sum of all costs. This must be + * a zero sum cost for this to make sense. + */ + public double getMaxStDevForMultiVar(int[] total, double numServers) { + if (total == null || total.length == 0 || numServers == 0) + { + return 0; + } + + double variance = 0; + for (int i = 0; i < total.length; i++) { + double mean = total[i] / numServers; + variance += (total[i] - mean) * (total[i] - mean) + (numServers - 1) * mean * mean; + } + + return Math.sqrt(variance / numServers); + } + + /** + * Return the min standard deviation of distribution + */ + public double getMinStDev(double total, double numServers) { + double mean = total / numServers; + // It's possible that there aren't enough regions to go around + double min; + if (numServers > total) { + min = Math.sqrt(((numServers - total) * mean * mean + + (1 - mean) * (1 - mean) * total) / numServers); + } else { + // Some will have 1 more than everything else. + int numHigh = (int) (total - (Math.floor(mean) * numServers)); + int numLow = (int) (numServers - numHigh); + min = Math.sqrt(((numHigh * (Math.ceil(mean) - mean) * (Math.ceil(mean) - mean)) + + (numLow * (mean - Math.floor(mean)) * (mean - Math.floor(mean)))) / numServers); + } + return min; + } + + /** + * Return the min standard deviation of distribution of multiple variables + */ + public double getMinStDevForMultiVar(int[] total, double numServers) { + if (total == null || total.length == 0 || numServers == 0) + { + return 0; + } + + double variance = 0; + for (int i = 0; i < total.length; i++) { + double mean = total[i] / numServers; + // It's possible that there aren't enough regions to go around + if (numServers > total[i]) { + variance += (numServers - total[i]) * mean * mean + (1 - mean) * (1 - mean) * total[i]; + } else { + // Some will have 1 more than everything else. + int numHigh = (int) (total[i] - (Math.floor(mean) * numServers)); + int numLow = (int) (numServers - numHigh); + variance += numHigh * (Math.ceil(mean) - mean) * (Math.ceil(mean) - mean) + + (numLow * (mean - Math.floor(mean)) * (mean - Math.floor(mean))); + } + } + + return Math.sqrt(variance / numServers); + } + + /** + * Scale the value between 0 and 1. + * + * @param min Min value + * @param max The Max value + * @param value The value to be scaled. + * @return The scaled value. + */ + public double scale(double min, double max, double value) { + if (max <= min || value <= min) { + return 0; + } + if ((max - min) == 0) return 0; + + return Math.max(0d, Math.min(1d, (value - min) / (max - min))); + } + /** * Returns true iff a given server has less regions than the balanced amount */ @@ -837,19 +953,8 @@ void regionMoved(int region, int oldServer, int newServer) { } numRegionsPerServerPerTable[newServer][tableIndex]++; - //check whether this caused maxRegionsPerTable in the new Server to be updated - if (numRegionsPerServerPerTable[newServer][tableIndex] > numMaxRegionsPerTable[tableIndex]) { - numMaxRegionsPerTable[tableIndex] = numRegionsPerServerPerTable[newServer][tableIndex]; - } else if (oldServer >= 0 && (numRegionsPerServerPerTable[oldServer][tableIndex] + 1) - == numMaxRegionsPerTable[tableIndex]) { - //recompute maxRegionsPerTable since the previous value was coming from the old server - numMaxRegionsPerTable[tableIndex] = 0; - for (int[] aNumRegionsPerServerPerTable : numRegionsPerServerPerTable) { - if (aNumRegionsPerServerPerTable[tableIndex] > numMaxRegionsPerTable[tableIndex]) { - numMaxRegionsPerTable[tableIndex] = aNumRegionsPerServerPerTable[tableIndex]; - } - } - } + // recalculate stdev + recomputeRegionStDevByTable(); // update for servers int primary = regionIndexToPrimaryIndex[region]; @@ -1019,7 +1124,7 @@ public String toString() { .append(Arrays.toString(serverIndicesSortedByRegionCount)) .append(", regionsPerServer=").append(Arrays.deepToString(regionsPerServer)); - desc.append(", numMaxRegionsPerTable=").append(Arrays.toString(numMaxRegionsPerTable)) + desc.append(", regionStDevByTable=").append(regionStDevByTable) .append(", numRegions=").append(numRegions).append(", numServers=").append(numServers) .append(", numTables=").append(numTables).append(", numMovedRegions=") .append(numMovedRegions).append('}'); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 3f249b890133..ffc77e7db586 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -817,31 +817,14 @@ protected double costFromArray(double[] stats) { double count = stats.length; double mean = total/count; - // Compute max as if all region servers had 0 and one had the sum of all costs. This must be - // a zero sum cost for this to make sense. - double max = ((count - 1) * mean) + (total - mean); - - // It's possible that there aren't enough regions to go around - double min; - if (count > total) { - min = ((count - total) * mean) + ((1 - mean) * total); - } else { - // Some will have 1 more than everything else. - int numHigh = (int) (total - (Math.floor(mean) * count)); - int numLow = (int) (count - numHigh); - - min = (numHigh * (Math.ceil(mean) - mean)) + (numLow * (mean - Math.floor(mean))); - - } - min = Math.max(0, min); for (int i=0; i= 0); + assertTrue(cost >= -0.01); assertTrue(cost <= 1.01); } } @@ -470,7 +470,7 @@ public void testCostFromArray() { statThree[i] = (0); statThree[i+100] = 100; } - assertEquals(0.5, costFunction.costFromArray(statThree), 0.01); + assertEquals(0.0708881205008336, costFunction.costFromArray(statThree), 0.01); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java index da38187cce67..9b43360b7ccc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerLargeCluster.java @@ -38,6 +38,8 @@ public void testLargeCluster() { int numRegionsPerServer = 80; // all servers except one int numTables = 100; int replication = 1; + + // we need to capture the outlier and generate a move testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true); } }