From 907b14dd3766202a5e34741cd1a275ee480efd5d Mon Sep 17 00:00:00 2001 From: Zheng Wang <18031031@qq.com> Date: Sun, 19 Jul 2020 15:04:10 +0800 Subject: [PATCH 1/2] HBASE-24709 Support MoveCostFunction use a lower multiplier in offpeak hours Signed-off-by: Viraj Jasani Signed-off-by: Mingliang Liu --- .../balancer/StochasticLoadBalancer.java | 19 ++++++++++++----- .../balancer/TestStochasticLoadBalancer.java | 21 +++++++++++++++++++ .../asciidoc/_chapters/configuration.adoc | 1 + 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 56b7ae461683..786becd53039 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.LocalityType; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.SwapRegionsAction; +import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -827,26 +828,34 @@ protected double scale(double min, double max, double value) { */ static class MoveCostFunction extends CostFunction { private static final String MOVE_COST_KEY = "hbase.master.balancer.stochastic.moveCost"; + private static final String MOVE_COST_OFFPEAK_KEY = + "hbase.master.balancer.stochastic.moveCost.offpeak"; private static final String MAX_MOVES_PERCENT_KEY = "hbase.master.balancer.stochastic.maxMovePercent"; - private static final float DEFAULT_MOVE_COST = 7; + static final float DEFAULT_MOVE_COST = 7; + static final float DEFAULT_MOVE_COST_OFFPEAK = 3; private static final int DEFAULT_MAX_MOVES = 600; private static final float DEFAULT_MAX_MOVE_PERCENT = 0.25f; private final float maxMovesPercent; + private final Configuration conf; MoveCostFunction(Configuration conf) { super(conf); - - // Move cost multiplier should be the same cost or higher than the rest of the costs to ensure - // that large benefits are need to overcome the cost of a move. - this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST)); + this.conf = conf; // What percent of the number of regions a single run of the balancer can move. maxMovesPercent = conf.getFloat(MAX_MOVES_PERCENT_KEY, DEFAULT_MAX_MOVE_PERCENT); } @Override protected double cost() { + // Move cost multiplier should be the same cost or higher than the rest of the costs to ensure + // that large benefits are need to overcome the cost of a move. + if (OffPeakHours.getInstance(conf).isOffPeakHour()) { + this.setMultiplier(conf.getFloat(MOVE_COST_OFFPEAK_KEY, DEFAULT_MOVE_COST_OFFPEAK)); + } else { + this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST)); + } // Try and size the max number of Moves, but always be prepared to move some. int maxMoves = Math.max((int) (cluster.numRegions * maxMovesPercent), DEFAULT_MAX_MOVES); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index a01165d40aab..4cb92a2b3187 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -284,6 +284,27 @@ public void testLocalityCost() throws Exception { } } + @Test + public void testMoveCostMultiplier() throws Exception { + Configuration conf = HBaseConfiguration.create(); + StochasticLoadBalancer.CostFunction + costFunction = new StochasticLoadBalancer.MoveCostFunction(conf); + BaseLoadBalancer.Cluster cluster = mockCluster(clusterStateMocks[0]); + costFunction.init(cluster); + costFunction.cost(); + assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST, + costFunction.getMultiplier(), 0.01); + + //In offpeak hours, the multiplier of move cost should be lower + conf.setInt("hbase.offpeak.start.hour",0); + conf.setInt("hbase.offpeak.end.hour",23); + costFunction = new StochasticLoadBalancer.MoveCostFunction(conf); + costFunction.init(cluster); + costFunction.cost(); + assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST_OFFPEAK + , costFunction.getMultiplier(), 0.01); + } + @Test public void testMoveCost() throws Exception { Configuration conf = HBaseConfiguration.create(); diff --git a/src/main/asciidoc/_chapters/configuration.adoc b/src/main/asciidoc/_chapters/configuration.adoc index 0851c7179234..b997101d60d9 100644 --- a/src/main/asciidoc/_chapters/configuration.adoc +++ b/src/main/asciidoc/_chapters/configuration.adoc @@ -1384,6 +1384,7 @@ Here are those configurations: | hbase.master.balancer.stochastic.regionCountCost | hbase.master.balancer.stochastic.primaryRegionCountCost | hbase.master.balancer.stochastic.moveCost +| hbase.master.balancer.stochastic.moveCost.offpeak | hbase.master.balancer.stochastic.maxMovePercent | hbase.master.balancer.stochastic.tableSkewCost | hbase.master.regions.recovery.check.interval From 9945df4fc6534387096c961ec5102945d002df81 Mon Sep 17 00:00:00 2001 From: Zheng Wang <18031031@qq.com> Date: Sun, 19 Jul 2020 18:31:24 +0800 Subject: [PATCH 2/2] try again whitout doc change --- src/main/asciidoc/_chapters/configuration.adoc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/asciidoc/_chapters/configuration.adoc b/src/main/asciidoc/_chapters/configuration.adoc index b997101d60d9..0851c7179234 100644 --- a/src/main/asciidoc/_chapters/configuration.adoc +++ b/src/main/asciidoc/_chapters/configuration.adoc @@ -1384,7 +1384,6 @@ Here are those configurations: | hbase.master.balancer.stochastic.regionCountCost | hbase.master.balancer.stochastic.primaryRegionCountCost | hbase.master.balancer.stochastic.moveCost -| hbase.master.balancer.stochastic.moveCost.offpeak | hbase.master.balancer.stochastic.maxMovePercent | hbase.master.balancer.stochastic.tableSkewCost | hbase.master.regions.recovery.check.interval