From 33337b21b15be289b357ada6aa254dbac2340502 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Wed, 10 Nov 2021 17:14:26 +0800 Subject: [PATCH 01/12] HDDS-5517. Support multiple container moves from a source datanode --- .../container/balancer/ContainerBalancer.java | 131 ++++++++---------- .../ContainerBalancerConfiguration.java | 21 +-- .../SourceDataNodeSelectionCriteria.java | 67 +++++++++ 3 files changed, 129 insertions(+), 90 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 646ff8a5a025..2377584595dd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -78,7 +78,6 @@ public class ContainerBalancer { private List unBalancedNodes; private List overUtilizedNodes; private List underUtilizedNodes; - private List withinThresholdUtilizedNodes; private ContainerBalancerConfiguration config; private ContainerBalancerMetrics metrics; private long clusterCapacity; @@ -86,10 +85,12 @@ public class ContainerBalancer { private long clusterRemaining; private double clusterAvgUtilisation; private double upperLimit; + private double lowerLimit; private volatile boolean balancerRunning; private volatile Thread currentBalancingThread; private Lock lock; private ContainerBalancerSelectionCriteria selectionCriteria; + private SourceDataNodeSelectionCriteria srcDnSelectionCriteria; private Map sourceToTargetMap; private Map sizeLeavingNode; private Map sizeEnteringNode; @@ -127,7 +128,6 @@ public ContainerBalancer( this.selectedContainers = new HashSet<>(); this.overUtilizedNodes = new ArrayList<>(); this.underUtilizedNodes = new ArrayList<>(); - this.withinThresholdUtilizedNodes = new ArrayList<>(); this.unBalancedNodes = new ArrayList<>(); this.lock = new ReentrantLock(); @@ -251,7 +251,6 @@ private boolean initializeIteration() { this.selectedContainers.clear(); this.overUtilizedNodes.clear(); this.underUtilizedNodes.clear(); - this.withinThresholdUtilizedNodes.clear(); this.unBalancedNodes.clear(); this.countDatanodesInvolvedPerIteration = 0; this.sizeMovedPerIteration = 0; @@ -262,20 +261,17 @@ private boolean initializeIteration() { clusterAvgUtilisation); } - // under utilized nodes have utilization(that is, used / capacity) less - // than lower limit - double lowerLimit = clusterAvgUtilisation - threshold; - // over utilized nodes have utilization(that is, used / capacity) greater // than upper limit this.upperLimit = clusterAvgUtilisation + threshold; + this.lowerLimit = clusterAvgUtilisation - threshold; if (LOG.isDebugEnabled()) { LOG.debug("Lower limit for utilization is {} and Upper limit for " + "utilization is {}", lowerLimit, upperLimit); } - long overUtilizedBytes = 0L, underUtilizedBytes = 0L; + long totalOverUtilizedBytes = 0L, totalUnderUtilizedBytes = 0L; // find over and under utilized nodes for (DatanodeUsageInfo datanodeUsageInfo : datanodeUsageInfos) { if (!isBalancerRunning()) { @@ -291,7 +287,7 @@ private boolean initializeIteration() { datanodeUsageInfo.getScmNodeStat().getRemaining().get(), utilization); } - if (utilization > upperLimit) { + if (Double.compare(utilization, upperLimit) > 0) { overUtilizedNodes.add(datanodeUsageInfo); metrics.incrementDatanodesNumToBalance(1); @@ -300,27 +296,28 @@ private boolean initializeIteration() { ratioToPercent(utilization))); // amount of bytes greater than upper limit in this node - overUtilizedBytes += ratioToBytes( + Long overUtilizedBytes = ratioToBytes( datanodeUsageInfo.getScmNodeStat().getCapacity().get(), utilization) - ratioToBytes( datanodeUsageInfo.getScmNodeStat().getCapacity().get(), upperLimit); - } else if (utilization < lowerLimit) { + totalOverUtilizedBytes += overUtilizedBytes; + } else if (Double.compare(utilization, lowerLimit) < 0) { underUtilizedNodes.add(datanodeUsageInfo); metrics.incrementDatanodesNumToBalance(1); // amount of bytes lesser than lower limit in this node - underUtilizedBytes += ratioToBytes( + Long underUtilizedBytes = ratioToBytes( datanodeUsageInfo.getScmNodeStat().getCapacity().get(), lowerLimit) - ratioToBytes( datanodeUsageInfo.getScmNodeStat().getCapacity().get(), utilization); - } else { - withinThresholdUtilizedNodes.add(datanodeUsageInfo); + totalUnderUtilizedBytes += underUtilizedBytes; } } metrics.setDataSizeToBalanceGB( - Math.max(overUtilizedBytes, underUtilizedBytes) / OzoneConsts.GB); + Math.max(totalOverUtilizedBytes, totalUnderUtilizedBytes) / + OzoneConsts.GB); Collections.reverse(underUtilizedNodes); unBalancedNodes = new ArrayList<>( @@ -339,24 +336,19 @@ private boolean initializeIteration() { selectionCriteria = new ContainerBalancerSelectionCriteria(config, nodeManager, replicationManager, containerManager); - sourceToTargetMap = new HashMap<>(overUtilizedNodes.size() + - withinThresholdUtilizedNodes.size()); + sourceToTargetMap = new HashMap<>(overUtilizedNodes.size()); // initialize maps to track how much size is leaving and entering datanodes - sizeLeavingNode = new HashMap<>(overUtilizedNodes.size() + - withinThresholdUtilizedNodes.size()); + sizeLeavingNode = new HashMap<>(overUtilizedNodes.size()); overUtilizedNodes.forEach(datanodeUsageInfo -> sizeLeavingNode .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); - withinThresholdUtilizedNodes.forEach(datanodeUsageInfo -> sizeLeavingNode - .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); - sizeEnteringNode = new HashMap<>(underUtilizedNodes.size() + - withinThresholdUtilizedNodes.size()); + sizeEnteringNode = new HashMap<>(underUtilizedNodes.size()); underUtilizedNodes.forEach(datanodeUsageInfo -> sizeEnteringNode .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); - withinThresholdUtilizedNodes.forEach(datanodeUsageInfo -> sizeEnteringNode - .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); + srcDnSelectionCriteria = new + SourceDataNodeSelectionCriteria(overUtilizedNodes, sizeLeavingNode); return true; } @@ -371,11 +363,16 @@ private IterationResult doIteration() { try { // match each overUtilized node with a target - for (DatanodeUsageInfo datanodeUsageInfo : overUtilizedNodes) { + while (true) { + DatanodeDetails source = + srcDnSelectionCriteria.getNextCandidateSourceDataNode(); + if (source == null) { + break; + } if (!isBalancerRunning()) { return IterationResult.ITERATION_INTERRUPTED; } - DatanodeDetails source = datanodeUsageInfo.getDatanodeDetails(); + IterationResult result = checkConditionsForBalancing(); if (result != null) { return result; @@ -387,7 +384,8 @@ private IterationResult doIteration() { isMoveGenerated = true; LOG.info("ContainerBalancer is trying to move container {} from " + "source datanode {} to target datanode {}", - moveSelection.getContainerID().toString(), source.getUuidString(), + moveSelection.getContainerID().toString(), + source.getUuidString(), moveSelection.getTargetNode().getUuidString()); if (moveContainer(source, moveSelection)) { @@ -395,45 +393,14 @@ private IterationResult doIteration() { potentialTargets = updateTargetsAndSelectionCriteria( potentialTargets, selectedTargets, moveSelection, source); } - } - } - - // if not all underUtilized nodes have been selected, try to match - // withinThresholdUtilized nodes with underUtilized nodes - if (selectedTargets.size() < underUtilizedNodes.size()) { - potentialTargets.removeAll(selectedTargets); - Collections.reverse(withinThresholdUtilizedNodes); - - for (DatanodeUsageInfo datanodeUsageInfo : - withinThresholdUtilizedNodes) { - if (!balancerRunning) { - return IterationResult.ITERATION_INTERRUPTED; - } - DatanodeDetails source = datanodeUsageInfo.getDatanodeDetails(); - IterationResult result = checkConditionsForBalancing(); - if (result != null) { - return result; - } - - ContainerMoveSelection moveSelection = - matchSourceWithTarget(source, potentialTargets); - if (moveSelection != null) { - isMoveGenerated = true; - LOG.info("ContainerBalancer is trying to move container {} from " + - "source datanode {} to target datanode {}", - moveSelection.getContainerID().toString(), - source.getUuidString(), - moveSelection.getTargetNode().getUuidString()); - if (moveContainer(source, moveSelection)) { - // consider move successful for now, and update selection criteria - potentialTargets = - updateTargetsAndSelectionCriteria(potentialTargets, - selectedTargets, moveSelection, source); - } - } + } else { + // can not find any target for this source + srcDnSelectionCriteria.removeCandidateSourceDataNode(source); } + } + if (!isMoveGenerated) { //no move option is generated, so the cluster can not be //balanced any more, just stop iteration and exit @@ -518,6 +485,27 @@ private ContainerMoveSelection matchSourceWithTarget( } return null; } + + //if the utilization of the source data node becomes lower than lowerLimit + //after the container is moved out , then the container can not be + // a candidate one, and we should remove it from the candidateContainers. + candidateContainers.removeIf(c -> { + ContainerInfo cInfo; + try { + cInfo = containerManager.getContainer(c); + } catch (ContainerNotFoundException e) { + LOG.warn("Could not find container {} when " + + "be matched with a move target", c); + //remove this not found container + return true; + } + Long totalLeavingSize = sizeLeavingNode.get(source) + + cInfo.getUsedBytes(); + return Double.compare(nodeManager.getUsageInfo(source) + .calculateUtilization(totalLeavingSize), lowerLimit) < 0 || + totalLeavingSize > config.getMaxSizeLeavingSource(); + }); + if (LOG.isDebugEnabled()) { LOG.debug("ContainerBalancer is finding suitable target for source " + "datanode {}", source.getUuidString()); @@ -708,8 +696,8 @@ boolean canSizeEnterTarget(DatanodeDetails target, long size) { //2 current usage of target datanode plus sizeEnteringAfterMove // is smaller than or equal to upperLimit return sizeEnteringAfterMove <= config.getMaxSizeEnteringTarget() && - nodeManager.getUsageInfo(target) - .calculateUtilization(sizeEnteringAfterMove) <= upperLimit; + Double.compare(nodeManager.getUsageInfo(target) + .calculateUtilization(sizeEnteringAfterMove), upperLimit) < 1; } return false; } @@ -721,13 +709,10 @@ boolean canSizeEnterTarget(DatanodeDetails target, long size) { * @return A list of potential target DatanodeDetails. */ private List getPotentialTargets() { - List potentialTargets = new ArrayList<>( - underUtilizedNodes.size() + withinThresholdUtilizedNodes.size()); - - underUtilizedNodes - .forEach(node -> potentialTargets.add(node.getDatanodeDetails())); - withinThresholdUtilizedNodes - .forEach(node -> potentialTargets.add(node.getDatanodeDetails())); + List potentialTargets = + new ArrayList<>(underUtilizedNodes.size()); + underUtilizedNodes.forEach( + node -> potentialTargets.add(node.getDatanodeDetails())); return potentialTargets; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index 4e7b88999824..b3697c44c67c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -24,9 +24,7 @@ import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.ConfigType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.fs.DUFactory; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.ozone.OzoneConsts; import org.slf4j.Logger; @@ -75,17 +73,15 @@ public final class ContainerBalancerConfiguration { defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can enter a target datanode in each " + "iteration while balancing. This is the sum of data from multiple " + - "sources. The default value is greater than the configured" + - " (or default) ozone.scm.container.size by 1GB.") - private long maxSizeEnteringTarget; + "sources. by default, we do not limit this value") + private long maxSizeEnteringTarget = Long.MAX_VALUE; @Config(key = "size.leaving.source.max", type = ConfigType.SIZE, defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can leave a source datanode in each " + "iteration while balancing. This is the sum of data moving to multiple " + - "targets. The default value is greater than the configured" + - " (or default) ozone.scm.container.size by 1GB.") - private long maxSizeLeavingSource; + "targets. by default, we do not limit this value") + private long maxSizeLeavingSource = Long.MAX_VALUE;; @Config(key = "idle.iterations", type = ConfigType.INT, defaultValue = "10", tags = {ConfigTag.BALANCER}, @@ -121,15 +117,6 @@ public ContainerBalancerConfiguration(OzoneConfiguration config) { "OzoneConfiguration should not be null."); this.ozoneConfiguration = config; - // maxSizeEnteringTarget and maxSizeLeavingSource should by default be - // greater than container size - long size = (long) ozoneConfiguration.getStorageSize( - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.GB) + - OzoneConsts.GB; - maxSizeEnteringTarget = size; - maxSizeLeavingSource = size; - // balancing interval should be greater than DUFactory refresh period duConf = ozoneConfiguration.getObject(DUFactory.Conf.class); balancingInterval = duConf.getRefreshPeriod().toMillis() + diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java new file mode 100644 index 000000000000..8ebfefdb6d10 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + +/** + * The selection criteria for selecting source datanodes , the containers of + * which will be moved out. + */ +public class SourceDataNodeSelectionCriteria { + private static final Logger LOG = + LoggerFactory.getLogger(SourceDataNodeSelectionCriteria.class); + private Map sizeLeavingNode; + private List overUtilizedNodes; + + public SourceDataNodeSelectionCriteria( + List overUtilizedNodes, + Map sizeLeavingNode) { + this.sizeLeavingNode = sizeLeavingNode; + this.overUtilizedNodes = overUtilizedNodes; + } + + public DatanodeDetails getNextCandidateSourceDataNode() { + if (overUtilizedNodes.isEmpty()) { + LOG.info("no more candidate data node"); + return null; + } + //TODO：use a more quick data structure, which will hava a + // better performance when changing or deleting one element at once + overUtilizedNodes.sort((a, b) -> { + double currentUsageOfA = a.calculateUtilization( + sizeLeavingNode.get(a.getDatanodeDetails())); + double currentUsageOfB = b.calculateUtilization( + sizeLeavingNode.get(b.getDatanodeDetails())); + //in descending order + return Double.compare(currentUsageOfB, currentUsageOfA); + }); + + return overUtilizedNodes.get(0).getDatanodeDetails(); + } + + public void removeCandidateSourceDataNode(DatanodeDetails dui){ + overUtilizedNodes.removeIf(a -> a.getDatanodeDetails().equals(dui)); + } +} From ff275278a9045923fd5f2221bb94af037357b0f9 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Fri, 12 Nov 2021 17:02:05 +0800 Subject: [PATCH 02/12] fix comments --- .../container/balancer/ContainerBalancer.java | 35 +++++++++++++++---- .../SourceDataNodeSelectionCriteria.java | 6 ++-- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 2377584595dd..e7a4ee28e44d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -264,6 +264,8 @@ private boolean initializeIteration() { // over utilized nodes have utilization(that is, used / capacity) greater // than upper limit this.upperLimit = clusterAvgUtilisation + threshold; + // under utilized nodes have utilization(that is, used / capacity) less + // than lower limit this.lowerLimit = clusterAvgUtilisation - threshold; if (LOG.isDebugEnabled()) { @@ -499,11 +501,7 @@ private ContainerMoveSelection matchSourceWithTarget( //remove this not found container return true; } - Long totalLeavingSize = sizeLeavingNode.get(source) + - cInfo.getUsedBytes(); - return Double.compare(nodeManager.getUsageInfo(source) - .calculateUtilization(totalLeavingSize), lowerLimit) < 0 || - totalLeavingSize > config.getMaxSizeLeavingSource(); + return !canSizeLeaveSource(source, cInfo.getUsedBytes()); }); if (LOG.isDebugEnabled()) { @@ -697,7 +695,32 @@ boolean canSizeEnterTarget(DatanodeDetails target, long size) { // is smaller than or equal to upperLimit return sizeEnteringAfterMove <= config.getMaxSizeEnteringTarget() && Double.compare(nodeManager.getUsageInfo(target) - .calculateUtilization(sizeEnteringAfterMove), upperLimit) < 1; + .calculateUtilization(sizeEnteringAfterMove), upperLimit) <= 0; + } + return false; + } + + /** + * Checks if specified size can leave a specified target datanode + * according to {@link ContainerBalancerConfiguration} + * "size.entering.target.max". + * + * @param source target datanode in which size is entering + * @param size size in bytes + * @return true if size can leave, else false + */ + boolean canSizeLeaveSource(DatanodeDetails source, long size) { + if (sizeLeavingNode.containsKey(source)) { + long sizeLeavingAfterMove = sizeLeavingNode.get(source) + size; + //size can be moved out of source datanode only when the following + //two condition are met. + //1 sizeLeavingAfterMove does not succeed the configured + // MaxSizeLeavingTarget + //2 after subtracting sizeLeavingAfterMove, the usage is bigger + // than or equal to lowerLimit + return sizeLeavingAfterMove <= config.getMaxSizeLeavingSource() && + Double.compare(nodeManager.getUsageInfo(source) + .calculateUtilization(-sizeLeavingAfterMove), lowerLimit) >= 0; } return false; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java index 8ebfefdb6d10..ee62f0d9592d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java @@ -44,16 +44,16 @@ public SourceDataNodeSelectionCriteria( public DatanodeDetails getNextCandidateSourceDataNode() { if (overUtilizedNodes.isEmpty()) { - LOG.info("no more candidate data node"); + LOG.info("no more candidate source data node"); return null; } //TODO：use a more quick data structure, which will hava a // better performance when changing or deleting one element at once overUtilizedNodes.sort((a, b) -> { double currentUsageOfA = a.calculateUtilization( - sizeLeavingNode.get(a.getDatanodeDetails())); + -sizeLeavingNode.get(a.getDatanodeDetails())); double currentUsageOfB = b.calculateUtilization( - sizeLeavingNode.get(b.getDatanodeDetails())); + -sizeLeavingNode.get(b.getDatanodeDetails())); //in descending order return Double.compare(currentUsageOfB, currentUsageOfA); }); From 77e1ba7bcce6f0234ce5b06f094513bb62854be3 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Mon, 15 Nov 2021 11:13:36 +0800 Subject: [PATCH 03/12] sort potentialTargets before selecting a target data node --- .../container/balancer/ContainerBalancer.java | 27 +++++++++---------- .../container/balancer/FindTargetGreedy.java | 22 +++++++++++---- .../balancer/FindTargetStrategy.java | 5 ++-- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index e7a4ee28e44d..89c0475ac749 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -129,10 +129,11 @@ public ContainerBalancer( this.overUtilizedNodes = new ArrayList<>(); this.underUtilizedNodes = new ArrayList<>(); this.unBalancedNodes = new ArrayList<>(); + this.sizeEnteringNode = new HashMap<>(); this.lock = new ReentrantLock(); - findTargetStrategy = - new FindTargetGreedy(containerManager, placementPolicy); + findTargetStrategy = new FindTargetGreedy( + containerManager, placementPolicy, sizeEnteringNode); } /** @@ -345,7 +346,7 @@ private boolean initializeIteration() { overUtilizedNodes.forEach(datanodeUsageInfo -> sizeLeavingNode .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); - sizeEnteringNode = new HashMap<>(underUtilizedNodes.size()); + sizeEnteringNode.clear(); underUtilizedNodes.forEach(datanodeUsageInfo -> sizeEnteringNode .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); @@ -357,7 +358,7 @@ private boolean initializeIteration() { private IterationResult doIteration() { // note that potential and selected targets are updated in the following // loop - List potentialTargets = getPotentialTargets(); + List potentialTargets = getPotentialTargets(); Set selectedTargets = new HashSet<>(potentialTargets.size()); moveSelectionToFutureMap = new HashMap<>(unBalancedNodes.size()); @@ -476,7 +477,7 @@ private void checkIterationMoveResults(Set selectedTargets) { * @return ContainerMoveSelection containing the selected target and container */ private ContainerMoveSelection matchSourceWithTarget( - DatanodeDetails source, Collection potentialTargets) { + DatanodeDetails source, List potentialTargets) { NavigableSet candidateContainers = selectionCriteria.getCandidateContainers(source); @@ -615,8 +616,8 @@ private boolean moveContainer(DatanodeDetails source, * @param source the source datanode * @return List of updated potential targets */ - private List updateTargetsAndSelectionCriteria( - Collection potentialTargets, + private List updateTargetsAndSelectionCriteria( + Collection potentialTargets, Set selectedTargets, ContainerMoveSelection moveSelection, DatanodeDetails source) { // count source if it has not been involved in move earlier @@ -635,7 +636,7 @@ private List updateTargetsAndSelectionCriteria( selectionCriteria.setSelectedContainers(selectedContainers); return potentialTargets.stream() - .filter(node -> sizeEnteringNode.get(node) < + .filter(node -> sizeEnteringNode.get(node.getDatanodeDetails()) < config.getMaxSizeEnteringTarget()).collect(Collectors.toList()); } @@ -729,14 +730,10 @@ boolean canSizeLeaveSource(DatanodeDetails source, long size) { * Get potential targets for container move. Potential targets are under * utilized and within threshold utilized nodes. * - * @return A list of potential target DatanodeDetails. + * @return A list of potential target DatanodeUsageInfo. */ - private List getPotentialTargets() { - List potentialTargets = - new ArrayList<>(underUtilizedNodes.size()); - underUtilizedNodes.forEach( - node -> potentialTargets.add(node.getDatanodeDetails())); - return potentialTargets; + private List getPotentialTargets() { + return underUtilizedNodes; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java index 9d3deb435e9e..a2da1a61c028 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java @@ -26,11 +26,12 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -44,12 +45,15 @@ public class FindTargetGreedy implements FindTargetStrategy { private ContainerManager containerManager; private PlacementPolicy placementPolicy; + private Map sizeEnteringNode; public FindTargetGreedy( ContainerManager containerManager, - PlacementPolicy placementPolicy) { + PlacementPolicy placementPolicy, + Map sizeEnteringNode) { this.containerManager = containerManager; this.placementPolicy = placementPolicy; + this.sizeEnteringNode = sizeEnteringNode; } /** @@ -67,14 +71,22 @@ public FindTargetGreedy( */ @Override public ContainerMoveSelection findTargetForContainerMove( - DatanodeDetails source, Collection potentialTargets, + DatanodeDetails source, List potentialTargets, Set candidateContainers, BiFunction canSizeEnterTarget) { - for (DatanodeDetails target : potentialTargets) { + potentialTargets.sort((a, b) -> { + double currentUsageOfA = a.calculateUtilization( + sizeEnteringNode.get(a.getDatanodeDetails())); + double currentUsageOfB = b.calculateUtilization( + sizeEnteringNode.get(b.getDatanodeDetails())); + return Double.compare(currentUsageOfA, currentUsageOfB); + }); + + for (DatanodeUsageInfo targetInfo : potentialTargets) { + DatanodeDetails target = targetInfo.getDatanodeDetails(); for (ContainerID container : candidateContainers) { Set replicas; ContainerInfo containerInfo; - try { replicas = containerManager.getContainerReplicas(container); containerInfo = containerManager.getContainer(container); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java index 444f365cf9ee..1ef652e22539 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java @@ -21,8 +21,9 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; -import java.util.Collection; +import java.util.List; import java.util.Set; import java.util.function.BiFunction; @@ -50,7 +51,7 @@ public interface FindTargetStrategy { * selected container */ ContainerMoveSelection findTargetForContainerMove( - DatanodeDetails source, Collection potentialTargets, + DatanodeDetails source, List potentialTargets, Set candidateContainers, BiFunction canSizeEnterTarget); From d7f4b656c82d406f4d71232dd3731ead65d31ea9 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Wed, 17 Nov 2021 11:22:18 +0800 Subject: [PATCH 04/12] refactor --- .../container/balancer/ContainerBalancer.java | 134 +++++----------- .../container/balancer/FindSourceGreedy.java | 150 ++++++++++++++++++ .../balancer/FindSourceStrategy.java | 67 ++++++++ .../container/balancer/FindTargetGreedy.java | 93 +++++++++-- .../balancer/FindTargetStrategy.java | 29 ++-- .../SourceDataNodeSelectionCriteria.java | 67 -------- 6 files changed, 350 insertions(+), 190 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceStrategy.java delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 89c0475ac749..bd96fa69d50d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -38,7 +38,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -52,7 +51,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; /** * Container balancer is a service in SCM to move containers between over- and @@ -78,6 +76,7 @@ public class ContainerBalancer { private List unBalancedNodes; private List overUtilizedNodes; private List underUtilizedNodes; + private List withinThresholdUtilizedNodes; private ContainerBalancerConfiguration config; private ContainerBalancerMetrics metrics; private long clusterCapacity; @@ -90,12 +89,10 @@ public class ContainerBalancer { private volatile Thread currentBalancingThread; private Lock lock; private ContainerBalancerSelectionCriteria selectionCriteria; - private SourceDataNodeSelectionCriteria srcDnSelectionCriteria; private Map sourceToTargetMap; - private Map sizeLeavingNode; - private Map sizeEnteringNode; private Set selectedContainers; private FindTargetStrategy findTargetStrategy; + private FindSourceStrategy findSourceStrategy; private Map> moveSelectionToFutureMap; @@ -128,12 +125,13 @@ public ContainerBalancer( this.selectedContainers = new HashSet<>(); this.overUtilizedNodes = new ArrayList<>(); this.underUtilizedNodes = new ArrayList<>(); + this.withinThresholdUtilizedNodes = new ArrayList<>(); this.unBalancedNodes = new ArrayList<>(); - this.sizeEnteringNode = new HashMap<>(); this.lock = new ReentrantLock(); findTargetStrategy = new FindTargetGreedy( - containerManager, placementPolicy, sizeEnteringNode); + containerManager, placementPolicy, nodeManager); + findSourceStrategy = new FindSourceGreedy(nodeManager); } /** @@ -316,6 +314,8 @@ private boolean initializeIteration() { datanodeUsageInfo.getScmNodeStat().getCapacity().get(), utilization); totalUnderUtilizedBytes += underUtilizedBytes; + } else { + withinThresholdUtilizedNodes.add(datanodeUsageInfo); } } metrics.setDataSizeToBalanceGB( @@ -339,36 +339,29 @@ private boolean initializeIteration() { selectionCriteria = new ContainerBalancerSelectionCriteria(config, nodeManager, replicationManager, containerManager); - sourceToTargetMap = new HashMap<>(overUtilizedNodes.size()); - - // initialize maps to track how much size is leaving and entering datanodes - sizeLeavingNode = new HashMap<>(overUtilizedNodes.size()); - overUtilizedNodes.forEach(datanodeUsageInfo -> sizeLeavingNode - .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); - - sizeEnteringNode.clear(); - underUtilizedNodes.forEach(datanodeUsageInfo -> sizeEnteringNode - .put(datanodeUsageInfo.getDatanodeDetails(), 0L)); - - srcDnSelectionCriteria = new - SourceDataNodeSelectionCriteria(overUtilizedNodes, sizeLeavingNode); + sourceToTargetMap = new HashMap<>(overUtilizedNodes.size() + + withinThresholdUtilizedNodes.size()); return true; } private IterationResult doIteration() { // note that potential and selected targets are updated in the following // loop + //TODO(jacksonyao): take withinThresholdUtilizedNodes as candidate for both + // source and target + findSourceStrategy.reInitialize(getPotentialSources(), config, lowerLimit); List potentialTargets = getPotentialTargets(); + findTargetStrategy.reInitialize(potentialTargets, config, upperLimit); + Set selectedTargets = new HashSet<>(potentialTargets.size()); moveSelectionToFutureMap = new HashMap<>(unBalancedNodes.size()); boolean isMoveGenerated = false; - try { // match each overUtilized node with a target while (true) { DatanodeDetails source = - srcDnSelectionCriteria.getNextCandidateSourceDataNode(); + findSourceStrategy.getNextCandidateSourceDataNode(); if (source == null) { break; } @@ -381,8 +374,7 @@ private IterationResult doIteration() { return result; } - ContainerMoveSelection moveSelection = - matchSourceWithTarget(source, potentialTargets); + ContainerMoveSelection moveSelection = matchSourceWithTarget(source); if (moveSelection != null) { isMoveGenerated = true; LOG.info("ContainerBalancer is trying to move container {} from " + @@ -393,15 +385,13 @@ private IterationResult doIteration() { if (moveContainer(source, moveSelection)) { // consider move successful for now, and update selection criteria - potentialTargets = updateTargetsAndSelectionCriteria( - potentialTargets, selectedTargets, moveSelection, source); + updateTargetsAndSelectionCriteria( + selectedTargets, moveSelection, source); } - } else { // can not find any target for this source - srcDnSelectionCriteria.removeCandidateSourceDataNode(source); + findSourceStrategy.removeCandidateSourceDataNode(source); } - } if (!isMoveGenerated) { @@ -472,12 +462,9 @@ private void checkIterationMoveResults(Set selectedTargets) { * Match a source datanode with a target datanode and identify the container * to move. * - * @param potentialTargets Collection of potential targets to move - * container to * @return ContainerMoveSelection containing the selected target and container */ - private ContainerMoveSelection matchSourceWithTarget( - DatanodeDetails source, List potentialTargets) { + private ContainerMoveSelection matchSourceWithTarget(DatanodeDetails source) { NavigableSet candidateContainers = selectionCriteria.getCandidateContainers(source); @@ -502,7 +489,8 @@ private ContainerMoveSelection matchSourceWithTarget( //remove this not found container return true; } - return !canSizeLeaveSource(source, cInfo.getUsedBytes()); + return !findSourceStrategy.canSizeLeaveSource( + source, cInfo.getUsedBytes()); }); if (LOG.isDebugEnabled()) { @@ -511,8 +499,7 @@ private ContainerMoveSelection matchSourceWithTarget( } ContainerMoveSelection moveSelection = findTargetStrategy.findTargetForContainerMove( - source, potentialTargets, candidateContainers, - this::canSizeEnterTarget); + source, candidateContainers); if (moveSelection == null) { if (LOG.isDebugEnabled()) { @@ -609,15 +596,13 @@ private boolean moveContainer(DatanodeDetails source, /** * Update targets and selection criteria after a move. * - * @param potentialTargets potential target datanodes * @param selectedTargets selected target datanodes * @param moveSelection the target datanode and container that has been * just selected * @param source the source datanode * @return List of updated potential targets */ - private List updateTargetsAndSelectionCriteria( - Collection potentialTargets, + private void updateTargetsAndSelectionCriteria( Set selectedTargets, ContainerMoveSelection moveSelection, DatanodeDetails source) { // count source if it has not been involved in move earlier @@ -634,10 +619,6 @@ private List updateTargetsAndSelectionCriteria( selectedTargets.add(moveSelection.getTargetNode()); selectedContainers.add(moveSelection.getContainerID()); selectionCriteria.setSelectedContainers(selectedContainers); - - return potentialTargets.stream() - .filter(node -> sizeEnteringNode.get(node.getDatanodeDetails()) < - config.getMaxSizeEnteringTarget()).collect(Collectors.toList()); } /** @@ -676,55 +657,8 @@ private long ratioToBytes(Long nodeCapacity, double utilizationRatio) { return (clusterCapacity - clusterRemaining) / (double) clusterCapacity; } - /** - * Checks if specified size can enter specified target datanode - * according to {@link ContainerBalancerConfiguration} - * "size.entering.target.max". - * - * @param target target datanode in which size is entering - * @param size size in bytes - * @return true if size can enter target, else false - */ - boolean canSizeEnterTarget(DatanodeDetails target, long size) { - if (sizeEnteringNode.containsKey(target)) { - long sizeEnteringAfterMove = sizeEnteringNode.get(target) + size; - //size can be moved into target datanode only when the following - //two condition are met. - //1 sizeEnteringAfterMove does not succeed the configured - // MaxSizeEnteringTarget - //2 current usage of target datanode plus sizeEnteringAfterMove - // is smaller than or equal to upperLimit - return sizeEnteringAfterMove <= config.getMaxSizeEnteringTarget() && - Double.compare(nodeManager.getUsageInfo(target) - .calculateUtilization(sizeEnteringAfterMove), upperLimit) <= 0; - } - return false; - } - /** - * Checks if specified size can leave a specified target datanode - * according to {@link ContainerBalancerConfiguration} - * "size.entering.target.max". - * - * @param source target datanode in which size is entering - * @param size size in bytes - * @return true if size can leave, else false - */ - boolean canSizeLeaveSource(DatanodeDetails source, long size) { - if (sizeLeavingNode.containsKey(source)) { - long sizeLeavingAfterMove = sizeLeavingNode.get(source) + size; - //size can be moved out of source datanode only when the following - //two condition are met. - //1 sizeLeavingAfterMove does not succeed the configured - // MaxSizeLeavingTarget - //2 after subtracting sizeLeavingAfterMove, the usage is bigger - // than or equal to lowerLimit - return sizeLeavingAfterMove <= config.getMaxSizeLeavingSource() && - Double.compare(nodeManager.getUsageInfo(source) - .calculateUtilization(-sizeLeavingAfterMove), lowerLimit) >= 0; - } - return false; - } + /** * Get potential targets for container move. Potential targets are under @@ -733,9 +667,23 @@ boolean canSizeLeaveSource(DatanodeDetails source, long size) { * @return A list of potential target DatanodeUsageInfo. */ private List getPotentialTargets() { + //TODO(jacksonyao): take withinThresholdUtilizedNodes as candidate for both + // source and target return underUtilizedNodes; } + /** + * Get potential sourecs for container move. Potential sourecs are over + * utilized and within threshold utilized nodes. + * + * @return A list of potential source DatanodeUsageInfo. + */ + private List getPotentialSources() { + //TODO(jacksonyao): take withinThresholdUtilizedNodes as candidate for both + // source and target + return overUtilizedNodes; + } + /** * Updates conditions for balancing, such as total size moved by balancer, * total size that has entered a datanode, and total size that has left a @@ -761,10 +709,10 @@ private void incSizeSelectedForMoving(DatanodeDetails source, sizeMovedPerIteration += size; // update sizeLeavingNode map with the recent moveSelection - sizeLeavingNode.put(source, sizeLeavingNode.get(source) + size); + findSourceStrategy.increaseSizeLeaving(source, size); // update sizeEnteringNode map with the recent moveSelection - sizeEnteringNode.put(target, sizeEnteringNode.get(target) + size); + findTargetStrategy.increaseSizeEntering(target, size); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java new file mode 100644 index 000000000000..35434f360dbb --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * The selection criteria for selecting source datanodes , the containers of + * which will be moved out. + */ +public class FindSourceGreedy implements FindSourceStrategy{ + private static final Logger LOG = + LoggerFactory.getLogger(FindSourceGreedy.class); + private Map sizeLeavingNode; + private List potentialSources; + private NodeManager nodeManager; + private ContainerBalancerConfiguration config; + private Double lowerLimit; + + FindSourceGreedy(NodeManager nodeManager) { + sizeLeavingNode = new HashMap<>(); + this.nodeManager = nodeManager; + } + + private void setLowerLimit(Double lowerLimit) { + this.lowerLimit = lowerLimit; + } + + private void setPotentialSources( + List potentialSources) { + this.potentialSources = potentialSources; + sizeLeavingNode.clear(); + potentialSources.forEach( + c -> sizeLeavingNode.put(c.getDatanodeDetails(), 0L)); + } + + private void setConfiguration(ContainerBalancerConfiguration conf) { + this.config = conf; + } + + /** + * increase the Leaving size of a candidate source data node. + */ + @Override + public void increaseSizeLeaving(DatanodeDetails dui, long size) { + Long currentSize = sizeLeavingNode.get(dui); + if(currentSize != null) { + sizeLeavingNode.put(dui, currentSize + size); + return; + } + LOG.warn("Cannot find datanode {} in candidate source datanodes", + dui.getUuid()); + } + + /** + * get the next candidate source data node according to + * the strategy. + * + * @return the nex candidate source data node. + */ + @Override + public DatanodeDetails getNextCandidateSourceDataNode() { + if (potentialSources.isEmpty()) { + LOG.info("no more candidate source data node"); + return null; + } + //TODO：use a more quick data structure, which will hava a + // better performance when changing or deleting one element at once + potentialSources.sort((a, b) -> { + double currentUsageOfA = a.calculateUtilization( + -sizeLeavingNode.get(a.getDatanodeDetails())); + double currentUsageOfB = b.calculateUtilization( + -sizeLeavingNode.get(b.getDatanodeDetails())); + //in descending order + return Double.compare(currentUsageOfB, currentUsageOfA); + }); + + return potentialSources.get(0).getDatanodeDetails(); + } + + /** + * remove the specified data node from candidate source + * data nodes. + */ + @Override + public void removeCandidateSourceDataNode(DatanodeDetails dui){ + potentialSources.removeIf(a -> a.getDatanodeDetails().equals(dui)); + } + + /** + * Checks if specified size can leave a specified target datanode + * according to {@link ContainerBalancerConfiguration} + * "size.entering.target.max". + * + * @param source target datanode in which size is entering + * @param size size in bytes + * @return true if size can leave, else false + */ + @Override + public boolean canSizeLeaveSource(DatanodeDetails source, long size) { + if (sizeLeavingNode.containsKey(source)) { + long sizeLeavingAfterMove = sizeLeavingNode.get(source) + size; + //size can be moved out of source datanode only when the following + //two condition are met. + //1 sizeLeavingAfterMove does not succeed the configured + // MaxSizeLeavingTarget + //2 after subtracting sizeLeavingAfterMove, the usage is bigger + // than or equal to lowerLimit + return sizeLeavingAfterMove <= config.getMaxSizeLeavingSource() && + Double.compare(nodeManager.getUsageInfo(source) + .calculateUtilization(-sizeLeavingAfterMove), lowerLimit) >= 0; + } + return false; + } + + /** + * reInitialize FindSourceStrategy. + */ + @Override + public void reInitialize(List potentialDataNodes, + ContainerBalancerConfiguration conf, + Double lowLimit) { + setConfiguration(conf); + setLowerLimit(lowLimit); + setPotentialSources(potentialDataNodes); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceStrategy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceStrategy.java new file mode 100644 index 000000000000..826ecac6d069 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceStrategy.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; + +import java.util.List; + +/** + * This interface can be used to implement strategies to get a + * source datanode. + */ +public interface FindSourceStrategy { + + /** + * get the next candidate source data node according to + * the strategy. + * + * @return the nex candidate source data node. + */ + DatanodeDetails getNextCandidateSourceDataNode(); + + /** + * remove the specified data node from candidate source + * data nodes. + */ + void removeCandidateSourceDataNode(DatanodeDetails dui); + + /** + * increase the Leaving size of a candidate source data node. + */ + void increaseSizeLeaving(DatanodeDetails dui, long size); + + /** + * Checks if specified size can leave a specified source datanode + * according to {@link ContainerBalancerConfiguration} + * "size.entering.target.max". + * + * @param source target datanode in which size is entering + * @param size size in bytes + * @return true if size can leave, else false + */ + boolean canSizeLeaveSource(DatanodeDetails source, long size); + + /** + * reInitialize FindSourceStrategy. + */ + void reInitialize(List potentialDataNodes, + ContainerBalancerConfiguration config, Double lowerLimit); +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java index a2da1a61c028..3a7b8c4c28c5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java @@ -27,13 +27,14 @@ import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; +import org.apache.hadoop.hdds.scm.node.NodeManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.BiFunction; import java.util.stream.Collectors; /** @@ -46,34 +47,50 @@ public class FindTargetGreedy implements FindTargetStrategy { private ContainerManager containerManager; private PlacementPolicy placementPolicy; private Map sizeEnteringNode; + private NodeManager nodeManager; + private ContainerBalancerConfiguration config; + private Double upperLimit; + private List potentialTargets; public FindTargetGreedy( ContainerManager containerManager, PlacementPolicy placementPolicy, - Map sizeEnteringNode) { + NodeManager nodeManager) { + sizeEnteringNode = new HashMap<>(); this.containerManager = containerManager; this.placementPolicy = placementPolicy; - this.sizeEnteringNode = sizeEnteringNode; + this.nodeManager = nodeManager; + } + + private void setUpperLimit(Double upperLimit){ + this.upperLimit = upperLimit; + } + + private void setPotentialTargets(List potentialTargets) { + this.potentialTargets = potentialTargets; + sizeEnteringNode.clear(); + potentialTargets.forEach( + p -> sizeEnteringNode.put(p.getDatanodeDetails(), 0L)); + } + + private void setConfiguration(ContainerBalancerConfiguration conf) { + this.config = conf; } /** * Find a {@link ContainerMoveSelection} consisting of a target and * container to move for a source datanode. Favours more under-utilized nodes. * @param source Datanode to find a target for - * @param potentialTargets Collection of potential target datanodes * @param candidateContainers Set of candidate containers satisfying * selection criteria * {@link ContainerBalancerSelectionCriteria} - * @param canSizeEnterTarget A functional interface whose apply * (DatanodeDetails, Long) method returns true if the size specified in the * second argument can enter the specified DatanodeDetails node * @return Found target and container */ @Override public ContainerMoveSelection findTargetForContainerMove( - DatanodeDetails source, List potentialTargets, - Set candidateContainers, - BiFunction canSizeEnterTarget) { + DatanodeDetails source, Set candidateContainers) { potentialTargets.sort((a, b) -> { double currentUsageOfA = a.calculateUtilization( sizeEnteringNode.get(a.getDatanodeDetails())); @@ -100,7 +117,7 @@ public ContainerMoveSelection findTargetForContainerMove( replica -> replica.getDatanodeDetails().equals(target)) && containerMoveSatisfiesPlacementPolicy(container, replicas, source, target) && - canSizeEnterTarget.apply(target, containerInfo.getUsedBytes())) { + canSizeEnterTarget(target, containerInfo.getUsedBytes())) { return new ContainerMoveSelection(target, container); } } @@ -119,8 +136,7 @@ public ContainerMoveSelection findTargetForContainerMove( * @param target Target datanode for container move * @return true if placement policy is satisfied, otherwise false */ - @Override - public boolean containerMoveSatisfiesPlacementPolicy( + private boolean containerMoveSatisfiesPlacementPolicy( ContainerID containerID, Set replicas, DatanodeDetails source, DatanodeDetails target) { ContainerInfo containerInfo; @@ -144,4 +160,59 @@ public boolean containerMoveSatisfiesPlacementPolicy( return placementStatus.isPolicySatisfied(); } + + /** + * Checks if specified size can enter specified target datanode + * according to {@link ContainerBalancerConfiguration} + * "size.entering.target.max". + * + * @param target target datanode in which size is entering + * @param size size in bytes + * @return true if size can enter target, else false + */ + private boolean canSizeEnterTarget(DatanodeDetails target, long size) { + if (sizeEnteringNode.containsKey(target)) { + long sizeEnteringAfterMove = sizeEnteringNode.get(target) + size; + //size can be moved into target datanode only when the following + //two condition are met. + //1 sizeEnteringAfterMove does not succeed the configured + // MaxSizeEnteringTarget + //2 current usage of target datanode plus sizeEnteringAfterMove + // is smaller than or equal to upperLimit + return sizeEnteringAfterMove <= config.getMaxSizeEnteringTarget() && + Double.compare(nodeManager.getUsageInfo(target) + .calculateUtilization(sizeEnteringAfterMove), upperLimit) <= 0; + } + return false; + } + + /** + * increase the Entering size of a candidate target data node. + */ + @Override + public void increaseSizeEntering(DatanodeDetails target, long size) { + if(sizeEnteringNode.containsKey(target)) { + long totalEnteringSize = sizeEnteringNode.get(target) + size; + sizeEnteringNode.put(target, totalEnteringSize); + if(totalEnteringSize >= config.getMaxSizeEnteringTarget()) { + potentialTargets.removeIf( + c -> c.getDatanodeDetails().equals(target)); + } + return; + } + LOG.warn("Cannot find {} in the candidates target nodes", + target.getUuid()); + } + + /** + * reInitialize FindTargetStrategy with the given new parameters. + */ + @Override + public void reInitialize(List potentialDataNodes, + ContainerBalancerConfiguration conf, + Double upLimit) { + setConfiguration(conf); + setUpperLimit(upLimit); + setPotentialTargets(potentialDataNodes); + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java index 1ef652e22539..de87860fd31f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java @@ -20,12 +20,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; import java.util.List; import java.util.Set; -import java.util.function.BiFunction; /** * This interface can be used to implement strategies to find a target for a @@ -40,33 +38,26 @@ public interface FindTargetStrategy { * enter a potential target. * * @param source Datanode to find a target for - * @param potentialTargets Collection of potential target datanodes * @param candidateContainers Set of candidate containers satisfying * selection criteria * {@link ContainerBalancerSelectionCriteria} - * @param canSizeEnterTarget A functional interface whose apply * (DatanodeDetails, Long) method returns true if the size specified in the * second argument can enter the specified DatanodeDetails node * @return {@link ContainerMoveSelection} containing the target node and * selected container */ ContainerMoveSelection findTargetForContainerMove( - DatanodeDetails source, List potentialTargets, - Set candidateContainers, - BiFunction canSizeEnterTarget); + DatanodeDetails source, Set candidateContainers); /** - * Checks whether moving the specified container from the specified source - * to target datanode will satisfy the placement policy. - * - * @param containerID Container to be moved from source to target - * @param replicas Set of replicas of the given container - * @param source Source datanode for container move - * @param target Target datanode for container move - * @return true if placement policy is satisfied + * increase the Entering size of a candidate target data node. */ - boolean containerMoveSatisfiesPlacementPolicy(ContainerID containerID, - Set replicas, - DatanodeDetails source, - DatanodeDetails target); + void increaseSizeEntering(DatanodeDetails target, long size); + + /** + * reInitialize FindTargetStrategy. + */ + void reInitialize(List potentialDataNodes, + ContainerBalancerConfiguration config, Double upperLimit); + } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java deleted file mode 100644 index ee62f0d9592d..000000000000 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/SourceDataNodeSelectionCriteria.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.hdds.scm.container.balancer; - -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Map; - -/** - * The selection criteria for selecting source datanodes , the containers of - * which will be moved out. - */ -public class SourceDataNodeSelectionCriteria { - private static final Logger LOG = - LoggerFactory.getLogger(SourceDataNodeSelectionCriteria.class); - private Map sizeLeavingNode; - private List overUtilizedNodes; - - public SourceDataNodeSelectionCriteria( - List overUtilizedNodes, - Map sizeLeavingNode) { - this.sizeLeavingNode = sizeLeavingNode; - this.overUtilizedNodes = overUtilizedNodes; - } - - public DatanodeDetails getNextCandidateSourceDataNode() { - if (overUtilizedNodes.isEmpty()) { - LOG.info("no more candidate source data node"); - return null; - } - //TODO：use a more quick data structure, which will hava a - // better performance when changing or deleting one element at once - overUtilizedNodes.sort((a, b) -> { - double currentUsageOfA = a.calculateUtilization( - -sizeLeavingNode.get(a.getDatanodeDetails())); - double currentUsageOfB = b.calculateUtilization( - -sizeLeavingNode.get(b.getDatanodeDetails())); - //in descending order - return Double.compare(currentUsageOfB, currentUsageOfA); - }); - - return overUtilizedNodes.get(0).getDatanodeDetails(); - } - - public void removeCandidateSourceDataNode(DatanodeDetails dui){ - overUtilizedNodes.removeIf(a -> a.getDatanodeDetails().equals(dui)); - } -} From 23ff5bda3ffa0c2a7a383f2327ae40cd46ac090c Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Wed, 17 Nov 2021 20:23:36 +0800 Subject: [PATCH 05/12] fix comments --- .../container/balancer/ContainerBalancer.java | 19 +---------- .../ContainerBalancerSelectionCriteria.java | 22 ++++++++++++- .../container/balancer/FindSourceGreedy.java | 33 ++++++++++--------- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index bd96fa69d50d..a234682f7412 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -338,7 +338,7 @@ private boolean initializeIteration() { overUtilizedNodes.size(), underUtilizedNodes.size()); selectionCriteria = new ContainerBalancerSelectionCriteria(config, - nodeManager, replicationManager, containerManager); + nodeManager, replicationManager, containerManager, findSourceStrategy); sourceToTargetMap = new HashMap<>(overUtilizedNodes.size() + withinThresholdUtilizedNodes.size()); return true; @@ -476,23 +476,6 @@ private ContainerMoveSelection matchSourceWithTarget(DatanodeDetails source) { return null; } - //if the utilization of the source data node becomes lower than lowerLimit - //after the container is moved out , then the container can not be - // a candidate one, and we should remove it from the candidateContainers. - candidateContainers.removeIf(c -> { - ContainerInfo cInfo; - try { - cInfo = containerManager.getContainer(c); - } catch (ContainerNotFoundException e) { - LOG.warn("Could not find container {} when " + - "be matched with a move target", c); - //remove this not found container - return true; - } - return !findSourceStrategy.canSizeLeaveSource( - source, cInfo.getUsedBytes()); - }); - if (LOG.isDebugEnabled()) { LOG.debug("ContainerBalancer is finding suitable target for source " + "datanode {}", source.getUuidString()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java index 11d157184e53..b5f5acd8fa5b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java @@ -48,18 +48,21 @@ public class ContainerBalancerSelectionCriteria { private ContainerManager containerManager; private Set selectedContainers; private Set excludeContainers; + private FindSourceStrategy findSourceStrategy; public ContainerBalancerSelectionCriteria( ContainerBalancerConfiguration balancerConfiguration, NodeManager nodeManager, ReplicationManager replicationManager, - ContainerManager containerManager) { + ContainerManager containerManager, + FindSourceStrategy findSourceStrategy) { this.balancerConfiguration = balancerConfiguration; this.nodeManager = nodeManager; this.replicationManager = replicationManager; this.containerManager = containerManager; selectedContainers = new HashSet<>(); excludeContainers = balancerConfiguration.getExcludeContainers(); + this.findSourceStrategy = findSourceStrategy; } /** @@ -115,6 +118,23 @@ public NavigableSet getCandidateContainers( } }); + //if the utilization of the source data node becomes lower than lowerLimit + //after the container is moved out , then the container can not be + // a candidate one, and we should remove it from the candidateContainers. + containerIDSet.removeIf(c -> { + ContainerInfo cInfo; + try { + cInfo = containerManager.getContainer(c); + } catch (ContainerNotFoundException e) { + LOG.warn("Could not find container {} when " + + "be matched with a move target", c); + //remove this not found container + return true; + } + return !findSourceStrategy.canSizeLeaveSource( + node, cInfo.getUsedBytes()); + }); + containerIDSet.removeIf(this::isContainerReplicatingOrDeleting); return containerIDSet; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java index 35434f360dbb..634ddc41f2d3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.PriorityQueue; /** * The selection criteria for selecting source datanodes , the containers of @@ -35,13 +36,21 @@ public class FindSourceGreedy implements FindSourceStrategy{ private static final Logger LOG = LoggerFactory.getLogger(FindSourceGreedy.class); private Map sizeLeavingNode; - private List potentialSources; + private PriorityQueue potentialSources; private NodeManager nodeManager; private ContainerBalancerConfiguration config; private Double lowerLimit; FindSourceGreedy(NodeManager nodeManager) { sizeLeavingNode = new HashMap<>(); + potentialSources = new PriorityQueue<>((a, b) -> { + double currentUsageOfA = a.calculateUtilization( + -sizeLeavingNode.get(a.getDatanodeDetails())); + double currentUsageOfB = b.calculateUtilization( + -sizeLeavingNode.get(b.getDatanodeDetails())); + //in descending order + return Double.compare(currentUsageOfB, currentUsageOfA); + }); this.nodeManager = nodeManager; } @@ -50,11 +59,12 @@ private void setLowerLimit(Double lowerLimit) { } private void setPotentialSources( - List potentialSources) { - this.potentialSources = potentialSources; + List potentialSourceDataNodes) { + potentialSources.clear(); sizeLeavingNode.clear(); - potentialSources.forEach( + potentialSourceDataNodes.forEach( c -> sizeLeavingNode.put(c.getDatanodeDetails(), 0L)); + potentialSources.addAll(potentialSourceDataNodes); } private void setConfiguration(ContainerBalancerConfiguration conf) { @@ -69,6 +79,8 @@ public void increaseSizeLeaving(DatanodeDetails dui, long size) { Long currentSize = sizeLeavingNode.get(dui); if(currentSize != null) { sizeLeavingNode.put(dui, currentSize + size); + //reorder according to the latest sizeLeavingNode + potentialSources.add(nodeManager.getUsageInfo(dui)); return; } LOG.warn("Cannot find datanode {} in candidate source datanodes", @@ -87,18 +99,7 @@ public DatanodeDetails getNextCandidateSourceDataNode() { LOG.info("no more candidate source data node"); return null; } - //TODO：use a more quick data structure, which will hava a - // better performance when changing or deleting one element at once - potentialSources.sort((a, b) -> { - double currentUsageOfA = a.calculateUtilization( - -sizeLeavingNode.get(a.getDatanodeDetails())); - double currentUsageOfB = b.calculateUtilization( - -sizeLeavingNode.get(b.getDatanodeDetails())); - //in descending order - return Double.compare(currentUsageOfB, currentUsageOfA); - }); - - return potentialSources.get(0).getDatanodeDetails(); + return potentialSources.poll().getDatanodeDetails(); } /** From 93c32408316d5e18df75a24c3ea6831d95b02be6 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Wed, 17 Nov 2021 21:09:43 +0800 Subject: [PATCH 06/12] triger CI From 911dfbe5ac7e3c7dadab031619723f83731097dd Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Thu, 18 Nov 2021 16:49:38 +0800 Subject: [PATCH 07/12] fix comments --- .../ContainerBalancerConfiguration.java | 21 ++++++++--- .../container/balancer/FindTargetGreedy.java | 35 +++++++++++-------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index b3697c44c67c..4e7b88999824 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -24,7 +24,9 @@ import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.ConfigType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.fs.DUFactory; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.ozone.OzoneConsts; import org.slf4j.Logger; @@ -73,15 +75,17 @@ public final class ContainerBalancerConfiguration { defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can enter a target datanode in each " + "iteration while balancing. This is the sum of data from multiple " + - "sources. by default, we do not limit this value") - private long maxSizeEnteringTarget = Long.MAX_VALUE; + "sources. The default value is greater than the configured" + + " (or default) ozone.scm.container.size by 1GB.") + private long maxSizeEnteringTarget; @Config(key = "size.leaving.source.max", type = ConfigType.SIZE, defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can leave a source datanode in each " + "iteration while balancing. This is the sum of data moving to multiple " + - "targets. by default, we do not limit this value") - private long maxSizeLeavingSource = Long.MAX_VALUE;; + "targets. The default value is greater than the configured" + + " (or default) ozone.scm.container.size by 1GB.") + private long maxSizeLeavingSource; @Config(key = "idle.iterations", type = ConfigType.INT, defaultValue = "10", tags = {ConfigTag.BALANCER}, @@ -117,6 +121,15 @@ public ContainerBalancerConfiguration(OzoneConfiguration config) { "OzoneConfiguration should not be null."); this.ozoneConfiguration = config; + // maxSizeEnteringTarget and maxSizeLeavingSource should by default be + // greater than container size + long size = (long) ozoneConfiguration.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.GB) + + OzoneConsts.GB; + maxSizeEnteringTarget = size; + maxSizeLeavingSource = size; + // balancing interval should be greater than DUFactory refresh period duConf = ozoneConfiguration.getObject(DUFactory.Conf.class); balancingInterval = duConf.getRefreshPeriod().toMillis() + diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java index 3a7b8c4c28c5..97c9bfe1550f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; /** @@ -50,7 +51,7 @@ public class FindTargetGreedy implements FindTargetStrategy { private NodeManager nodeManager; private ContainerBalancerConfiguration config; private Double upperLimit; - private List potentialTargets; + private TreeSet potentialTargets; public FindTargetGreedy( ContainerManager containerManager, @@ -60,17 +61,27 @@ public FindTargetGreedy( this.containerManager = containerManager; this.placementPolicy = placementPolicy; this.nodeManager = nodeManager; + + potentialTargets = new TreeSet<>((a, b) -> { + double currentUsageOfA = a.calculateUtilization( + sizeEnteringNode.get(a.getDatanodeDetails())); + double currentUsageOfB = b.calculateUtilization( + sizeEnteringNode.get(b.getDatanodeDetails())); + return Double.compare(currentUsageOfA, currentUsageOfB); + }); } private void setUpperLimit(Double upperLimit){ this.upperLimit = upperLimit; } - private void setPotentialTargets(List potentialTargets) { - this.potentialTargets = potentialTargets; + private void setPotentialTargets( + List potentialTargetDataNodes) { sizeEnteringNode.clear(); - potentialTargets.forEach( + potentialTargetDataNodes.forEach( p -> sizeEnteringNode.put(p.getDatanodeDetails(), 0L)); + potentialTargets.clear(); + potentialTargets.addAll(potentialTargetDataNodes); } private void setConfiguration(ContainerBalancerConfiguration conf) { @@ -91,14 +102,6 @@ private void setConfiguration(ContainerBalancerConfiguration conf) { @Override public ContainerMoveSelection findTargetForContainerMove( DatanodeDetails source, Set candidateContainers) { - potentialTargets.sort((a, b) -> { - double currentUsageOfA = a.calculateUtilization( - sizeEnteringNode.get(a.getDatanodeDetails())); - double currentUsageOfB = b.calculateUtilization( - sizeEnteringNode.get(b.getDatanodeDetails())); - return Double.compare(currentUsageOfA, currentUsageOfB); - }); - for (DatanodeUsageInfo targetInfo : potentialTargets) { DatanodeDetails target = targetInfo.getDatanodeDetails(); for (ContainerID container : candidateContainers) { @@ -194,9 +197,11 @@ public void increaseSizeEntering(DatanodeDetails target, long size) { if(sizeEnteringNode.containsKey(target)) { long totalEnteringSize = sizeEnteringNode.get(target) + size; sizeEnteringNode.put(target, totalEnteringSize); - if(totalEnteringSize >= config.getMaxSizeEnteringTarget()) { - potentialTargets.removeIf( - c -> c.getDatanodeDetails().equals(target)); + potentialTargets.removeIf( + c -> c.getDatanodeDetails().equals(target)); + if(totalEnteringSize < config.getMaxSizeEnteringTarget()) { + //reorder + potentialTargets.add(nodeManager.getUsageInfo(target)); } return; } From 7bca9dc17c9c6245a15e60214593c56abf05b221 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Thu, 18 Nov 2021 19:36:53 +0800 Subject: [PATCH 08/12] fix comments --- .../hdds/scm/container/balancer/FindSourceGreedy.java | 6 +++++- .../hdds/scm/container/balancer/FindTargetGreedy.java | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java index 634ddc41f2d3..b15558bbaf57 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java @@ -49,7 +49,11 @@ public class FindSourceGreedy implements FindSourceStrategy{ double currentUsageOfB = b.calculateUtilization( -sizeLeavingNode.get(b.getDatanodeDetails())); //in descending order - return Double.compare(currentUsageOfB, currentUsageOfA); + int ret = Double.compare(currentUsageOfB, currentUsageOfA); + if (ret != 0) { + return ret; + } + return a.hashCode() - b.hashCode(); }); this.nodeManager = nodeManager; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java index 97c9bfe1550f..e545fc91243c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java @@ -67,7 +67,11 @@ public FindTargetGreedy( sizeEnteringNode.get(a.getDatanodeDetails())); double currentUsageOfB = b.calculateUtilization( sizeEnteringNode.get(b.getDatanodeDetails())); - return Double.compare(currentUsageOfA, currentUsageOfB); + int ret = Double.compare(currentUsageOfA, currentUsageOfB); + if (ret != 0) { + return ret; + } + return a.hashCode() - b.hashCode(); }); } From 8a320eeda36181100d5ebe5af4a902710b349391 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Thu, 18 Nov 2021 21:10:13 +0800 Subject: [PATCH 09/12] display more container configuration parameters --- .../balancer/ContainerBalancerConfiguration.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index 4e7b88999824..a77c1b7698ed 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -307,9 +307,16 @@ public String toString() { "%-50s %s%n" + "%-50s %s%n" + "%-50s %s%n" + - "%-50s %dB%n", "Key", "Value", "Threshold", + "%-50s %dGB%n"+ + "%-50s %dGB%n"+ + "%-50s %dGB%n", "Key", "Value", "Threshold", threshold, "Max Datanodes to Involve per Iteration(ratio)", maxDatanodesRatioToInvolvePerIteration, - "Max Size to Move per Iteration", maxSizeToMovePerIteration); + "Max Size to Move per Iteration", + maxSizeToMovePerIteration / OzoneConsts.GB, + "Max Size Entering Target per Iteration", + maxSizeEnteringTarget / OzoneConsts.GB, + "Max Size Leaving Source per Iteration", + maxSizeLeavingSource / OzoneConsts.GB); } } From a8d24b5dbabdf6f4bc8ef0a8ce91932a1cfd185e Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Fri, 19 Nov 2021 14:11:21 +0800 Subject: [PATCH 10/12] use uuid for comparition --- .../hadoop/hdds/scm/container/balancer/FindSourceGreedy.java | 5 ++++- .../hadoop/hdds/scm/container/balancer/FindTargetGreedy.java | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java index b15558bbaf57..591461d88750 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.PriorityQueue; +import java.util.UUID; /** * The selection criteria for selecting source datanodes , the containers of @@ -53,7 +54,9 @@ public class FindSourceGreedy implements FindSourceStrategy{ if (ret != 0) { return ret; } - return a.hashCode() - b.hashCode(); + UUID uuidA = a.getDatanodeDetails().getUuid(); + UUID uuidB = b.getDatanodeDetails().getUuid(); + return uuidA.compareTo(uuidB); }); this.nodeManager = nodeManager; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java index e545fc91243c..550894314655 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.UUID; import java.util.stream.Collectors; /** @@ -71,7 +72,9 @@ public FindTargetGreedy( if (ret != 0) { return ret; } - return a.hashCode() - b.hashCode(); + UUID uuidA = a.getDatanodeDetails().getUuid(); + UUID uuidB = b.getDatanodeDetails().getUuid(); + return uuidA.compareTo(uuidB); }); } From 9f4cfc12a5aa475f4e8d04218747694c4639d60e Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Fri, 19 Nov 2021 15:10:31 +0800 Subject: [PATCH 11/12] triger CI From 83c195c3924ada266a15fdd7596ce2c28215e408 Mon Sep 17 00:00:00 2001 From: Jackson Yao Date: Fri, 19 Nov 2021 17:04:33 +0800 Subject: [PATCH 12/12] triger CI