From a05598276e11bf7981b5c142a470e993293f375a Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Sat, 4 Jul 2020 22:14:33 +0800 Subject: [PATCH 1/3] HDDS-3921. IllegalArgumentException triggered in SCMContainerPlacementRackAware.chooseDatanodes --- .../hadoop/hdds/scm/container/ReplicationManager.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index 617e231073c7..65f9e24f3f6a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -512,7 +512,7 @@ private void forceCloseContainer(final ContainerInfo container, */ private void handleUnderReplicatedContainer(final ContainerInfo container, final Set replicas) { - LOG.debug("Handling underreplicated container: {}", + LOG.debug("Handling under-replicated container: {}", container.getContainerID()); try { final ContainerID id = container.containerID(); @@ -543,11 +543,15 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, List targetReplicas = new ArrayList<>(source); // Then add any pending additions targetReplicas.addAll(replicationInFlight); - - int delta = replicationFactor - getReplicaCount(id, replicas); final ContainerPlacementStatus placementStatus = containerPlacement.validateContainerPlacement( targetReplicas, replicationFactor); + int delta = replicationFactor - getReplicaCount(id, replicas); + if (placementStatus.isPolicySatisfied() && delta <= 0) { + LOG.debug( + "Container {} meets placement policy with inflight replicas", id); + return; + } final int misRepDelta = placementStatus.misReplicationCount(); final int replicasNeeded = delta < misRepDelta ? misRepDelta : delta; From 091a7b634b2b9342efdc28ef4acc9e3e77be4300 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 9 Jul 2020 16:26:53 +0800 Subject: [PATCH 2/3] address comments --- .../hdds/scm/container/ReplicationManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index 65f9e24f3f6a..7a250687c7ee 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -547,14 +547,14 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, containerPlacement.validateContainerPlacement( targetReplicas, replicationFactor); int delta = replicationFactor - getReplicaCount(id, replicas); - if (placementStatus.isPolicySatisfied() && delta <= 0) { - LOG.debug( - "Container {} meets placement policy with inflight replicas", id); - return; - } final int misRepDelta = placementStatus.misReplicationCount(); final int replicasNeeded = delta < misRepDelta ? misRepDelta : delta; + if (replicasNeeded <= 0) { + LOG.debug("Container {} meets replication requirement with " + + "inflight replicas", id); + return; + } final List excludeList = replicas.stream() .map(ContainerReplica::getDatanodeDetails) @@ -615,7 +615,7 @@ private void handleOverReplicatedContainer(final ContainerInfo container, final ContainerID id = container.containerID(); final int replicationFactor = container.getReplicationFactor().getNumber(); - // Dont consider inflight replication while calculating excess here. + // Don't consider inflight replication while calculating excess here. int excess = replicas.size() - replicationFactor - inflightDeletion.getOrDefault(id, Collections.emptyList()).size(); From 277b028818ffe992416c2b412805f2a790db0c50 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 10 Jul 2020 08:15:05 +0200 Subject: [PATCH 3/3] trigger new CI check