From cff663f89f7e7595af4d43a2abc3afe86a6e4009 Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Fri, 11 Apr 2025 20:23:48 +0530 Subject: [PATCH 1/5] YARN-11801: NPE in FifoCandidatesSelector.selectCandidates when preempting resources for an auto-created queue without child queues --- .../ProportionalCapacityPreemptionPolicy.java | 28 +++++-- .../capacity/AbstractParentQueue.java | 5 ++ ...tProportionalCapacityPreemptionPolicy.java | 79 ++++++++++++------- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 235c60e8b876e..0dc50a0554683 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -22,6 +22,7 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -430,15 +431,28 @@ private void cleanupStaledPreemptionCandidates(long currentTime) { } private Set getLeafQueueNames(TempQueuePerPartition q) { - // Also exclude ParentQueues, which might be without children - if (CollectionUtils.isEmpty(q.children) - && !(q.parentQueue instanceof ManagedParentQueue) - && (q.parentQueue == null - || !q.parentQueue.isEligibleForAutoQueueCreation())) { - return ImmutableSet.of(q.queueName); + Set leafQueueNames = new HashSet<>(); + + boolean isAutoQueueEligible = q.parentQueue != null && + (q.parentQueue.isEligibleForAutoQueueCreation() || + q.parentQueue.isEligibleForLegacyAutoQueueCreation()); + boolean isManagedParent = q.parentQueue instanceof ManagedParentQueue; + + if (isAutoQueueEligible || isManagedParent) { + return leafQueueNames; + } + + // Only consider this a leaf queue if: + // It has no children AND + // It is a concrete leaf queue (not a childless parent) + if (CollectionUtils.isEmpty(q.children)) { + CSQueue queue = scheduler.getQueue(q.queueName); + if (queue instanceof LeafQueue) { + leafQueueNames.add(q.queueName); + } + return leafQueueNames; } - Set leafQueueNames = new HashSet<>(); for (TempQueuePerPartition child : q.children) { leafQueueNames.addAll(getLeafQueueNames(child)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java index 87333bf50a143..5cfce3ca38ef1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java @@ -553,6 +553,11 @@ public boolean isEligibleForAutoQueueCreation() { isAutoQueueCreationV2Enabled(getQueuePathObject()); } + public boolean isEligibleForLegacyAutoQueueCreation() { + return isDynamicQueue() || queueContext.getConfiguration(). + isAutoCreateChildQueueEnabled(getQueuePathObject()); + } + @Override public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index 5b40000dd349a..54eb488498dbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -1083,46 +1083,67 @@ public void testRefreshPreemptionProperties() throws Exception { } @Test - public void testLeafQueueNameExtraction() throws Exception { - ProportionalCapacityPreemptionPolicy policy = - buildPolicy(Q_DATA_FOR_IGNORE); + public void testLeafQueueNameExtractionWithFlexibleAQC() throws Exception { + runLeafQueueNameExtractionTest(true); + } + + @Test + public void testLeafQueueNameExtractionWithLegacyAQC() throws Exception { + runLeafQueueNameExtractionTest(false); + } + + private void runLeafQueueNameExtractionTest(boolean isFlexible) throws Exception { + ProportionalCapacityPreemptionPolicy policy = buildPolicy(Q_DATA_FOR_IGNORE); ParentQueue root = (ParentQueue) mCS.getRootQueue(); - root.addDynamicParentQueue("childlessFlexible"); + + String queueName = isFlexible ? "childlessFlexible" : "childlessLegacy"; + String dynamicQueuePath = isFlexible ? "root.dynamicParent" : "root.dynamicLegacyParent"; + + root.addDynamicParentQueue(queueName); List queues = root.getChildQueues(); ArrayList extendedQueues = new ArrayList<>(); LinkedList pqs = new LinkedList<>(); - ParentQueue dynamicParent = mockParentQueue( - null, 0, pqs); - when(dynamicParent.getQueuePath()).thenReturn("root.dynamicParent"); - when(dynamicParent.getQueueCapacities()).thenReturn( - new QueueCapacities(false)); - QueueResourceQuotas dynamicParentQr = new QueueResourceQuotas(); - dynamicParentQr.setEffectiveMaxResource(Resource.newInstance(1, 1)); - dynamicParentQr.setEffectiveMinResource(Resources.createResource(1)); - dynamicParentQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL, - Resource.newInstance(1, 1)); - dynamicParentQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, - Resources.createResource(1)); - when(dynamicParent.getQueueResourceQuotas()).thenReturn(dynamicParentQr); - when(dynamicParent.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL)) - .thenReturn(Resources.createResource(1)); - when(dynamicParent.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL)) - .thenReturn(Resource.newInstance(1, 1)); - ResourceUsage resUsage = new ResourceUsage(); - resUsage.setUsed(Resources.createResource(1024)); - resUsage.setReserved(Resources.createResource(1024)); - when(dynamicParent.getQueueResourceUsage()).thenReturn(resUsage); - when(dynamicParent.isEligibleForAutoQueueCreation()).thenReturn(true); + + ParentQueue dynamicParent = mockParentQueue(null, 0, pqs); + mockQueueFields(dynamicParent, dynamicQueuePath); + + if (isFlexible) { + when(dynamicParent.isEligibleForAutoQueueCreation()).thenReturn(true); + } else { + when(dynamicParent.isEligibleForLegacyAutoQueueCreation()).thenReturn(true); + } + extendedQueues.add(dynamicParent); extendedQueues.addAll(queues); when(root.getChildQueues()).thenReturn(extendedQueues); policy.editSchedule(); - assertFalse(policy.getLeafQueueNames().contains("root.dynamicParent"), + assertFalse(policy.getLeafQueueNames().contains(dynamicQueuePath), "dynamicParent should not be a LeafQueue candidate"); } + private void mockQueueFields(ParentQueue queue, String queuePath) { + when(queue.getQueuePath()).thenReturn(queuePath); + when(queue.getQueueCapacities()).thenReturn(new QueueCapacities(false)); + + QueueResourceQuotas qrq = new QueueResourceQuotas(); + qrq.setEffectiveMaxResource(Resource.newInstance(1, 1)); + qrq.setEffectiveMinResource(Resources.createResource(1)); + qrq.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL, Resource.newInstance(1, 1)); + qrq.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, Resources.createResource(1)); + + when(queue.getQueueResourceQuotas()).thenReturn(qrq); + when(queue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL)) + .thenReturn(Resources.createResource(1)); + when(queue.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL)) + .thenReturn(Resource.newInstance(1, 1)); + + ResourceUsage usage = new ResourceUsage(); + usage.setUsed(Resources.createResource(1024)); + usage.setReserved(Resources.createResource(1024)); + when(queue.getQueueResourceUsage()).thenReturn(usage); + } static class IsPreemptionRequestFor implements ArgumentMatcher { private final ApplicationAttemptId appAttId; @@ -1369,6 +1390,10 @@ LeafQueue mockLeafQueue(ParentQueue p, Resource tot, int i, Resource[] abs, Resource[] used, Resource[] pending, Resource[] reserved, int[] apps, Resource[] gran) { LeafQueue lq = mock(LeafQueue.class); + + String queuePath = p.getQueuePath() + ".queue" + (char)('A' + i - 1); + when(mCS.getQueue(queuePath)).thenReturn(lq); + ResourceCalculator rc = mCS.getResourceCalculator(); List appAttemptIdList = new ArrayList(); From c8777cc540fb67a4cc26634ad839c4f2d7410301 Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Sat, 12 Apr 2025 12:21:58 +0530 Subject: [PATCH 2/5] similar changes as prev code --- .../ProportionalCapacityPreemptionPolicy.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 0dc50a0554683..2563cf49f16e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -431,28 +431,26 @@ private void cleanupStaledPreemptionCandidates(long currentTime) { } private Set getLeafQueueNames(TempQueuePerPartition q) { - Set leafQueueNames = new HashSet<>(); - boolean isAutoQueueEligible = q.parentQueue != null && (q.parentQueue.isEligibleForAutoQueueCreation() || q.parentQueue.isEligibleForLegacyAutoQueueCreation()); boolean isManagedParent = q.parentQueue instanceof ManagedParentQueue; - if (isAutoQueueEligible || isManagedParent) { - return leafQueueNames; + if ((isAutoQueueEligible || isManagedParent)) { + return Collections.emptySet(); } // Only consider this a leaf queue if: - // It has no children AND + // It has no children and // It is a concrete leaf queue (not a childless parent) if (CollectionUtils.isEmpty(q.children)) { CSQueue queue = scheduler.getQueue(q.queueName); if (queue instanceof LeafQueue) { - leafQueueNames.add(q.queueName); + return ImmutableSet.of(q.queueName); } - return leafQueueNames; + return Collections.emptySet(); } - + Set leafQueueNames = new HashSet<>(); for (TempQueuePerPartition child : q.children) { leafQueueNames.addAll(getLeafQueueNames(child)); } From 711051f4bac30e41b8daa32311627b2e1496f1a5 Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Tue, 15 Apr 2025 16:27:43 +0530 Subject: [PATCH 3/5] included auto created leaf queue --- .../ProportionalCapacityPreemptionPolicy.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 2563cf49f16e7..1970320aee90d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -41,8 +42,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptableQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; @@ -431,25 +430,16 @@ private void cleanupStaledPreemptionCandidates(long currentTime) { } private Set getLeafQueueNames(TempQueuePerPartition q) { - boolean isAutoQueueEligible = q.parentQueue != null && - (q.parentQueue.isEligibleForAutoQueueCreation() || - q.parentQueue.isEligibleForLegacyAutoQueueCreation()); - boolean isManagedParent = q.parentQueue instanceof ManagedParentQueue; - - if ((isAutoQueueEligible || isManagedParent)) { - return Collections.emptySet(); - } - // Only consider this a leaf queue if: - // It has no children and // It is a concrete leaf queue (not a childless parent) if (CollectionUtils.isEmpty(q.children)) { CSQueue queue = scheduler.getQueue(q.queueName); - if (queue instanceof LeafQueue) { + if (queue instanceof LeafQueue || queue instanceof AutoCreatedLeafQueue) { return ImmutableSet.of(q.queueName); } return Collections.emptySet(); } + Set leafQueueNames = new HashSet<>(); for (TempQueuePerPartition child : q.children) { leafQueueNames.addAll(getLeafQueueNames(child)); From c548f0b158fb6f6402c32626faea1075cf37302f Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Wed, 16 Apr 2025 09:48:08 +0530 Subject: [PATCH 4/5] empty commit From 327031d5535c66e5b0c5c1ed405479f271f93679 Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Tue, 22 Apr 2025 18:05:48 +0530 Subject: [PATCH 5/5] used abstractleafqueue class for instance checking --- .../ProportionalCapacityPreemptionPolicy.java | 5 +- .../capacity/AbstractParentQueue.java | 6 ++- ...tProportionalCapacityPreemptionPolicy.java | 47 +++++++++++-------- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 1970320aee90d..866012d1427e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -22,8 +22,7 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -434,7 +433,7 @@ private Set getLeafQueueNames(TempQueuePerPartition q) { // It is a concrete leaf queue (not a childless parent) if (CollectionUtils.isEmpty(q.children)) { CSQueue queue = scheduler.getQueue(q.queueName); - if (queue instanceof LeafQueue || queue instanceof AutoCreatedLeafQueue) { + if (queue instanceof AbstractLeafQueue) { return ImmutableSet.of(q.queueName); } return Collections.emptySet(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java index 5cfce3ca38ef1..369c44285d6dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java @@ -552,7 +552,11 @@ public boolean isEligibleForAutoQueueCreation() { return isDynamicQueue() || queueContext.getConfiguration(). isAutoQueueCreationV2Enabled(getQueuePathObject()); } - + /** + * Check whether this queue supports legacy(v1) dynamic child queue creation. + * @return true if queue is eligible to create child queues dynamically using + * the legacy system, false otherwise + */ public boolean isEligibleForLegacyAutoQueueCreation() { return isDynamicQueue() || queueContext.getConfiguration(). isAutoCreateChildQueueEnabled(getQueuePathObject()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index 54eb488498dbb..87394eed0da69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -1084,28 +1084,35 @@ public void testRefreshPreemptionProperties() throws Exception { @Test public void testLeafQueueNameExtractionWithFlexibleAQC() throws Exception { - runLeafQueueNameExtractionTest(true); + ProportionalCapacityPreemptionPolicy policy = buildPolicy(Q_DATA_FOR_IGNORE); + ParentQueue root = (ParentQueue) mCS.getRootQueue(); + + root.addDynamicParentQueue("childlessFlexible"); + ParentQueue dynamicParent = setupDynamicParentQueue("root.dynamicParent", true); + extendRootQueueWithMock(root, dynamicParent); + + policy.editSchedule(); + assertFalse(policy.getLeafQueueNames().contains( "root.dynamicParent"), + "root.dynamicLegacyParent" + " should not be a LeafQueue candidate"); } @Test public void testLeafQueueNameExtractionWithLegacyAQC() throws Exception { - runLeafQueueNameExtractionTest(false); - } - - private void runLeafQueueNameExtractionTest(boolean isFlexible) throws Exception { ProportionalCapacityPreemptionPolicy policy = buildPolicy(Q_DATA_FOR_IGNORE); ParentQueue root = (ParentQueue) mCS.getRootQueue(); - String queueName = isFlexible ? "childlessFlexible" : "childlessLegacy"; - String dynamicQueuePath = isFlexible ? "root.dynamicParent" : "root.dynamicLegacyParent"; + root.addDynamicParentQueue("childlessLegacy"); + ParentQueue dynamicParent = setupDynamicParentQueue("root.dynamicLegacyParent", false); + extendRootQueueWithMock(root, dynamicParent); - root.addDynamicParentQueue(queueName); - List queues = root.getChildQueues(); - ArrayList extendedQueues = new ArrayList<>(); - LinkedList pqs = new LinkedList<>(); + policy.editSchedule(); + assertFalse(policy.getLeafQueueNames().contains( "root.dynamicLegacyParent"), + "root.dynamicLegacyParent" + " should not be a LeafQueue candidate"); + } - ParentQueue dynamicParent = mockParentQueue(null, 0, pqs); - mockQueueFields(dynamicParent, dynamicQueuePath); + private ParentQueue setupDynamicParentQueue(String queuePath, boolean isFlexible) { + ParentQueue dynamicParent = mockParentQueue(null, 0, new LinkedList<>()); + mockQueueFields(dynamicParent, queuePath); if (isFlexible) { when(dynamicParent.isEligibleForAutoQueueCreation()).thenReturn(true); @@ -1113,14 +1120,15 @@ private void runLeafQueueNameExtractionTest(boolean isFlexible) throws Exception when(dynamicParent.isEligibleForLegacyAutoQueueCreation()).thenReturn(true); } - extendedQueues.add(dynamicParent); + return dynamicParent; + } + + private void extendRootQueueWithMock(ParentQueue root, ParentQueue mockQueue) { + List queues = root.getChildQueues(); + ArrayList extendedQueues = new ArrayList<>(); + extendedQueues.add(mockQueue); extendedQueues.addAll(queues); when(root.getChildQueues()).thenReturn(extendedQueues); - - policy.editSchedule(); - - assertFalse(policy.getLeafQueueNames().contains(dynamicQueuePath), - "dynamicParent should not be a LeafQueue candidate"); } private void mockQueueFields(ParentQueue queue, String queuePath) { @@ -1144,6 +1152,7 @@ private void mockQueueFields(ParentQueue queue, String queuePath) { usage.setReserved(Resources.createResource(1024)); when(queue.getQueueResourceUsage()).thenReturn(usage); } + static class IsPreemptionRequestFor implements ArgumentMatcher { private final ApplicationAttemptId appAttId;