diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java index d7eb78ad1619..1aed5b76d3d4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java @@ -23,11 +23,11 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import static org.apache.hadoop.ozone.container.TestHelper.waitForContainerClose; +import static org.apache.hadoop.ozone.container.TestHelper.waitForContainerStateInSCM; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.io.OutputStream; @@ -41,7 +41,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.ObjectStore; @@ -52,15 +51,12 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.tag.Flaky; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; /** * Tests for container report handling. */ -@Flaky("HDDS-12535") public class TestContainerReportHandling { private static final String VOLUME = "vol1"; private static final String BUCKET = "bucket1"; @@ -97,6 +93,9 @@ void testDeletingOrDeletedContainerTransitionsToClosedWhenNonEmptyReplicaIsRepor ContainerID containerID = ContainerID.valueOf(keyLocation.getContainerID()); waitForContainerClose(cluster, containerID.getId()); + // also wait till the container is closed in SCM + waitForContainerStateInSCM(cluster.getStorageContainerManager(), containerID, HddsProtos.LifeCycleState.CLOSED); + // move the container to DELETING ContainerManager containerManager = cluster.getStorageContainerManager().getContainerManager(); containerManager.updateContainerState(containerID, HddsProtos.LifeCycleEvent.DELETE); @@ -111,14 +110,7 @@ void testDeletingOrDeletedContainerTransitionsToClosedWhenNonEmptyReplicaIsRepor // restart a DN and wait for the container to get CLOSED. HddsDatanodeService dn = cluster.getHddsDatanode(keyLocation.getPipeline().getFirstNode()); cluster.restartHddsDatanode(dn.getDatanodeDetails(), false); - GenericTestUtils.waitFor(() -> { - try { - return containerManager.getContainer(containerID).getState() == HddsProtos.LifeCycleState.CLOSED; - } catch (ContainerNotFoundException e) { - fail(e); - } - return false; - }, 2000, 20000); + waitForContainerStateInSCM(cluster.getStorageContainerManager(), containerID, HddsProtos.LifeCycleState.CLOSED); assertEquals(HddsProtos.LifeCycleState.CLOSED, containerManager.getContainer(containerID).getState()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java index 9517fd9e4598..357945a3fa5f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java @@ -23,11 +23,11 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import static org.apache.hadoop.ozone.container.TestHelper.waitForContainerClose; +import static org.apache.hadoop.ozone.container.TestHelper.waitForContainerStateInSCM; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.io.OutputStream; @@ -35,13 +35,14 @@ import java.nio.file.Paths; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; @@ -53,15 +54,12 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.tag.Flaky; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; /** * Tests for container report handling with SCM High Availability. */ -@Flaky("HDDS-12535") public class TestContainerReportHandlingWithHA { private static final String VOLUME = "vol1"; private static final String BUCKET = "bucket1"; @@ -99,6 +97,8 @@ void testDeletingOrDeletedContainerTransitionsToClosedWhenNonEmptyReplicaIsRepor ContainerID containerID = ContainerID.valueOf(keyLocation.getContainerID()); waitForContainerClose(cluster, containerID.getId()); + waitForContainerStateInAllSCMs(cluster, containerID, HddsProtos.LifeCycleState.CLOSED); + // move the container to DELETING ContainerManager containerManager = cluster.getScmLeader().getContainerManager(); containerManager.updateContainerState(containerID, HddsProtos.LifeCycleEvent.DELETE); @@ -113,23 +113,8 @@ void testDeletingOrDeletedContainerTransitionsToClosedWhenNonEmptyReplicaIsRepor // restart a DN and wait for the container to get CLOSED in all SCMs HddsDatanodeService dn = cluster.getHddsDatanode(keyLocation.getPipeline().getFirstNode()); cluster.restartHddsDatanode(dn.getDatanodeDetails(), false); - ContainerManager[] array = new ContainerManager[numSCM]; - for (int i = 0; i < numSCM; i++) { - array[i] = cluster.getStorageContainerManager(i).getContainerManager(); - } - GenericTestUtils.waitFor(() -> { - try { - for (ContainerManager manager : array) { - if (manager.getContainer(containerID).getState() != HddsProtos.LifeCycleState.CLOSED) { - return false; - } - } - return true; - } catch (ContainerNotFoundException e) { - fail(e); - } - return false; - }, 2000, 20000); + + waitForContainerStateInAllSCMs(cluster, containerID, HddsProtos.LifeCycleState.CLOSED); assertEquals(HddsProtos.LifeCycleState.CLOSED, containerManager.getContainer(containerID).getState()); } @@ -177,4 +162,12 @@ private void createTestData(OzoneClient client) throws IOException { } } + private static void waitForContainerStateInAllSCMs(MiniOzoneHAClusterImpl cluster, ContainerID containerID, + HddsProtos.LifeCycleState desiredState) + throws TimeoutException, InterruptedException { + for (StorageContainerManager scm : cluster.getStorageContainerManagersList()) { + waitForContainerStateInSCM(scm, containerID, desiredState); + } + } + } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java index 071386858623..c454facea070 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.security.MessageDigest; @@ -455,4 +456,20 @@ public static void setConfig(OzoneConfiguration conf, String key, String value) conf.set(key, value); } } + + public static void waitForContainerStateInSCM(StorageContainerManager scm, + ContainerID containerID, HddsProtos.LifeCycleState expectedState) + throws TimeoutException, InterruptedException { + ContainerManager containerManager = scm.getContainerManager(); + GenericTestUtils.waitFor(() -> { + try { + return containerManager.getContainer(containerID).getState() == expectedState; + } catch (ContainerNotFoundException e) { + LOG.error("Container {} not found while waiting for state {}", + containerID, expectedState, e); + fail("Container " + containerID + " not found while waiting for state " + expectedState + ": " + e); + return false; + } + }, 2000, 20000); + } }