From 39340a62a654edc9ed5b1eb8ffda14e5dbbde38a Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Thu, 6 Mar 2025 21:57:20 +0800 Subject: [PATCH 1/4] HDDS-12477. Do not force RATIS/ONE replication in TestDataUtil#createKey --- .../hadoop/hdds/scm/TestCloseContainer.java | 9 +-- .../org/apache/hadoop/ozone/TestDataUtil.java | 19 +++--- .../apache/hadoop/ozone/om/TestListKeys.java | 5 +- .../hadoop/ozone/om/TestListKeysWithFSO.java | 5 +- ...napshotDeletingServiceIntegrationTest.java | 59 ++++++------------- .../shell/TestOzoneContainerUpgradeShell.java | 6 +- 6 files changed, 33 insertions(+), 70 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java index 0b8449cb08db..f62dc8155399 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java @@ -35,9 +35,6 @@ import java.time.Duration; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -109,8 +106,7 @@ public void testReplicasAreReportedForClosedContainerAfterRestart() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, ReplicationConfig - .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), + TestDataUtil.createKey(bucket, "key" + i, null, "this is the content".getBytes(StandardCharsets.UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); @@ -155,8 +151,7 @@ public void testCloseClosedContainer() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, ReplicationConfig - .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), + TestDataUtil.createKey(bucket, "key" + i, null, "this is the content".getBytes(StandardCharsets.UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index b54bcd6f624b..e6a9c37dec63 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -32,8 +32,6 @@ import java.util.Scanner; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -106,18 +104,23 @@ public static OzoneVolume createVolume(OzoneClient client, public static void createKey(OzoneBucket bucket, String keyName, byte[] content) throws IOException { - ReplicationConfig replicationConfig = ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE); - createKey(bucket, keyName, replicationConfig, content); + createKey(bucket, keyName, null, content); } + public static OutputStream createOutputStream(OzoneBucket bucket, String keyName, + ReplicationConfig repConfig, byte[] content) + throws IOException { + return repConfig == null + ? bucket.createKey(keyName, content.length) + : bucket.createKey(keyName, content.length, repConfig, new HashMap<>()); + } + public static void createKey(OzoneBucket bucket, String keyName, ReplicationConfig repConfig, byte[] content) throws IOException { - try (OutputStream stream = bucket - .createKey(keyName, content.length, repConfig, - new HashMap<>())) { + try (OutputStream stream = createOutputStream(bucket, keyName, + repConfig, content)) { stream.write(content); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java index 2d6fbd1cfbc2..da9924b4ac97 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java @@ -34,8 +34,6 @@ import java.util.Optional; import java.util.stream.Stream; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.TestDataUtil; @@ -377,8 +375,7 @@ private static void createAndAssertKeys(OzoneBucket ozoneBucket, List ke byte[] input = new byte[length]; Arrays.fill(input, (byte) 96); for (String key : keys) { - createKey(ozoneBucket, key, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), input); + createKey(ozoneBucket, key, null, input); // Read the key with given key name. readkey(ozoneBucket, key, length, input); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 1f69f647c3ce..7246c7648fd8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java @@ -31,8 +31,6 @@ import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.IOUtils; @@ -649,8 +647,7 @@ private static void createAndAssertKeys(OzoneBucket ozoneBucket, List ke byte[] input = new byte[length]; Arrays.fill(input, (byte) 96); for (String key : keys) { - createKey(ozoneBucket, key, ReplicationConfig - .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), input); + createKey(ozoneBucket, key, null, input); // Read the key with given key name. readkey(ozoneBucket, key, length, input); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java index be7a929a83de..028956a79316 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java @@ -49,9 +49,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import org.apache.commons.compress.utils.Lists; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; @@ -202,10 +199,8 @@ public void testMultipleSnapshotKeyReclaim() throws Exception { OzoneBucket bucket2 = TestDataUtil.createBucket( client, VOLUME_NAME, bucketArgs, BUCKET_NAME_TWO); // Create key1 and key2 - TestDataUtil.createKey(bucket2, "bucket2key1", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket2, "bucket2key2", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket2, "bucket2key1", null, CONTENT.array()); + TestDataUtil.createKey(bucket2, "bucket2key2", null, CONTENT.array()); // Create Snapshot client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, @@ -265,16 +260,12 @@ public void testSnapshotWithFSO() throws Exception { // Create 10 keys for (int i = 1; i <= 10; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); } // Create 5 keys to overwrite for (int i = 11; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); } // Create Directory and Sub @@ -287,8 +278,7 @@ public void testSnapshotWithFSO() throws Exception { String childDir = "/childDir" + j; client.getProxy().createDirectory(VOLUME_NAME, BUCKET_NAME_FSO, parent + childDir); - TestDataUtil.createKey(bucket2, parent + childFile, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket2, parent + childFile, null, CONTENT.array()); } } @@ -304,8 +294,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 3 keys -> Moves previous version to deletedTable for (int i = 11; i <= 13; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); } assertTableRowCount(keyTable, 24); @@ -369,9 +358,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 2 keys for (int i = 14; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); } // Delete 2 more keys @@ -727,12 +714,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) om.getMetadataManager(); - TestDataUtil.createKey(bucket, bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key1", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key0", null, CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key1", null, CONTENT.array()); assertTableRowCount(keyTable, 2); // Create Snapshot 1. @@ -742,10 +725,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket, bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key2", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key0", null, CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key2", null, CONTENT.array()); // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), @@ -769,10 +750,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // deletedTable when Snapshot 2 is taken. assertTableRowCount(deletedTable, 0); - TestDataUtil.createKey(bucket, bucket.getName() + "key3", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key4", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key3", null, CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key4", null, CONTENT.array()); client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), bucket.getName() + "key4", false); assertTableRowCount(keyTable, 1); @@ -832,19 +811,15 @@ private synchronized void createSnapshotFSODataForBucket(OzoneBucket bucket) thr throw new RuntimeException(ex); } })); - TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, "dir1/" + bucket.getName() + "key1", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", null, CONTENT.array()); + TestDataUtil.createKey(bucket, "dir1/" + bucket.getName() + "key1", null, CONTENT.array()); assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 2); assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 2); // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, "dir2/" + bucket.getName() + "key2", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", null, CONTENT.array()); + TestDataUtil.createKey(bucket, "dir2/" + bucket.getName() + "key2", null, CONTENT.array()); assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 3); assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 3); assertTableRowCount(deletedTable, countMap.get(deletedTable.getName()) + 1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java index e6ada443ccd4..661307421c84 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java @@ -42,9 +42,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -183,8 +180,7 @@ private static void writeKey(String keyName) throws IOException { TestDataUtil.createVolumeAndBucket(client, VOLUME_NAME, BUCKET_NAME); TestDataUtil.createKey( client.getObjectStore().getVolume(VOLUME_NAME).getBucket(BUCKET_NAME), - keyName, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - "test".getBytes(StandardCharsets.UTF_8)); + keyName, null, "test".getBytes(StandardCharsets.UTF_8)); } } From 6a94ef7b9161c1558bb78cb14d023d31a2dbb39a Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Thu, 6 Mar 2025 23:24:58 +0800 Subject: [PATCH 2/4] Fixed tests --- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 6 +++++- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 11 ++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index e6a9c37dec63..fc2ce04ecef9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -32,6 +32,8 @@ import java.util.Scanner; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -217,7 +219,9 @@ public static Map createKeys(MiniOzoneCluster cluster, int nu OzoneBucket bucket = createVolumeAndBucket(client); for (int i = 0; i < numOfKeys; i++) { String keyName = RandomStringUtils.randomAlphabetic(5) + i; - createKey(bucket, keyName, RandomStringUtils.randomAlphabetic(5).getBytes(UTF_8)); + createKey(bucket, keyName,ReplicationConfig + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + RandomStringUtils.randomAlphabetic(5).getBytes(UTF_8)); keyLocationMap.put(keyName, lookupOmKeyInfo(cluster, bucket, keyName)); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 076bf2abd7a6..bf29ef48dfc8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -98,6 +98,9 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -676,9 +679,11 @@ private void prepSnapshotData() throws Exception { .createVolumeAndBucket(client); // Create dummy keys for snapshotting. - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), - "content".getBytes(StandardCharsets.UTF_8)); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE) + , "content".getBytes(StandardCharsets.UTF_8)); + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "content".getBytes(StandardCharsets.UTF_8)); snapshotDirName = From 78feceaa2ef9a9bea2977cea9a0cc5fc54b62022 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Thu, 6 Mar 2025 23:32:02 +0800 Subject: [PATCH 3/4] Fixed styles --- .../src/test/java/org/apache/hadoop/ozone/TestDataUtil.java | 2 +- .../org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index fc2ce04ecef9..8215c7a5a426 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -219,7 +219,7 @@ public static Map createKeys(MiniOzoneCluster cluster, int nu OzoneBucket bucket = createVolumeAndBucket(client); for (int i = 0; i < numOfKeys; i++) { String keyName = RandomStringUtils.randomAlphabetic(5) + i; - createKey(bucket, keyName,ReplicationConfig + createKey(bucket, keyName, ReplicationConfig .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), RandomStringUtils.randomAlphabetic(5).getBytes(UTF_8)); keyLocationMap.put(keyName, lookupOmKeyInfo(cluster, bucket, keyName)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index bf29ef48dfc8..75a6004643d4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -680,8 +680,8 @@ private void prepSnapshotData() throws Exception { // Create dummy keys for snapshotting. TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig - .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE) - , "content".getBytes(StandardCharsets.UTF_8)); + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + "content".getBytes(StandardCharsets.UTF_8)); TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "content".getBytes(StandardCharsets.UTF_8)); From 399a84bfbbf34d312c1f5f2a86e752297dc61967 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Fri, 7 Mar 2025 18:46:45 +0800 Subject: [PATCH 4/4] Apply comments --- .../hadoop/hdds/scm/TestCloseContainer.java | 4 +-- .../apache/hadoop/ozone/om/TestListKeys.java | 2 +- .../hadoop/ozone/om/TestListKeysWithFSO.java | 2 +- ...napshotDeletingServiceIntegrationTest.java | 34 +++++++++---------- .../shell/TestOzoneContainerUpgradeShell.java | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java index f62dc8155399..381c86d403aa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java @@ -106,7 +106,7 @@ public void testReplicasAreReportedForClosedContainerAfterRestart() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, null, + TestDataUtil.createKey(bucket, "key" + i, "this is the content".getBytes(StandardCharsets.UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); @@ -151,7 +151,7 @@ public void testCloseClosedContainer() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, null, + TestDataUtil.createKey(bucket, "key" + i, "this is the content".getBytes(StandardCharsets.UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java index da9924b4ac97..2f98caeb6965 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java @@ -375,7 +375,7 @@ private static void createAndAssertKeys(OzoneBucket ozoneBucket, List ke byte[] input = new byte[length]; Arrays.fill(input, (byte) 96); for (String key : keys) { - createKey(ozoneBucket, key, null, input); + createKey(ozoneBucket, key, input); // Read the key with given key name. readkey(ozoneBucket, key, length, input); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 7246c7648fd8..c8e8c21995ce 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java @@ -647,7 +647,7 @@ private static void createAndAssertKeys(OzoneBucket ozoneBucket, List ke byte[] input = new byte[length]; Arrays.fill(input, (byte) 96); for (String key : keys) { - createKey(ozoneBucket, key, null, input); + createKey(ozoneBucket, key, input); // Read the key with given key name. readkey(ozoneBucket, key, length, input); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java index 028956a79316..87dd31c2178b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java @@ -199,8 +199,8 @@ public void testMultipleSnapshotKeyReclaim() throws Exception { OzoneBucket bucket2 = TestDataUtil.createBucket( client, VOLUME_NAME, bucketArgs, BUCKET_NAME_TWO); // Create key1 and key2 - TestDataUtil.createKey(bucket2, "bucket2key1", null, CONTENT.array()); - TestDataUtil.createKey(bucket2, "bucket2key2", null, CONTENT.array()); + TestDataUtil.createKey(bucket2, "bucket2key1", CONTENT.array()); + TestDataUtil.createKey(bucket2, "bucket2key2", CONTENT.array()); // Create Snapshot client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, @@ -260,12 +260,12 @@ public void testSnapshotWithFSO() throws Exception { // Create 10 keys for (int i = 1; i <= 10; i++) { - TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Create 5 keys to overwrite for (int i = 11; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Create Directory and Sub @@ -278,7 +278,7 @@ public void testSnapshotWithFSO() throws Exception { String childDir = "/childDir" + j; client.getProxy().createDirectory(VOLUME_NAME, BUCKET_NAME_FSO, parent + childDir); - TestDataUtil.createKey(bucket2, parent + childFile, null, CONTENT.array()); + TestDataUtil.createKey(bucket2, parent + childFile, CONTENT.array()); } } @@ -294,7 +294,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 3 keys -> Moves previous version to deletedTable for (int i = 11; i <= 13; i++) { - TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } assertTableRowCount(keyTable, 24); @@ -358,7 +358,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 2 keys for (int i = 14; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, null, CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Delete 2 more keys @@ -714,8 +714,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) om.getMetadataManager(); - TestDataUtil.createKey(bucket, bucket.getName() + "key0", null, CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key1", null, CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key1", CONTENT.array()); assertTableRowCount(keyTable, 2); // Create Snapshot 1. @@ -725,8 +725,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket, bucket.getName() + "key0", null, CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key2", null, CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key2", CONTENT.array()); // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), @@ -750,8 +750,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // deletedTable when Snapshot 2 is taken. assertTableRowCount(deletedTable, 0); - TestDataUtil.createKey(bucket, bucket.getName() + "key3", null, CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key4", null, CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key3", CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key4", CONTENT.array()); client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), bucket.getName() + "key4", false); assertTableRowCount(keyTable, 1); @@ -811,15 +811,15 @@ private synchronized void createSnapshotFSODataForBucket(OzoneBucket bucket) thr throw new RuntimeException(ex); } })); - TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", null, CONTENT.array()); - TestDataUtil.createKey(bucket, "dir1/" + bucket.getName() + "key1", null, CONTENT.array()); + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, "dir1/" + bucket.getName() + "key1", CONTENT.array()); assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 2); assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 2); // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", null, CONTENT.array()); - TestDataUtil.createKey(bucket, "dir2/" + bucket.getName() + "key2", null, CONTENT.array()); + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, "dir2/" + bucket.getName() + "key2", CONTENT.array()); assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 3); assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 3); assertTableRowCount(deletedTable, countMap.get(deletedTable.getName()) + 1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java index 661307421c84..75ddd145d96d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java @@ -180,7 +180,7 @@ private static void writeKey(String keyName) throws IOException { TestDataUtil.createVolumeAndBucket(client, VOLUME_NAME, BUCKET_NAME); TestDataUtil.createKey( client.getObjectStore().getVolume(VOLUME_NAME).getBucket(BUCKET_NAME), - keyName, null, "test".getBytes(StandardCharsets.UTF_8)); + keyName, "test".getBytes(StandardCharsets.UTF_8)); } }