From f78a11acfeceed0a8a7c7b54989506b06c3f273b Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 2 Jun 2025 09:44:02 +0530 Subject: [PATCH 1/6] HDDS-13139. Introduce bucket layout flag in freon rk command --- .../hadoop/ozone/freon/RandomKeyGenerator.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java index 3c2e364668e5..d03948d520c6 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hdds.conf.StorageSize; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -67,6 +68,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.util.ShutdownHookManager; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; @@ -191,6 +193,12 @@ public final class RandomKeyGenerator implements Callable, FreonSubcommand ) private boolean cleanObjects = false; + @Option( + names = "--bucket-layout", + description = "Specifies the bucket layout (e.g., FILE_SYSTEM_OPTIMIZED, OBJECT_STORE, LEGACY)." + ) + private BucketLayout bucketLayout; + private ReplicationConfig replicationConfig; private int threadPoolSize; @@ -757,7 +765,14 @@ private boolean createBucket(int globalBucketNumber) { .createActivatedSpan("createBucket")) { long start = System.nanoTime(); - volume.createBucket(bucketName); + if (bucketLayout != null) { + BucketArgs bucketArgs = BucketArgs.newBuilder() + .setBucketLayout(bucketLayout) + .build(); + volume.createBucket(bucketName, bucketArgs); + } else { + volume.createBucket(bucketName); + } long bucketCreationDuration = System.nanoTime() - start; histograms.get(FreonOps.BUCKET_CREATE.ordinal()) .update(bucketCreationDuration); From 3f17b241f1371c5bd0554fa47144704d71dd95f1 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 2 Jun 2025 16:33:02 +0530 Subject: [PATCH 2/6] Added test for checking bucket layout option --- .../ozone/freon/RandomKeyGenerator.java | 11 ++++++++++ .../ozone/freon/TestRandomKeyGenerator.java | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java index d03948d520c6..5fdda9c58ef6 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java @@ -654,6 +654,17 @@ long getUnsuccessfulValidationCount() { return validateWrites ? writeValidationFailureCount.get() : 0; } + /** + * Returns the bucket for the given bucket number. + * + * @param bucketNumber bucket index + * @return corresponding OzoneBucket instance + */ + @VisibleForTesting + OzoneBucket getBucketForTesting(Integer bucketNumber) { + return waitUntilAddedToMap(buckets, bucketNumber); + } + /** * Wrapper to hold ozone keyValidate entry. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java index e7a5bf2f7906..4862839138d4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java @@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.Test; import picocli.CommandLine; @@ -185,4 +187,24 @@ void cleanObjectsTest() { assertEquals(2, randomKeyGenerator.getNumberOfVolumesCleaned()); assertEquals(6, randomKeyGenerator.getNumberOfBucketsCleaned()); } + + @Test + void testBucketLayoutOption() { + RandomKeyGenerator randomKeyGenerator = + new RandomKeyGenerator(cluster().getConf()); + CommandLine cmd = new CommandLine(randomKeyGenerator); + cmd.execute("--num-of-volumes", "1", + "--num-of-buckets", "1", + "--num-of-keys", "2", + "--bucket-layout", "OBJECT_STORE" + ); + + assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated()); + assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated()); + assertEquals(2, randomKeyGenerator.getNumberOfKeysAdded()); + + // Fetch the bucket and check its layout + OzoneBucket bucket = randomKeyGenerator.getBucketForTesting(0); + assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout()); + } } From d289f80e20b7b8c93305096e0677d38874f9ed27 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 2 Jun 2025 17:23:08 +0530 Subject: [PATCH 3/6] Added bucket map size check --- .../hadoop/ozone/freon/RandomKeyGenerator.java | 12 +++++++++++- .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java index 5fdda9c58ef6..8bf60189351c 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java @@ -662,7 +662,17 @@ long getUnsuccessfulValidationCount() { */ @VisibleForTesting OzoneBucket getBucketForTesting(Integer bucketNumber) { - return waitUntilAddedToMap(buckets, bucketNumber); + return getBucket(bucketNumber); + } + + /** + * Returns the current size of the buckets map. + * + * @return number of buckets created and added to the map + */ + @VisibleForTesting + int getBucketMapSizeForTesting() { + return buckets.size(); } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java index 4862839138d4..9bc5990f18ec 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java @@ -202,6 +202,7 @@ void testBucketLayoutOption() { assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated()); assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated()); assertEquals(2, randomKeyGenerator.getNumberOfKeysAdded()); + assertEquals(1, randomKeyGenerator.getBucketMapSizeForTesting()); // Fetch the bucket and check its layout OzoneBucket bucket = randomKeyGenerator.getBucketForTesting(0); From d773c95ebc1fe067eba2cb9b8a38a6517c74e1fb Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 2 Jun 2025 17:38:25 +0530 Subject: [PATCH 4/6] Using existing getBucket method in test --- .../hadoop/ozone/freon/RandomKeyGenerator.java | 13 ++----------- .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java index 8bf60189351c..7a14ed4cb210 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java @@ -654,16 +654,6 @@ long getUnsuccessfulValidationCount() { return validateWrites ? writeValidationFailureCount.get() : 0; } - /** - * Returns the bucket for the given bucket number. - * - * @param bucketNumber bucket index - * @return corresponding OzoneBucket instance - */ - @VisibleForTesting - OzoneBucket getBucketForTesting(Integer bucketNumber) { - return getBucket(bucketNumber); - } /** * Returns the current size of the buckets map. @@ -916,7 +906,8 @@ private OzoneVolume getVolume(Integer volumeNumber) { return waitUntilAddedToMap(volumes, volumeNumber); } - private OzoneBucket getBucket(Integer bucketNumber) { + @VisibleForTesting + public OzoneBucket getBucket(Integer bucketNumber) { return waitUntilAddedToMap(buckets, bucketNumber); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java index 9bc5990f18ec..88ec25363204 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java @@ -205,7 +205,7 @@ void testBucketLayoutOption() { assertEquals(1, randomKeyGenerator.getBucketMapSizeForTesting()); // Fetch the bucket and check its layout - OzoneBucket bucket = randomKeyGenerator.getBucketForTesting(0); + OzoneBucket bucket = randomKeyGenerator.getBucket(0); assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout()); } } From de6097a8c70f75762409ee625131b2e1a1e78e87 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 2 Jun 2025 17:48:51 +0530 Subject: [PATCH 5/6] Fixed access modifier --- .../java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java index 7a14ed4cb210..e2fe1ddfbba5 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java @@ -654,7 +654,6 @@ long getUnsuccessfulValidationCount() { return validateWrites ? writeValidationFailureCount.get() : 0; } - /** * Returns the current size of the buckets map. * @@ -907,7 +906,7 @@ private OzoneVolume getVolume(Integer volumeNumber) { } @VisibleForTesting - public OzoneBucket getBucket(Integer bucketNumber) { + OzoneBucket getBucket(Integer bucketNumber) { return waitUntilAddedToMap(buckets, bucketNumber); } From 8a6e157b66d2a5edad38a13628051e879894f2f1 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Tue, 3 Jun 2025 11:08:09 +0530 Subject: [PATCH 6/6] Updated method name --- .../java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java | 2 +- .../org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java index e2fe1ddfbba5..e73ddc77f2b4 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java @@ -660,7 +660,7 @@ long getUnsuccessfulValidationCount() { * @return number of buckets created and added to the map */ @VisibleForTesting - int getBucketMapSizeForTesting() { + int getBucketMapSize() { return buckets.size(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java index 88ec25363204..657ece8be81b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java @@ -202,7 +202,7 @@ void testBucketLayoutOption() { assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated()); assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated()); assertEquals(2, randomKeyGenerator.getNumberOfKeysAdded()); - assertEquals(1, randomKeyGenerator.getBucketMapSizeForTesting()); + assertEquals(1, randomKeyGenerator.getBucketMapSize()); // Fetch the bucket and check its layout OzoneBucket bucket = randomKeyGenerator.getBucket(0);