From 82e2ad4d84ffc5c64627d4c5d3fea3292b18dd10 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Tue, 29 Apr 2025 11:14:00 +0800 Subject: [PATCH 1/4] HDDS-12916. Support ETag in listObjects response --- .../hadoop/ozone/client/rpc/RpcClient.java | 9 +- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 89 ++++++++++++++++--- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index f1e26ed87e58..bcb08f0c3d42 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConsts.ETAG; import static org.apache.hadoop.ozone.OzoneConsts.MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD; import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH; @@ -1722,8 +1723,10 @@ public List listKeys(String volumeName, String bucketName, key.getCreationTime(), key.getModificationTime(), key.getReplicationConfig(), + Collections.singletonMap(ETAG, key.getETag()), key.isFile(), - key.getOwnerName())) + key.getOwnerName(), + Collections.emptyMap())) .collect(Collectors.toList()); } else { List keys = ozoneManagerClient.listKeys( @@ -1735,8 +1738,10 @@ public List listKeys(String volumeName, String bucketName, key.getCreationTime(), key.getModificationTime(), key.getReplicationConfig(), + key.getMetadata(), key.isFile(), - key.getOwnerName())) + key.getOwnerName(), + key.getTags())) .collect(Collectors.toList()); } } diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index dc8f098286b3..85f9d54dbd33 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -532,7 +532,7 @@ public void testGetObjectWithoutETag() throws Exception { } @Test - public void testListObjectsMany() { + public void testListObjectsMany() throws Exception { final String bucketName = getBucketName(); s3Client.createBucket(bucketName); final List keyNames = Arrays.asList( @@ -540,9 +540,32 @@ public void testListObjectsMany() { getKeyName("2"), getKeyName("3") ); + final List keyNamesWithoutETag = Arrays.asList( + getKeyName("4"), + getKeyName("5") + ); + final Map keyToEtag = new HashMap<>(); for (String keyName: keyNames) { - s3Client.putObject(bucketName, keyName, RandomStringUtils.secure().nextAlphanumeric(5)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, + RandomStringUtils.secure().nextAlphanumeric(5)); + keyToEtag.put(keyName, putObjectResult.getETag()); + } + try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(cluster.getConf())) { + ObjectStore store = ozoneClient.getObjectStore(); + + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + + for (String keyNameWithoutETag : keyNamesWithoutETag) { + byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); + try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, + valueBytes.length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + Collections.emptyMap())) { + out.write(valueBytes); + } + } } ListObjectsRequest listObjectsRequest = new ListObjectsRequest() @@ -554,24 +577,31 @@ public void testListObjectsMany() { assertEquals(listObjectsResponse.getObjectSummaries().stream() .map(S3ObjectSummary::getKey).collect(Collectors.toList()), keyNames.subList(0, 2)); + for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { + assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); + } assertTrue(listObjectsResponse.isTruncated()); + // Include both keys with and without ETag listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) - .withMaxKeys(2) + .withMaxKeys(5) .withMarker(listObjectsResponse.getNextMarker()); listObjectsResponse = s3Client.listObjects(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(1); + assertThat(listObjectsResponse.getObjectSummaries()).hasSize(3); assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(2, keyNames.size())); + assertEquals(keyNames.get(2), listObjectsResponse.getObjectSummaries().get(0).getKey()); + assertEquals(keyNamesWithoutETag.get(0), listObjectsResponse.getObjectSummaries().get(1).getKey()); + assertEquals(keyNamesWithoutETag.get(1), listObjectsResponse.getObjectSummaries().get(2).getKey()); + for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { + assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); + } assertFalse(listObjectsResponse.isTruncated()); } @Test - public void testListObjectsManyV2() { + public void testListObjectsManyV2() throws Exception { final String bucketName = getBucketName(); s3Client.createBucket(bucketName); final List keyNames = Arrays.asList( @@ -579,9 +609,32 @@ public void testListObjectsManyV2() { getKeyName("2"), getKeyName("3") ); + final List keyNamesWithoutETag = Arrays.asList( + getKeyName("4"), + getKeyName("5") + ); + final Map keyToEtag = new HashMap<>(); for (String keyName: keyNames) { - s3Client.putObject(bucketName, keyName, RandomStringUtils.secure().nextAlphanumeric(5)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, + RandomStringUtils.secure().nextAlphanumeric(5)); + keyToEtag.put(keyName, putObjectResult.getETag()); + } + try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(cluster.getConf())) { + ObjectStore store = ozoneClient.getObjectStore(); + + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + + for (String keyNameWithoutETag : keyNamesWithoutETag) { + byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); + try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, + valueBytes.length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + Collections.emptyMap())) { + out.write(valueBytes); + } + } } ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() @@ -593,19 +646,27 @@ public void testListObjectsManyV2() { assertEquals(listObjectsResponse.getObjectSummaries().stream() .map(S3ObjectSummary::getKey).collect(Collectors.toList()), keyNames.subList(0, 2)); + for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { + assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); + } assertTrue(listObjectsResponse.isTruncated()); + // Include both keys with and without ETag listObjectsRequest = new ListObjectsV2Request() .withBucketName(bucketName) - .withMaxKeys(2) + .withMaxKeys(5) .withContinuationToken(listObjectsResponse.getNextContinuationToken()); listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(1); + assertThat(listObjectsResponse.getObjectSummaries()).hasSize(3); assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(2, keyNames.size())); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertEquals(keyNames.get(2), listObjectsResponse.getObjectSummaries().get(0).getKey()); + assertEquals(keyNamesWithoutETag.get(0), listObjectsResponse.getObjectSummaries().get(1).getKey()); + assertEquals(keyNamesWithoutETag.get(1), listObjectsResponse.getObjectSummaries().get(2).getKey()); + for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { + assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); + } assertFalse(listObjectsResponse.isTruncated()); } From c62da7d093e85461f2678ecb67449778e1379836 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 3 May 2025 10:06:53 +0700 Subject: [PATCH 2/4] Extract listObjects test --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 144 +++++++----------- 1 file changed, 55 insertions(+), 89 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 85f9d54dbd33..6b97c8491ef6 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -533,75 +533,15 @@ public void testGetObjectWithoutETag() throws Exception { @Test public void testListObjectsMany() throws Exception { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - final List keyNames = Arrays.asList( - getKeyName("1"), - getKeyName("2"), - getKeyName("3") - ); - final List keyNamesWithoutETag = Arrays.asList( - getKeyName("4"), - getKeyName("5") - ); - - final Map keyToEtag = new HashMap<>(); - for (String keyName: keyNames) { - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, - RandomStringUtils.secure().nextAlphanumeric(5)); - keyToEtag.put(keyName, putObjectResult.getETag()); - } - try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(cluster.getConf())) { - ObjectStore store = ozoneClient.getObjectStore(); - - OzoneVolume volume = store.getS3Volume(); - OzoneBucket bucket = volume.getBucket(bucketName); - - for (String keyNameWithoutETag : keyNamesWithoutETag) { - byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); - try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, - valueBytes.length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), - Collections.emptyMap())) { - out.write(valueBytes); - } - } - } - - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(2); - ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(2); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(0, 2)); - for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { - assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); - } - assertTrue(listObjectsResponse.isTruncated()); - - - // Include both keys with and without ETag - listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(5) - .withMarker(listObjectsResponse.getNextMarker()); - listObjectsResponse = s3Client.listObjects(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(3); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(keyNames.get(2), listObjectsResponse.getObjectSummaries().get(0).getKey()); - assertEquals(keyNamesWithoutETag.get(0), listObjectsResponse.getObjectSummaries().get(1).getKey()); - assertEquals(keyNamesWithoutETag.get(1), listObjectsResponse.getObjectSummaries().get(2).getKey()); - for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { - assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); - } - assertFalse(listObjectsResponse.isTruncated()); + testListObjectsMany(false); } @Test public void testListObjectsManyV2() throws Exception { + testListObjectsMany(true); + } + + private void testListObjectsMany(boolean isListV2) throws Exception { final String bucketName = getBucketName(); s3Client.createBucket(bucketName); final List keyNames = Arrays.asList( @@ -637,37 +577,63 @@ public void testListObjectsManyV2() throws Exception { } } - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(2); - ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(2); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() + List objectSummaries; + String continuationToken; + if (isListV2) { + ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() + .withBucketName(bucketName) + .withMaxKeys(2); + ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.getNextContinuationToken(); + } else { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withMaxKeys(2); + ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.getNextMarker(); + } + assertThat(objectSummaries).hasSize(2); + assertEquals(objectSummaries.stream() .map(S3ObjectSummary::getKey).collect(Collectors.toList()), keyNames.subList(0, 2)); - for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { + for (S3ObjectSummary objectSummary : objectSummaries) { assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); } - assertTrue(listObjectsResponse.isTruncated()); - // Include both keys with and without ETag - listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(5) - .withContinuationToken(listObjectsResponse.getNextContinuationToken()); - listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(3); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(keyNames.get(2), listObjectsResponse.getObjectSummaries().get(0).getKey()); - assertEquals(keyNamesWithoutETag.get(0), listObjectsResponse.getObjectSummaries().get(1).getKey()); - assertEquals(keyNamesWithoutETag.get(1), listObjectsResponse.getObjectSummaries().get(2).getKey()); - for (S3ObjectSummary objectSummary : listObjectsResponse.getObjectSummaries()) { + if (isListV2) { + ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() + .withBucketName(bucketName) + .withMaxKeys(5) + .withContinuationToken(continuationToken); + ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertFalse(listObjectsResponse.isTruncated()); + } else { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withMaxKeys(5) + .withMarker(continuationToken); + ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertFalse(listObjectsResponse.isTruncated()); + } + + assertThat(objectSummaries).hasSize(3); + assertEquals(keyNames.get(2), objectSummaries.get(0).getKey()); + assertEquals(keyNamesWithoutETag.get(0), objectSummaries.get(1).getKey()); + assertEquals(keyNamesWithoutETag.get(1), objectSummaries.get(2).getKey()); + for (S3ObjectSummary objectSummary : objectSummaries) { assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); } - assertFalse(listObjectsResponse.isTruncated()); } @Test @@ -1022,7 +988,7 @@ private boolean isBucketEmpty(Bucket bucket) { } private String getBucketName() { - return getBucketName(null); + return getBucketName(""); } private String getBucketName(String suffix) { From 4061bb81d7f498904a7c0a924f8391d00b6d6e34 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 3 May 2025 10:28:18 +0700 Subject: [PATCH 3/4] Add tests for SDK V2 --- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 834580b6e8b7..21b4480c4976 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -20,7 +20,9 @@ import static org.apache.hadoop.ozone.OzoneConsts.MB; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; @@ -28,17 +30,30 @@ import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; import javax.xml.bind.DatatypeConverter; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.s3.S3ClientFactory; import org.apache.hadoop.ozone.s3.S3GatewayService; import org.apache.ozone.test.OzoneTestBase; @@ -57,7 +72,12 @@ import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.Tag; import software.amazon.awssdk.services.s3.model.Tagging; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -133,6 +153,116 @@ public void testPutObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); } + @Test + public void testListObjectsMany() throws Exception { + testListObjectsMany(false); + } + + @Test + public void testListObjectsManyV2() throws Exception { + testListObjectsMany(true); + } + + private void testListObjectsMany(boolean isListV2) throws Exception { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + final List keyNames = Arrays.asList( + getKeyName("1"), + getKeyName("2"), + getKeyName("3") + ); + final List keyNamesWithoutETag = Arrays.asList( + getKeyName("4"), + getKeyName("5") + ); + final Map keyToEtag = new HashMap<>(); + for (String keyName: keyNames) { + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName), + RequestBody.fromString(RandomStringUtils.secure().nextAlphanumeric(5))); + keyToEtag.put(keyName, putObjectResponse.eTag()); + } + try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(cluster.getConf())) { + ObjectStore store = ozoneClient.getObjectStore(); + + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + + for (String keyNameWithoutETag : keyNamesWithoutETag) { + byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); + try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, + valueBytes.length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + Collections.emptyMap())) { + out.write(valueBytes); + } + } + } + + List s3Objects; + String continuationToken; + if (isListV2) { + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(2) + .build(); + ListObjectsV2Response listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.nextContinuationToken(); + } else { + ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder() + .bucket(bucketName) + .maxKeys(2) + .build(); + ListObjectsResponse listObjectsResponse = s3Client.listObjects(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.nextMarker(); + } + assertThat(s3Objects).hasSize(2); + assertEquals(s3Objects.stream() + .map(S3Object::key).collect(Collectors.toList()), + keyNames.subList(0, 2)); + for (S3Object s3Object : s3Objects) { + assertEquals(keyToEtag.get(s3Object.key()), s3Object.eTag()); + } + + // Include both keys with and without ETag + if (isListV2) { + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(5) + .continuationToken(continuationToken) + .build(); + ListObjectsV2Response listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertFalse(listObjectsResponse.isTruncated()); + } else { + ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder() + .bucket(bucketName) + .maxKeys(5) + .marker(continuationToken) + .build(); + ListObjectsResponse listObjectsResponse = s3Client.listObjects(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertFalse(listObjectsResponse.isTruncated()); + } + + assertThat(s3Objects).hasSize(3); + assertEquals(keyNames.get(2), s3Objects.get(0).key()); + assertEquals(keyNamesWithoutETag.get(0), s3Objects.get(1).key()); + assertEquals(keyNamesWithoutETag.get(1), s3Objects.get(2).key()); + for (S3Object s3Object : s3Objects) { + assertEquals(keyToEtag.get(s3Object.key()), s3Object.eTag()); + } + } + @Test public void testCopyObject() { final String sourceBucketName = getBucketName("source"); From 251fbfb5bbd77e85340599841aa86282aaec87c2 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 3 May 2025 17:12:54 +0700 Subject: [PATCH 4/4] Use Cluster#newClient and fix null in bucket and key names suffix --- .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 11 ++++------- .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 7 +++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 6b97c8491ef6..cee69f0f3603 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -98,7 +98,6 @@ import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -373,9 +372,8 @@ public void testPutDoubleSlashPrefixObject() throws IOException { final String bucketName = getBucketName(); final String keyName = "//dir1"; final String content = "bar"; - OzoneConfiguration conf = cluster.getConf(); // Create a FSO bucket for test - try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { + try (OzoneClient ozoneClient = cluster.newClient()) { ObjectStore store = ozoneClient.getObjectStore(); OzoneVolume volume = store.getS3Volume(); OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder() @@ -502,8 +500,7 @@ public void testGetObjectWithoutETag() throws Exception { String value = "sample value"; byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); - OzoneConfiguration conf = cluster.getConf(); - try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { + try (OzoneClient ozoneClient = cluster.newClient()) { ObjectStore store = ozoneClient.getObjectStore(); OzoneVolume volume = store.getS3Volume(); @@ -560,7 +557,7 @@ private void testListObjectsMany(boolean isListV2) throws Exception { RandomStringUtils.secure().nextAlphanumeric(5)); keyToEtag.put(keyName, putObjectResult.getETag()); } - try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(cluster.getConf())) { + try (OzoneClient ozoneClient = cluster.newClient()) { ObjectStore store = ozoneClient.getObjectStore(); OzoneVolume volume = store.getS3Volume(); @@ -996,7 +993,7 @@ private String getBucketName(String suffix) { } private String getKeyName() { - return getKeyName(null); + return getKeyName(""); } private String getKeyName(String suffix) { diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 21b4480c4976..7ef1342886c6 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -51,7 +51,6 @@ import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.s3.S3ClientFactory; @@ -183,7 +182,7 @@ private void testListObjectsMany(boolean isListV2) throws Exception { RequestBody.fromString(RandomStringUtils.secure().nextAlphanumeric(5))); keyToEtag.put(keyName, putObjectResponse.eTag()); } - try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(cluster.getConf())) { + try (OzoneClient ozoneClient = cluster.newClient()) { ObjectStore store = ozoneClient.getObjectStore(); OzoneVolume volume = store.getS3Volume(); @@ -326,7 +325,7 @@ public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception } private String getBucketName() { - return getBucketName(null); + return getBucketName(""); } private String getBucketName(String suffix) { @@ -334,7 +333,7 @@ private String getBucketName(String suffix) { } private String getKeyName() { - return getKeyName(null); + return getKeyName(""); } private String getKeyName(String suffix) {