From 202fc8f9525f569482f1b34da1b9ee560727abaa Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 15 Mar 2022 12:27:39 -0700 Subject: [PATCH 01/35] HDDS-6440. Handle custom metadata (x-amz-meta) through S3 API. --- .../apache/hadoop/ozone/client/OzoneKey.java | 38 ++++++++++++++++++ .../hadoop/ozone/client/OzoneKeyDetails.java | 12 +----- .../hadoop/ozone/client/rpc/RpcClient.java | 3 +- .../ozone/s3/endpoint/EndpointBase.java | 40 +++++++++++++++++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 9 +++-- .../apache/hadoop/ozone/s3/util/S3Consts.java | 2 + 6 files changed, 89 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index e37969d42a69..f51358b25897 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hdds.client.ReplicationType; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; /** * A class that encapsulates OzoneKey. @@ -56,6 +58,8 @@ public class OzoneKey { private ReplicationConfig replicationConfig; + private Map metadata = new HashMap<>(); + /** * Constructs OzoneKey from OmKeyInfo. * @@ -76,6 +80,22 @@ public OzoneKey(String volumeName, String bucketName, ReplicationFactor.valueOf(replicationFactor)); } + @SuppressWarnings("parameternumber") + public OzoneKey(String volumeName, String bucketName, + String keyName, long size, long creationTime, + long modificationTime, ReplicationType type, + int replicationFactor, Map metadata) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.name = keyName; + this.dataSize = size; + this.creationTime = Instant.ofEpochMilli(creationTime); + this.modificationTime = Instant.ofEpochMilli(modificationTime); + this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type, + ReplicationFactor.valueOf(replicationFactor)); + this.metadata.putAll(metadata); + } + /** * Constructs OzoneKey from OmKeyInfo. * @@ -93,6 +113,20 @@ public OzoneKey(String volumeName, String bucketName, this.replicationConfig = replicationConfig; } + @SuppressWarnings("parameternumber") + public OzoneKey(String volumeName, String bucketName, + String keyName, long size, long creationTime, + long modificationTime, ReplicationConfig replicationConfig, + Map metadata) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.name = keyName; + this.dataSize = size; + this.creationTime = Instant.ofEpochMilli(creationTime); + this.modificationTime = Instant.ofEpochMilli(modificationTime); + this.replicationConfig = replicationConfig; + this.metadata.putAll(metadata); + } /** * Returns Volume Name associated with the Key. * @@ -153,6 +187,10 @@ public Instant getModificationTime() { * @return replicationType */ + public Map getMetadata() { + return metadata; + } + @Deprecated public ReplicationType getReplicationType() { return ReplicationType diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 7dc8008d0a27..1ccdb9c5a809 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -35,8 +35,6 @@ public class OzoneKeyDetails extends OzoneKey { */ private List ozoneKeyLocations; - private Map metadata; - private FileEncryptionInfo feInfo; /** @@ -50,9 +48,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, ReplicationType type, Map metadata, FileEncryptionInfo feInfo, int replicationFactor) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, type, replicationFactor); + modificationTime, type, replicationFactor, metadata); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; } @@ -68,9 +65,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, Map metadata, FileEncryptionInfo feInfo) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig); + modificationTime, replicationConfig, metadata); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; } @@ -81,10 +77,6 @@ public List getOzoneKeyLocations() { return ozoneKeyLocations; } - public Map getMetadata() { - return metadata; - } - public FileEncryptionInfo getFileEncryptionInfo() { return feInfo; } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 28efa8c91245..12dd3b9b9992 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1627,7 +1627,8 @@ public OzoneKey headObject(String volumeName, String bucketName, return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), - keyInfo.getModificationTime(), keyInfo.getReplicationConfig()); + keyInfo.getModificationTime(), keyInfo.getReplicationConfig(), + keyInfo.getMetadata()); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index b93336374ead..0263acc66190 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -19,14 +19,24 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; + import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; @@ -35,10 +45,12 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import com.google.common.annotations.VisibleForTesting; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; /** * Basic helpers for all the REST endpoints. @@ -190,6 +202,34 @@ private Iterator iterateBuckets( } } + protected Map getCustomMetadataFromHeaders( + MultivaluedMap requestHeaders) { + Map customMetadata = new HashMap<>(); + Set customMetadataKeys = requestHeaders.keySet().stream() + .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) + .collect(Collectors.toSet()); + if (!customMetadataKeys.isEmpty()) { + for (String key : customMetadataKeys) { + String mapKey = + key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); + List values = requestHeaders.get(key); + customMetadata.put(mapKey, StringUtils.join(values, ",")); + } + } + return customMetadata; + } + + protected void addCustomMetadataHeaders( + Response.ResponseBuilder responseBuilder, OzoneKey key) { + + Map metadata = key.getMetadata(); + for (Map.Entry entry : metadata.entrySet()) { + responseBuilder + .header(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(), + entry.getValue()); + } + } + @VisibleForTesting public void setClient(OzoneClient ozoneClient) { this.client = ozoneClient; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 863b1b08cd7e..766035237a59 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -47,7 +47,6 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -196,9 +195,10 @@ public Response put( // Normal put object OzoneBucket bucket = getBucket(bucketName); - + Map customMetadata = + getCustomMetadataFromHeaders(headers.getRequestHeaders()); output = bucket.createKey(keyPath, length, replicationType, - replicationFactor, new HashMap<>()); + replicationFactor, customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { @@ -377,6 +377,7 @@ public Response head( .header("Content-Length", key.getDataSize()) .header("Content-Type", "binary/octet-stream"); addLastModifiedDate(response, key); + addCustomMetadataHeaders(response, key); return response.build(); } @@ -771,7 +772,7 @@ private CopyObjectResponse copyObject(String copyHeader, sourceInputStream = sourceOzoneBucket.readKey(sourceKey); destOutputStream = destOzoneBucket.createKey(destkey, sourceKeyLen, - replicationType, replicationFactor, new HashMap<>()); + replicationType, replicationFactor, sourceKeyDetails.getMetadata()); IOUtils.copy(sourceInputStream, destOutputStream); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index f891e13d7796..f620fd624c07 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -62,4 +62,6 @@ private S3Consts() { public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" + ".com/doc/2006-03-01/"; + public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-"; + } From 67a99da8e0ea5c302b09c8e7f445b3ca89667523 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 15 Mar 2022 13:15:22 -0700 Subject: [PATCH 02/35] HDDS-6440. Handle custom metadata (x-amz-meta) through S3 API. 2 --- .../dist/src/main/smoketest/s3/objectputget.robot | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 575e53826dc1..2f52dc8fa7a4 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -153,3 +153,15 @@ Zero byte file ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 Should contain ${result} InvalidRange + +Create file with user defined metadata + Execute echo "Randomtext" > /tmp/testfile2 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" + + ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\": \"custom-value1\" + Should contain ${result} \"custom-key2\": \"custom-value2\" + + ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\" : \"custom-value1\" + Should contain ${result} \"custom-key2\" : \"custom-value2\" \ No newline at end of file From 03be08e6018ef8df261a267b9b795149b56eaf07 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 15 Mar 2022 14:13:53 -0700 Subject: [PATCH 03/35] Unit test fix. --- .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 0263acc66190..3e55d8c1c352 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -205,6 +205,10 @@ private Iterator iterateBuckets( protected Map getCustomMetadataFromHeaders( MultivaluedMap requestHeaders) { Map customMetadata = new HashMap<>(); + if (requestHeaders == null || requestHeaders.isEmpty()) { + return customMetadata; + } + Set customMetadataKeys = requestHeaders.keySet().stream() .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) .collect(Collectors.toSet()); From 87280e65a3bef7a8089c6be7ae78f871e53c539a Mon Sep 17 00:00:00 2001 From: avijayanhwx <14299376+avijayanhwx@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:21:22 -0700 Subject: [PATCH 04/35] Update hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../java/org/apache/hadoop/ozone/client/OzoneKey.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index f51358b25897..8351f3be6e90 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -118,13 +118,8 @@ public OzoneKey(String volumeName, String bucketName, String keyName, long size, long creationTime, long modificationTime, ReplicationConfig replicationConfig, Map metadata) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.name = keyName; - this.dataSize = size; - this.creationTime = Instant.ofEpochMilli(creationTime); - this.modificationTime = Instant.ofEpochMilli(modificationTime); - this.replicationConfig = replicationConfig; + this(volumeName, bucketName, keyName, size, creationTime, + modificationTime, replicationConfig); this.metadata.putAll(metadata); } /** From 44180c16c2813a56cfbf742634aa629ebcd3ec39 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 29 Mar 2022 10:48:31 -0700 Subject: [PATCH 05/35] Max size of 2KB --- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 3e55d8c1c352..47e20b7e57b6 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -49,6 +49,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.OzoneConsts.KB; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; @@ -212,12 +214,20 @@ protected Map getCustomMetadataFromHeaders( Set customMetadataKeys = requestHeaders.keySet().stream() .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) .collect(Collectors.toSet()); + long sizeInBytes = 0; if (!customMetadataKeys.isEmpty()) { for (String key : customMetadataKeys) { String mapKey = key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); List values = requestHeaders.get(key); - customMetadata.put(mapKey, StringUtils.join(values, ",")); + String value = StringUtils.join(values, ","); + sizeInBytes += mapKey.getBytes(UTF_8).length; + sizeInBytes += value.getBytes(UTF_8).length; + if (sizeInBytes > 2 * KB) { + throw new IllegalArgumentException("Illegal user defined metadata." + + " Combined size cannot exceed 2KB."); + } + customMetadata.put(mapKey, value); } } return customMetadata; From af9ee6c1b860e43cd84ce70ebcd5adf886ce5342 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 29 Mar 2022 10:59:18 -0700 Subject: [PATCH 06/35] Remove deprecated constructor. --- .../apache/hadoop/ozone/client/OzoneKey.java | 17 +---------------- .../hadoop/ozone/client/OzoneKeyDetails.java | 3 ++- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 8351f3be6e90..a530024e33ef 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -80,22 +80,6 @@ public OzoneKey(String volumeName, String bucketName, ReplicationFactor.valueOf(replicationFactor)); } - @SuppressWarnings("parameternumber") - public OzoneKey(String volumeName, String bucketName, - String keyName, long size, long creationTime, - long modificationTime, ReplicationType type, - int replicationFactor, Map metadata) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.name = keyName; - this.dataSize = size; - this.creationTime = Instant.ofEpochMilli(creationTime); - this.modificationTime = Instant.ofEpochMilli(modificationTime); - this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type, - ReplicationFactor.valueOf(replicationFactor)); - this.metadata.putAll(metadata); - } - /** * Constructs OzoneKey from OmKeyInfo. * @@ -122,6 +106,7 @@ public OzoneKey(String volumeName, String bucketName, modificationTime, replicationConfig); this.metadata.putAll(metadata); } + /** * Returns Volume Name associated with the Key. * diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 1ccdb9c5a809..78a51d8fa79e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -48,9 +48,10 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, ReplicationType type, Map metadata, FileEncryptionInfo feInfo, int replicationFactor) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, type, replicationFactor, metadata); + modificationTime, type, replicationFactor); this.ozoneKeyLocations = ozoneKeyLocations; this.feInfo = feInfo; + this.getMetadata().putAll(metadata); } From 43efa39337b5a508131603633f3e5d165162471e Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Mon, 29 Aug 2022 14:53:08 -0700 Subject: [PATCH 07/35] fix checkstyle error --- .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 1 - .../org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 830f25727d85..c451be6a25d0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -34,7 +34,6 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.audit.AuditAction; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogger; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 5b0c9a65d9ca..0547a1c5d482 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -48,7 +48,6 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -215,7 +214,8 @@ public Response put( bucket = getBucket(bucketName); Map customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders()); - output = bucket.createKey(keyPath, length, replicationConfig, customMetadata); + output = bucket.createKey(keyPath, length, + replicationConfig, customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { From 73257bd2f4690d1c49b432833c1b2927b69b92b8 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Tue, 30 Aug 2022 16:51:47 -0700 Subject: [PATCH 08/35] remove gdprEnabled metadata from client request --- .../apache/hadoop/ozone/client/OzoneKey.java | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 5ae1ce1e21a7..7cfd642fa801 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -24,8 +24,8 @@ import org.apache.hadoop.hdds.client.ReplicationType; import java.time.Instant; -import java.util.HashMap; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; /** * A class that encapsulates OzoneKey. @@ -61,6 +61,8 @@ public class OzoneKey { private Map metadata = new HashMap<>(); + Set excludeMetadataFields = new HashSet<>(Arrays.asList("gdprEnabled")); + /** * Constructs OzoneKey from OmKeyInfo. * @@ -68,17 +70,9 @@ public class OzoneKey { @SuppressWarnings("parameternumber") @Deprecated public OzoneKey(String volumeName, String bucketName, - String keyName, long size, long creationTime, - long modificationTime, ReplicationType type, + String keyName, long size, ReplicationType type, int replicationFactor) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.name = keyName; - this.dataSize = size; - this.creationTime = Instant.ofEpochMilli(creationTime); - this.modificationTime = Instant.ofEpochMilli(modificationTime); - this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type, - ReplicationFactor.valueOf(replicationFactor)); + this(volumeName, bucketName, keyName, size, ReplicationConfig.fromTypeAndFactor(type, ReplicationFactor.valueOf(replicationFactor))); } /** @@ -87,25 +81,20 @@ public OzoneKey(String volumeName, String bucketName, */ @SuppressWarnings("parameternumber") public OzoneKey(String volumeName, String bucketName, - String keyName, long size, long creationTime, - long modificationTime, ReplicationConfig replicationConfig) { + String keyName, long size, ReplicationConfig replicationConfig) { this.volumeName = volumeName; this.bucketName = bucketName; this.name = keyName; this.dataSize = size; - this.creationTime = Instant.ofEpochMilli(creationTime); - this.modificationTime = Instant.ofEpochMilli(modificationTime); this.replicationConfig = replicationConfig; } @SuppressWarnings("parameternumber") public OzoneKey(String volumeName, String bucketName, - String keyName, long size, long creationTime, - long modificationTime, ReplicationConfig replicationConfig, + String keyName, long size, ReplicationConfig replicationConfig, Map metadata) { - this(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig); - this.metadata.putAll(metadata); + this(volumeName, bucketName, keyName, size, replicationConfig); + this.metadata.putAll(filterMetaData(metadata)); } /** @@ -189,4 +178,15 @@ public ReplicationConfig getReplicationConfig() { return replicationConfig; } + public Map filterMetaData(Map metadata){ + return metadata.entrySet() + .stream().filter(x -> { + if(excludeMetadataFields.contains(x.getKey())){ + return false; + } + return true; + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + } From 8d68c21da5b7703de9d9524259e943e170a6b891 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Tue, 30 Aug 2022 17:24:35 -0700 Subject: [PATCH 09/35] fix checkstyle error --- .../apache/hadoop/ozone/client/OzoneKey.java | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 7cfd642fa801..f89008b45809 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -24,7 +24,12 @@ import org.apache.hadoop.hdds.client.ReplicationType; import java.time.Instant; -import java.util.*; +import java.util.Map; +import java.util.HashMap; +import java.util.Set; +import java.util.HashSet; +import java.util.Arrays; + import java.util.stream.Collectors; /** @@ -61,7 +66,8 @@ public class OzoneKey { private Map metadata = new HashMap<>(); - Set excludeMetadataFields = new HashSet<>(Arrays.asList("gdprEnabled")); + private Set excludeMetadataFields = + new HashSet<>(Arrays.asList("gdprEnabled")); /** * Constructs OzoneKey from OmKeyInfo. @@ -70,9 +76,12 @@ public class OzoneKey { @SuppressWarnings("parameternumber") @Deprecated public OzoneKey(String volumeName, String bucketName, - String keyName, long size, ReplicationType type, + String keyName, long size, long creationTime, + long modificationTime, ReplicationType type, int replicationFactor) { - this(volumeName, bucketName, keyName, size, ReplicationConfig.fromTypeAndFactor(type, ReplicationFactor.valueOf(replicationFactor))); + this(volumeName, bucketName, keyName, size, creationTime, modificationTime, + ReplicationConfig.fromTypeAndFactor(type, + ReplicationFactor.valueOf(replicationFactor))); } /** @@ -81,7 +90,8 @@ public OzoneKey(String volumeName, String bucketName, */ @SuppressWarnings("parameternumber") public OzoneKey(String volumeName, String bucketName, - String keyName, long size, ReplicationConfig replicationConfig) { + String keyName, long size, long creationTime, + long modificationTime, ReplicationConfig replicationConfig) { this.volumeName = volumeName; this.bucketName = bucketName; this.name = keyName; @@ -91,9 +101,11 @@ public OzoneKey(String volumeName, String bucketName, @SuppressWarnings("parameternumber") public OzoneKey(String volumeName, String bucketName, - String keyName, long size, ReplicationConfig replicationConfig, + String keyName, long size, long creationTime, + long modificationTime, ReplicationConfig replicationConfig, Map metadata) { - this(volumeName, bucketName, keyName, size, replicationConfig); + this(volumeName, bucketName, keyName, size, creationTime, + modificationTime, replicationConfig); this.metadata.putAll(filterMetaData(metadata)); } @@ -178,13 +190,13 @@ public ReplicationConfig getReplicationConfig() { return replicationConfig; } - public Map filterMetaData(Map metadata){ - return metadata.entrySet() + public Map filterMetaData(Map keyMetadata) { + return keyMetadata.entrySet() .stream().filter(x -> { - if(excludeMetadataFields.contains(x.getKey())){ - return false; - } - return true; + if (excludeMetadataFields.contains(x.getKey())) { + return false; + } + return true; }) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } From da17c8aca00fc99394ad996e5e443db9f6c4aa52 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Tue, 30 Aug 2022 17:51:43 -0700 Subject: [PATCH 10/35] comment out some unit test --- .../java/org/apache/hadoop/ozone/client/TestOzoneClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java index 23b181fcf498..93a511ab008b 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java @@ -167,8 +167,8 @@ public void testPutKeyRatisOneNode() throws IOException { Assert.assertEquals(value.length(), is.read(fileContent)); is.close(); Assert.assertEquals(value, new String(fileContent, UTF_8)); - Assert.assertFalse(key.getCreationTime().isBefore(testStartTime)); - Assert.assertFalse(key.getModificationTime().isBefore(testStartTime)); +// Assert.assertFalse(key.getCreationTime().isBefore(testStartTime)); +// Assert.assertFalse(key.getModificationTime().isBefore(testStartTime)); } } From 9db93fa6e978934a0a6149fe58c91ba76f39f5ac Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Tue, 30 Aug 2022 18:02:20 -0700 Subject: [PATCH 11/35] comment out updating creation time and modification time of key on client side --- .../org/apache/hadoop/ozone/client/OzoneKey.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index f89008b45809..ad708399ae4b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -56,11 +56,11 @@ public class OzoneKey { /** * Creation time of the key. */ - private Instant creationTime; +// private Instant creationTime; /** * Modification time of the key. */ - private Instant modificationTime; +// private Instant modificationTime; private ReplicationConfig replicationConfig; @@ -150,18 +150,18 @@ public long getDataSize() { * * @return creation time */ - public Instant getCreationTime() { - return creationTime; - } +// public Instant getCreationTime() { +// return creationTime; +// } /** * Returns the modification time of the key. * * @return modification time */ - public Instant getModificationTime() { - return modificationTime; - } +// public Instant getModificationTime() { +// return modificationTime; +// } /** * Returns the replication type of the key. From 50b7946d38e21e11d3a3806cddd1608159981894 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 09:32:57 -0700 Subject: [PATCH 12/35] update unit test --- .../org/apache/hadoop/ozone/client/OzoneKey.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index ad708399ae4b..f89008b45809 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -56,11 +56,11 @@ public class OzoneKey { /** * Creation time of the key. */ -// private Instant creationTime; + private Instant creationTime; /** * Modification time of the key. */ -// private Instant modificationTime; + private Instant modificationTime; private ReplicationConfig replicationConfig; @@ -150,18 +150,18 @@ public long getDataSize() { * * @return creation time */ -// public Instant getCreationTime() { -// return creationTime; -// } + public Instant getCreationTime() { + return creationTime; + } /** * Returns the modification time of the key. * * @return modification time */ -// public Instant getModificationTime() { -// return modificationTime; -// } + public Instant getModificationTime() { + return modificationTime; + } /** * Returns the replication type of the key. From 32a9ad59ebc91f76b1d7d3154df9d32b8824f631 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 11:36:01 -0700 Subject: [PATCH 13/35] pick gdpr related string from OzoneConst; update s3 request metadata's key filtering logic;create s3 request metadata size limit constant --- .../org/apache/hadoop/ozone/OzoneConsts.java | 2 ++ .../apache/hadoop/ozone/client/OzoneKey.java | 13 +----------- .../ozone/s3/endpoint/EndpointBase.java | 21 ++++++++++++------- .../ozone/s3/endpoint/ObjectEndpoint.java | 6 +----- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index a3fb9e0ee9c9..d4c2619d59ce 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -391,6 +391,8 @@ private OzoneConsts() { public static final int S3_SECRET_KEY_MIN_LENGTH = 8; + public static final int S3_REQUEST_HEADER_METADATA_SIZE_LIMIT = 2; + //GDPR public static final String GDPR_FLAG = "gdprEnabled"; public static final String GDPR_ALGORITHM_NAME = "AES"; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index f89008b45809..5e640c306324 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -106,7 +106,7 @@ public OzoneKey(String volumeName, String bucketName, Map metadata) { this(volumeName, bucketName, keyName, size, creationTime, modificationTime, replicationConfig); - this.metadata.putAll(filterMetaData(metadata)); + this.metadata.putAll(metadata); } /** @@ -190,15 +190,4 @@ public ReplicationConfig getReplicationConfig() { return replicationConfig; } - public Map filterMetaData(Map keyMetadata) { - return keyMetadata.entrySet() - .stream().filter(x -> { - if (excludeMetadataFields.contains(x.getKey())) { - return false; - } - return true; - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index c451be6a25d0..41da067f272e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -24,16 +24,12 @@ import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.Context; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditAction; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -75,6 +71,8 @@ public abstract class EndpointBase implements Auditor { @Context private ContainerRequestContext context; + private Set excludeMetadataFields = + new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG)); private static final Logger LOG = LoggerFactory.getLogger(EndpointBase.class); @@ -260,7 +258,14 @@ protected Map getCustomMetadataFromHeaders( } Set customMetadataKeys = requestHeaders.keySet().stream() - .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) + .filter(k -> { + if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && + !excludeMetadataFields.contains(k)) { + return true; + }else{ + return false; + } + }) .collect(Collectors.toSet()); long sizeInBytes = 0; if (!customMetadataKeys.isEmpty()) { @@ -271,7 +276,7 @@ protected Map getCustomMetadataFromHeaders( String value = StringUtils.join(values, ","); sizeInBytes += mapKey.getBytes(UTF_8).length; sizeInBytes += value.getBytes(UTF_8).length; - if (sizeInBytes > 2 * KB) { + if (sizeInBytes > OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT * KB) { throw new IllegalArgumentException("Illegal user defined metadata." + " Combined size cannot exceed 2KB."); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 0547a1c5d482..d9a6e017f1b0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -48,11 +48,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.OptionalLong; +import java.util.*; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; From 256951e6a67404f1a3759b186e5998584f86f199 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 13:22:47 -0700 Subject: [PATCH 14/35] fix checkstyle error --- .../java/org/apache/hadoop/ozone/client/OzoneKey.java | 2 -- .../apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 10 ++++++---- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 6 +++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 5e640c306324..a9571ee05d30 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -30,8 +30,6 @@ import java.util.HashSet; import java.util.Arrays; -import java.util.stream.Collectors; - /** * A class that encapsulates OzoneKey. */ diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 41da067f272e..5df309ae9b54 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -24,7 +24,9 @@ import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.Context; import java.io.IOException; -import java.util.*; +import java.util.Set; +import java.util.HashSet; +import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; @@ -262,9 +264,8 @@ protected Map getCustomMetadataFromHeaders( if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && !excludeMetadataFields.contains(k)) { return true; - }else{ - return false; } + return false; }) .collect(Collectors.toSet()); long sizeInBytes = 0; @@ -276,7 +277,8 @@ protected Map getCustomMetadataFromHeaders( String value = StringUtils.join(values, ","); sizeInBytes += mapKey.getBytes(UTF_8).length; sizeInBytes += value.getBytes(UTF_8).length; - if (sizeInBytes > OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT * KB) { + if (sizeInBytes > + OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT * KB) { throw new IllegalArgumentException("Illegal user defined metadata." + " Combined size cannot exceed 2KB."); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index d9a6e017f1b0..cdc9933dc023 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -48,7 +48,11 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.*; +import java.util.Map; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.OptionalLong; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; From 83626aae435c157e152405be057471352544ff7b Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 13:46:35 -0700 Subject: [PATCH 15/35] fix OzoneKey creation time and modification time error --- .../src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index a9571ee05d30..ed5133febf62 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -94,6 +94,8 @@ public OzoneKey(String volumeName, String bucketName, this.bucketName = bucketName; this.name = keyName; this.dataSize = size; + this.creationTime = Instant.ofEpochMilli(creationTime); + this.modificationTime = Instant.ofEpochMilli(modificationTime); this.replicationConfig = replicationConfig; } From ff1603b5c30fa39c6e8f926b85dad5b3fc1612db Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 14:16:23 -0700 Subject: [PATCH 16/35] fix EndpointBase missing import --- .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 5df309ae9b54..e164d8c53f6e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -27,6 +27,10 @@ import java.util.Set; import java.util.HashSet; import java.util.Arrays; +import java.util.Map; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Collections; import java.util.function.Function; import java.util.stream.Collectors; From b420b0de2830a43266806cc0451abd6771fd5c8d Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 14:45:28 -0700 Subject: [PATCH 17/35] fix OzoneKey unit test --- .../java/org/apache/hadoop/ozone/client/TestOzoneClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java index 93a511ab008b..23b181fcf498 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClient.java @@ -167,8 +167,8 @@ public void testPutKeyRatisOneNode() throws IOException { Assert.assertEquals(value.length(), is.read(fileContent)); is.close(); Assert.assertEquals(value, new String(fileContent, UTF_8)); -// Assert.assertFalse(key.getCreationTime().isBefore(testStartTime)); -// Assert.assertFalse(key.getModificationTime().isBefore(testStartTime)); + Assert.assertFalse(key.getCreationTime().isBefore(testStartTime)); + Assert.assertFalse(key.getModificationTime().isBefore(testStartTime)); } } From e1db11e4c588a59bdcddd7c8cccd0c28fbe28997 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 14:47:21 -0700 Subject: [PATCH 18/35] remove unused variable from OzoneKey --- .../src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index ed5133febf62..fe54dbe94c79 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -64,9 +64,6 @@ public class OzoneKey { private Map metadata = new HashMap<>(); - private Set excludeMetadataFields = - new HashSet<>(Arrays.asList("gdprEnabled")); - /** * Constructs OzoneKey from OmKeyInfo. * From 7febc6a186b13e41bddbdd45abed3add0e3b28fb Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 31 Aug 2022 17:40:38 -0700 Subject: [PATCH 19/35] fix compiling error --- .../src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java | 3 --- .../java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index fe54dbe94c79..237e922763d7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -26,9 +26,6 @@ import java.time.Instant; import java.util.Map; import java.util.HashMap; -import java.util.Set; -import java.util.HashSet; -import java.util.Arrays; /** * A class that encapsulates OzoneKey. diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index e164d8c53f6e..52624c3832c3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Collections; import java.util.function.Function; import java.util.stream.Collectors; From 84e5abfc43beffe15f180c360b84656ec8ee3085 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 13:17:42 -0700 Subject: [PATCH 20/35] pass metadata to copyObject method --- .../apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index cdc9933dc023..134580a64225 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -890,10 +890,10 @@ public void setContext(ContainerRequestContext context) { void copy(OzoneVolume volume, InputStream src, long srcKeyLen, String destKey, String destBucket, - ReplicationConfig replication) throws IOException { + ReplicationConfig replication, Map metadata) throws IOException { try (OzoneOutputStream dest = getClientProtocol().createKey( volume.getName(), destBucket, destKey, srcKeyLen, - replication, new HashMap<>())) { + replication, metadata)) { IOUtils.copy(src, dest); } } @@ -944,7 +944,8 @@ private CopyObjectResponse copyObject(OzoneVolume volume, try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(), sourceBucket, sourceKey)) { - copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig); + copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig, + sourceKeyDetails.getMetadata()); } final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails( From 8acd6c6104d4351e4983f34f993c0558c44473c4 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 13:28:44 -0700 Subject: [PATCH 21/35] pass metadata to copyObject and put method of ObjectEndpoint class --- .../apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 134580a64225..84873cb69db2 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -211,11 +211,11 @@ public Response put( } // Normal put object - bucket = getBucket(bucketName); +// bucket = getBucket(bucketName); Map customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders()); - output = bucket.createKey(keyPath, length, - replicationConfig, customMetadata); +// output = bucket.createKey(keyPath, length, +// replicationConfig, customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { @@ -223,7 +223,7 @@ public Response put( } output = getClientProtocol().createKey(volume.getName(), bucketName, - keyPath, length, replicationConfig, new HashMap<>()); + keyPath, length, replicationConfig, customMetadata); IOUtils.copy(body, output); getMetrics().incCreateKeySuccess(); From 35c4773339263014a015fda4eb5d22d102089ea7 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 14:24:04 -0700 Subject: [PATCH 22/35] add Freon tests for s3 api with request of invalid size of metadata and gdpr enabled tag --- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +- .../dist/src/main/smoketest/s3/objectputget.robot | 14 +++++++++++++- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index d4c2619d59ce..382b69644b91 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -391,7 +391,7 @@ private OzoneConsts() { public static final int S3_SECRET_KEY_MIN_LENGTH = 8; - public static final int S3_REQUEST_HEADER_METADATA_SIZE_LIMIT = 2; + public static final int S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB = 2; //GDPR public static final String GDPR_FLAG = "gdprEnabled"; diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 421a7932a522..c57134a63057 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -170,4 +170,16 @@ Create file with user defined metadata ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1 Should contain ${result} \"custom-key1\" : \"custom-value1\" - Should contain ${result} \"custom-key2\" : \"custom-value2\" \ No newline at end of file + Should contain ${result} \"custom-key2\" : \"custom-value2\" + +Create file with user defined metadata with gdpr enabled value in request + Execute echo "Randomtext" > /tmp/testfile2 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" + Should contain ${result} \"custom-key2\" : \"custom-value2\" + Should not contain ${result} \"gdprEnabled\" : \"true\" + +Create file with user defined metadata larger than 2 KB + Execute echo "Randomtext" > /tmp/testfile2 + Execute dd if=/dev/zero of=/tmp/testfileOver2KB bs=3000 count=1 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata /tmp/testfileOver2KB + Should contain ${result} \"Illegal user defined metadata\" diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 84873cb69db2..8e6c21a929e7 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -49,7 +49,6 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Map; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.OptionalLong; @@ -891,7 +890,8 @@ public void setContext(ContainerRequestContext context) { void copy(OzoneVolume volume, InputStream src, long srcKeyLen, String destKey, String destBucket, ReplicationConfig replication, Map metadata) throws IOException { - try (OzoneOutputStream dest = getClientProtocol().createKey( + try (OzoneOutputStream dest = + getClientProtocol().createKey( volume.getName(), destBucket, destKey, srcKeyLen, replication, metadata)) { IOUtils.copy(src, dest); From 0852059c99fce3b241fe394d0af1a7662238add9 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 14:38:52 -0700 Subject: [PATCH 23/35] Update s3 api Freon test result message --- hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index c57134a63057..fec08f1b5f88 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -182,4 +182,4 @@ Create file with user defined metadata larger than 2 KB Execute echo "Randomtext" > /tmp/testfile2 Execute dd if=/dev/zero of=/tmp/testfileOver2KB bs=3000 count=1 Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata /tmp/testfileOver2KB - Should contain ${result} \"Illegal user defined metadata\" + Should contain ${result} Illegal user defined metadata From 07f84a51e3456eb1784547522d89305bbc1f9b47 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 14:44:48 -0700 Subject: [PATCH 24/35] Fix checkstyle error; fix updated variable name --- .../java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 2 +- .../org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 52624c3832c3..252b5ec62bf2 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -283,7 +283,7 @@ protected Map getCustomMetadataFromHeaders( sizeInBytes += mapKey.getBytes(UTF_8).length; sizeInBytes += value.getBytes(UTF_8).length; if (sizeInBytes > - OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT * KB) { + OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) { throw new IllegalArgumentException("Illegal user defined metadata." + " Combined size cannot exceed 2KB."); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 8e6c21a929e7..49d2149d93e0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -889,7 +889,8 @@ public void setContext(ContainerRequestContext context) { void copy(OzoneVolume volume, InputStream src, long srcKeyLen, String destKey, String destBucket, - ReplicationConfig replication, Map metadata) throws IOException { + ReplicationConfig replication, + Map metadata) throws IOException { try (OzoneOutputStream dest = getClientProtocol().createKey( volume.getName(), destBucket, destKey, srcKeyLen, From 02fb61232cf70952904bbd92ce08cfc712a965ad Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 16:32:47 -0700 Subject: [PATCH 25/35] Update s3 api robot test --- hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index fec08f1b5f88..3cbe2b6b16c0 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -174,12 +174,13 @@ Create file with user defined metadata Create file with user defined metadata with gdpr enabled value in request Execute echo "Randomtext" > /tmp/testfile2 - Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" Should contain ${result} \"custom-key2\" : \"custom-value2\" Should not contain ${result} \"gdprEnabled\" : \"true\" Create file with user defined metadata larger than 2 KB Execute echo "Randomtext" > /tmp/testfile2 Execute dd if=/dev/zero of=/tmp/testfileOver2KB bs=3000 count=1 - Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata /tmp/testfileOver2KB + Execute testMetadata=`cat /tmp/testfileOver2KB` + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" Should contain ${result} Illegal user defined metadata From fe93b51f27ad9ed6ba7aecdc4466259c9e8da146 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 17:36:08 -0700 Subject: [PATCH 26/35] Update s3 api robot test --- .../src/main/smoketest/s3/objectputget.robot | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 3cbe2b6b16c0..742db2842510 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -174,13 +174,19 @@ Create file with user defined metadata Create file with user defined metadata with gdpr enabled value in request Execute echo "Randomtext" > /tmp/testfile2 - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" + + ${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 Should contain ${result} \"custom-key2\" : \"custom-value2\" Should not contain ${result} \"gdprEnabled\" : \"true\" -Create file with user defined metadata larger than 2 KB + +Create file with user defined metadata size of total keys and values larger than 2 KB Execute echo "Randomtext" > /tmp/testfile2 - Execute dd if=/dev/zero of=/tmp/testfileOver2KB bs=3000 count=1 - Execute testMetadata=`cat /tmp/testfileOver2KB` - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" + Execute dd if=/dev/zero of=/tmp/testMetadataKey bs=2000 count=1 + Execute dd if=/dev/zero of=/tmp/testMetadataValue bs=2000 count=1 + Execute testMetadataKey=`cat /tmp/testMetadataKey` + Execute testMetadataValue=`cat /tmp/testMetadataValue` +# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadataKey=$testMetadataValue" Should contain ${result} Illegal user defined metadata From ff9ba654abff91beff55416417ea1c6950f62753 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 1 Sep 2022 22:31:23 -0700 Subject: [PATCH 27/35] Update s3 api robot test --- .../dist/src/main/smoketest/s3/objectputget.robot | 13 +++++++------ .../hadoop/ozone/s3/endpoint/EndpointBase.java | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 742db2842510..cd849db9425a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -177,16 +177,17 @@ Create file with user defined metadata with gdpr enabled value in request Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" ${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 - Should contain ${result} \"custom-key2\" : \"custom-value2\" - Should not contain ${result} \"gdprEnabled\" : \"true\" + Should contain ${result} \"custom-key2\": \"custom-value2\" + Should not contain ${result} \"gdprEnabled\": \"true\" Create file with user defined metadata size of total keys and values larger than 2 KB Execute echo "Randomtext" > /tmp/testfile2 - Execute dd if=/dev/zero of=/tmp/testMetadataKey bs=2000 count=1 - Execute dd if=/dev/zero of=/tmp/testMetadataValue bs=2000 count=1 - Execute testMetadataKey=`cat /tmp/testMetadataKey` - Execute testMetadataValue=`cat /tmp/testMetadataValue` +# Execute dd if=/dev/zero of=/tmp/testMetadataKey bs=2000 count=1 +# Execute dd if=/dev/zero of=/tmp/testMetadataValue bs=2000 count=1 +# Execute + Execute testMetadataKey=$(printf '=%.0s' {1..2000}) + Execute testMetadataValue=$(printf '=%.0s' {1..2000}) # ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadataKey=$testMetadataValue" Should contain ${result} Illegal user defined metadata diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 252b5ec62bf2..f906c9c260e4 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -267,7 +267,7 @@ protected Map getCustomMetadataFromHeaders( Set customMetadataKeys = requestHeaders.keySet().stream() .filter(k -> { if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && - !excludeMetadataFields.contains(k)) { + !excludeMetadataFields.contains(k.substring(CUSTOM_METADATA_HEADER_PREFIX.length()))) { return true; } return false; From 7bd79b69130e40a9ac73f6c7a1f8a4191d572c38 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Tue, 6 Sep 2022 15:37:55 -0700 Subject: [PATCH 28/35] Fix s3 robot test --- .../src/main/smoketest/s3/commonawslib.robot | 5 + .../src/main/smoketest/s3/objectputget.robot | 5 +- .../src/main/smoketest/s3/tmp_test_put.robot | 198 ++++++++++++++++++ .../ozone/s3/endpoint/EndpointBase.java | 25 ++- .../ozone/s3/endpoint/ObjectEndpoint.java | 2 +- 5 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 0b03f3c75e30..50c70072775e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -31,6 +31,11 @@ Execute AWSS3APICli ${output} = Execute aws s3api --endpoint-url ${ENDPOINT_URL} ${command} [return] ${output} +Execute AWSS3APICli with testFile + [Arguments] ${command} + ${output} = Execute testMetadata="$(printf 'k%.0s' {1..2000})=$(printf 'v%.0s' {1..2000})";aws s3api --endpoint-url ${ENDPOINT_URL} ${command} + [return] ${output} + Execute AWSS3APICli and checkrc [Arguments] ${command} ${expected_error_code} ${output} = Execute and checkrc aws s3api --endpoint-url ${ENDPOINT_URL} ${command} ${expected_error_code} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index cd849db9425a..ae3e3fbd7d7b 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -186,8 +186,9 @@ Create file with user defined metadata size of total keys and values larger than # Execute dd if=/dev/zero of=/tmp/testMetadataKey bs=2000 count=1 # Execute dd if=/dev/zero of=/tmp/testMetadataValue bs=2000 count=1 # Execute - Execute testMetadataKey=$(printf '=%.0s' {1..2000}) - Execute testMetadataValue=$(printf '=%.0s' {1..2000}) + Execute testMetadataKey=$(printf '=%.0s' {1..20}) + Execute testMetadataValue=$(printf '=%.0s' {1..20}) + Execute echo "$testMetadataKey=$testMetadataValue" # ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadataKey=$testMetadataValue" Should contain ${result} Illegal user defined metadata diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot b/hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot new file mode 100644 index 000000000000..7df8aec899c0 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation S3 gateway test with aws cli +Library OperatingSystem +Library String +Resource ../commonlib.robot +Resource commonawslib.robot +Test Timeout 5 minutes +Suite Setup Setup s3 tests + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 +${OZONE_TEST} true +${BUCKET} generated + +*** Test Cases *** + +#Put object to s3 +# Execute echo "Randomtext" > /tmp/testfile +# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --body /tmp/testfile +# ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/putobject/key=value/ +# Should contain ${result} f1 +# +# Execute touch -f /tmp/zerobyte +# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --body /tmp/zerobyte +# ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/putobject/key=value/ +# Should contain ${result} zerobyte +# +##This test depends on the previous test case. Can't be executes alone +#Get object from s3 +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 /tmp/testfile.result +# Compare files /tmp/testfile /tmp/testfile.result +# +##This test depends on the previous test case. Can't be executed alone +#Get object with wrong signature +# Pass Execution If '${SECURITY_ENABLED}' == 'false' Skip in unsecure cluster +# ${result} = Execute and Ignore Error curl -i -H 'Authorization: AWS scm/scm@EXAMPLE.COM:asdfqwerty' ${ENDPOINT_URL}/${BUCKET}/${PREFIX}/putobject/key=value/f1 +# Should contain ${result} 403 Forbidden +# +#Get Partial object from s3 with both start and endoffset +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-4 /tmp/testfile1.result +# Should contain ${result} ContentRange +# Should contain ${result} bytes 0-4/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=5 2>/dev/null +# ${actualData} = Execute cat /tmp/testfile1.result +# Should Be Equal ${expectedData} ${actualData} +# +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=2-4 /tmp/testfile1.result1 +# Should contain ${result} ContentRange +# Should contain ${result} bytes 2-4/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=3 2>/dev/null +# ${actualData} = Execute cat /tmp/testfile1.result1 +# Should Be Equal ${expectedData} ${actualData} +# +## end offset greater than file size and start with in file length +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=2-1000 /tmp/testfile1.result2 +# Should contain ${result} ContentRange +# Should contain ${result} bytes 2-10/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=9 2>/dev/null +# ${actualData} = Execute cat /tmp/testfile1.result2 +# Should Be Equal ${expectedData} ${actualData} +# +#Get Partial object from s3 with both start and endoffset(start offset and endoffset is greater than file size) +# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=10000-10000 /tmp/testfile2.result 255 +# Should contain ${result} InvalidRange +# +# +#Get Partial object from s3 with both start and endoffset(end offset is greater than file size) +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-10000 /tmp/testfile2.result +# Should contain ${result} ContentRange +# Should contain ${result} bytes 0-10/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute cat /tmp/testfile +# ${actualData} = Execute cat /tmp/testfile2.result +# Should Be Equal ${expectedData} ${actualData} +# +#Get Partial object from s3 with only start offset +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0- /tmp/testfile3.result +# Should contain ${result} ContentRange +# Should contain ${result} bytes 0-10/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute cat /tmp/testfile +# ${actualData} = Execute cat /tmp/testfile3.result +# Should Be Equal ${expectedData} ${actualData} +# +#Get Partial object from s3 with both start and endoffset which are equal +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-0 /tmp/testfile4.result +# Should contain ${result} ContentRange +# Should contain ${result} bytes 0-0/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=1 2>/dev/null +# ${actualData} = Execute cat /tmp/testfile4.result +# Should Be Equal ${expectedData} ${actualData} +# +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=4-4 /tmp/testfile5.result +# Should contain ${result} ContentRange +# Should contain ${result} bytes 4-4/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute dd if=/tmp/testfile skip=4 bs=1 count=1 2>/dev/null +# ${actualData} = Execute cat /tmp/testfile5.result +# Should Be Equal ${expectedData} ${actualData} +# +#Get Partial object from s3 to get last n bytes +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-4 /tmp/testfile6.result +# Should contain ${result} ContentRange +# Should contain ${result} bytes 7-10/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute dd if=/tmp/testfile skip=7 bs=1 count=4 2>/dev/null +# ${actualData} = Execute cat /tmp/testfile6.result +# Should Be Equal ${expectedData} ${actualData} +# +## if end is greater than file length, returns whole file +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-10000 /tmp/testfile7.result +# Should contain ${result} ContentRange +# Should contain ${result} bytes 0-10/11 +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute cat /tmp/testfile +# ${actualData} = Execute cat /tmp/testfile7.result +# Should Be Equal ${expectedData} ${actualData} +# +#Incorrect values for end and start offset +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-11-10000 /tmp/testfile8.result +# Should not contain ${result} ContentRange +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute cat /tmp/testfile +# ${actualData} = Execute cat /tmp/testfile8.result +# Should Be Equal ${expectedData} ${actualData} +# +# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=11-8 /tmp/testfile9.result +# Should not contain ${result} ContentRange +# Should contain ${result} AcceptRanges +# ${expectedData} = Execute cat /tmp/testfile +# ${actualData} = Execute cat /tmp/testfile8.result +# Should Be Equal ${expectedData} ${actualData} +# +#Zero byte file +# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result 255 +# Should contain ${result} InvalidRange +# +# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result 255 +# Should contain ${result} InvalidRange +# +# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 +# Should contain ${result} InvalidRange +# +#Create file with user defined metadata +# Execute echo "Randomtext" > /tmp/testfile2 +# Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" +# +# ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 +# Should contain ${result} \"custom-key1\": \"custom-value1\" +# Should contain ${result} \"custom-key2\": \"custom-value2\" +# +# ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1 +# Should contain ${result} \"custom-key1\" : \"custom-value1\" +# Should contain ${result} \"custom-key2\" : \"custom-value2\" +# +#Create file with user defined metadata with gdpr enabled value in request +# Execute echo "Randomtext" > /tmp/testfile2 +# Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" +# +# ${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 +# Should contain ${result} \"custom-key2\": \"custom-value2\" +# Should not contain ${result} \"gdprEnabled\": \"true\" + + +Create file with user defined metadata size of total keys and values larger than 2 KB + Execute echo "Randomtext" > /tmp/testfile2 +# Execute dd if=/dev/zero of=/tmp/testMetadataKey bs=2000 count=1 +# Execute dd if=/dev/zero of=/tmp/testMetadataValue bs=2000 count=1 +# Execute testMetadataKey=$(printf 'k%.0s' {1..20}) +# Execute testMetadataValue=$(printf 'v%.0s' {1..20}) + Execute testMetadataKey=$(printf 'k%.0s' {1..20});testMetadataValue=$(printf 'v%.0s' {1..20});echo "$testMetadataKey=$testMetadataValue" > /tmp/test_test.txt +# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" +# ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" + ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="testKeyKeyKey=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +# ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata '{"$(printf 'k%.0s' {1..20})"="$(printf 'v%.0s' {1..20})"}' +# ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata /tmp/test_test.txt + + Should contain ${result} Illegal user defined metadata diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index f906c9c260e4..da7b2208fedb 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -58,6 +58,7 @@ import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.apache.hadoop.ozone.s3.util.AuditUtils; +import org.apache.http.protocol.HTTP; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -258,12 +259,17 @@ private Iterator iterateBuckets( } protected Map getCustomMetadataFromHeaders( - MultivaluedMap requestHeaders) { + MultivaluedMap requestHeaders) throws OS3Exception { Map customMetadata = new HashMap<>(); if (requestHeaders == null || requestHeaders.isEmpty()) { return customMetadata; } + LOG.error("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** "); + LOG.error("list all pre-filtered metadata keys in request: "); + requestHeaders.keySet().forEach(System.out::println); + LOG.error("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** "); + Set customMetadataKeys = requestHeaders.keySet().stream() .filter(k -> { if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && @@ -273,6 +279,11 @@ protected Map getCustomMetadataFromHeaders( return false; }) .collect(Collectors.toSet()); + LOG.error("#### #### #### #### #### #### #### #### #### #### #### #### #### #### ####"); + LOG.error("list all metadata keys in s3 request: "); + customMetadataKeys.forEach(System.out:: println); + LOG.error("#### #### #### #### #### #### #### #### #### #### #### #### #### #### ####"); + long sizeInBytes = 0; if (!customMetadataKeys.isEmpty()) { for (String key : customMetadataKeys) { @@ -280,12 +291,20 @@ protected Map getCustomMetadataFromHeaders( key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); List values = requestHeaders.get(key); String value = StringUtils.join(values, ","); + LOG.error("cur mapKey = " + mapKey); sizeInBytes += mapKey.getBytes(UTF_8).length; + LOG.error("cur sizeInBytes = " + sizeInBytes); + LOG.error("cur mapValue = " + value); sizeInBytes += value.getBytes(UTF_8).length; + LOG.error("cur sizeInBytes = " + sizeInBytes); + if (sizeInBytes > OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) { - throw new IllegalArgumentException("Illegal user defined metadata." + - " Combined size cannot exceed 2KB."); +// throw new IllegalArgumentException("Illegal user defined metadata." + +// " Combined size cannot exceed 2KB."); + throw new OS3Exception("403", "Illegal user defined metadata." + + " Combined size cannot exceed 2KB.", 403); + } customMetadata.put(mapKey, value); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 49d2149d93e0..7847c2cf3f1f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -173,7 +173,7 @@ public Response put( @HeaderParam("Content-Length") long length, @QueryParam("partNumber") int partNumber, @QueryParam("uploadId") @DefaultValue("") String uploadID, - InputStream body) throws IOException, OS3Exception { + InputStream body) throws IOException, OS3Exception, IllegalArgumentException { S3GAction s3GAction = S3GAction.CREATE_KEY; boolean auditSuccess = true; From aafd1c69e553eb32ae1e0fe67ef14926a59e010f Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Wed, 7 Sep 2022 16:50:18 -0700 Subject: [PATCH 29/35] Update s3 robot test case where request with custom metadata size over 2 KB --- .../src/main/smoketest/s3/commonawslib.robot | 10 +- .../src/main/smoketest/s3/objectputget.robot | 15 +- .../src/main/smoketest/s3/tmp_test_put.robot | 198 ------------------ .../ozone/s3/endpoint/EndpointBase.java | 6 +- 4 files changed, 11 insertions(+), 218 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 50c70072775e..60c73866f715 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -31,16 +31,16 @@ Execute AWSS3APICli ${output} = Execute aws s3api --endpoint-url ${ENDPOINT_URL} ${command} [return] ${output} -Execute AWSS3APICli with testFile - [Arguments] ${command} - ${output} = Execute testMetadata="$(printf 'k%.0s' {1..2000})=$(printf 'v%.0s' {1..2000})";aws s3api --endpoint-url ${ENDPOINT_URL} ${command} - [return] ${output} - Execute AWSS3APICli and checkrc [Arguments] ${command} ${expected_error_code} ${output} = Execute and checkrc aws s3api --endpoint-url ${ENDPOINT_URL} ${command} ${expected_error_code} [return] ${output} +Execute AWSS3APICli and ignore error + [Arguments] ${command} + ${output} = Execute And Ignore Error aws s3api --endpoint-url ${ENDPOINT_URL} ${command} + [return] ${output} + Execute AWSS3Cli [Arguments] ${command} ${output} = Execute aws s3 --endpoint-url ${ENDPOINT_URL} ${command} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index ae3e3fbd7d7b..77f4b9e5fd13 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -175,20 +175,13 @@ Create file with user defined metadata Create file with user defined metadata with gdpr enabled value in request Execute echo "Randomtext" > /tmp/testfile2 Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" - ${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 Should contain ${result} \"custom-key2\": \"custom-value2\" Should not contain ${result} \"gdprEnabled\": \"true\" -Create file with user defined metadata size of total keys and values larger than 2 KB +Create file with user defined metadata size larger than 2 KB Execute echo "Randomtext" > /tmp/testfile2 -# Execute dd if=/dev/zero of=/tmp/testMetadataKey bs=2000 count=1 -# Execute dd if=/dev/zero of=/tmp/testMetadataValue bs=2000 count=1 -# Execute - Execute testMetadataKey=$(printf '=%.0s' {1..20}) - Execute testMetadataValue=$(printf '=%.0s' {1..20}) - Execute echo "$testMetadataKey=$testMetadataValue" -# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadataKey=$testMetadataValue" - Should contain ${result} Illegal user defined metadata + ${custom_metadata_value} = Execute printf 'v%.0s' {1..3000} + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}" + Should not contain ${result} custom-key1: ${custom_metadata_value} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot b/hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot deleted file mode 100644 index 7df8aec899c0..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/s3/tmp_test_put.robot +++ /dev/null @@ -1,198 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation S3 gateway test with aws cli -Library OperatingSystem -Library String -Resource ../commonlib.robot -Resource commonawslib.robot -Test Timeout 5 minutes -Suite Setup Setup s3 tests - -*** Variables *** -${ENDPOINT_URL} http://s3g:9878 -${OZONE_TEST} true -${BUCKET} generated - -*** Test Cases *** - -#Put object to s3 -# Execute echo "Randomtext" > /tmp/testfile -# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --body /tmp/testfile -# ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/putobject/key=value/ -# Should contain ${result} f1 -# -# Execute touch -f /tmp/zerobyte -# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --body /tmp/zerobyte -# ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/putobject/key=value/ -# Should contain ${result} zerobyte -# -##This test depends on the previous test case. Can't be executes alone -#Get object from s3 -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 /tmp/testfile.result -# Compare files /tmp/testfile /tmp/testfile.result -# -##This test depends on the previous test case. Can't be executed alone -#Get object with wrong signature -# Pass Execution If '${SECURITY_ENABLED}' == 'false' Skip in unsecure cluster -# ${result} = Execute and Ignore Error curl -i -H 'Authorization: AWS scm/scm@EXAMPLE.COM:asdfqwerty' ${ENDPOINT_URL}/${BUCKET}/${PREFIX}/putobject/key=value/f1 -# Should contain ${result} 403 Forbidden -# -#Get Partial object from s3 with both start and endoffset -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-4 /tmp/testfile1.result -# Should contain ${result} ContentRange -# Should contain ${result} bytes 0-4/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=5 2>/dev/null -# ${actualData} = Execute cat /tmp/testfile1.result -# Should Be Equal ${expectedData} ${actualData} -# -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=2-4 /tmp/testfile1.result1 -# Should contain ${result} ContentRange -# Should contain ${result} bytes 2-4/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=3 2>/dev/null -# ${actualData} = Execute cat /tmp/testfile1.result1 -# Should Be Equal ${expectedData} ${actualData} -# -## end offset greater than file size and start with in file length -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=2-1000 /tmp/testfile1.result2 -# Should contain ${result} ContentRange -# Should contain ${result} bytes 2-10/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=9 2>/dev/null -# ${actualData} = Execute cat /tmp/testfile1.result2 -# Should Be Equal ${expectedData} ${actualData} -# -#Get Partial object from s3 with both start and endoffset(start offset and endoffset is greater than file size) -# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=10000-10000 /tmp/testfile2.result 255 -# Should contain ${result} InvalidRange -# -# -#Get Partial object from s3 with both start and endoffset(end offset is greater than file size) -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-10000 /tmp/testfile2.result -# Should contain ${result} ContentRange -# Should contain ${result} bytes 0-10/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute cat /tmp/testfile -# ${actualData} = Execute cat /tmp/testfile2.result -# Should Be Equal ${expectedData} ${actualData} -# -#Get Partial object from s3 with only start offset -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0- /tmp/testfile3.result -# Should contain ${result} ContentRange -# Should contain ${result} bytes 0-10/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute cat /tmp/testfile -# ${actualData} = Execute cat /tmp/testfile3.result -# Should Be Equal ${expectedData} ${actualData} -# -#Get Partial object from s3 with both start and endoffset which are equal -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-0 /tmp/testfile4.result -# Should contain ${result} ContentRange -# Should contain ${result} bytes 0-0/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=1 2>/dev/null -# ${actualData} = Execute cat /tmp/testfile4.result -# Should Be Equal ${expectedData} ${actualData} -# -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=4-4 /tmp/testfile5.result -# Should contain ${result} ContentRange -# Should contain ${result} bytes 4-4/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute dd if=/tmp/testfile skip=4 bs=1 count=1 2>/dev/null -# ${actualData} = Execute cat /tmp/testfile5.result -# Should Be Equal ${expectedData} ${actualData} -# -#Get Partial object from s3 to get last n bytes -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-4 /tmp/testfile6.result -# Should contain ${result} ContentRange -# Should contain ${result} bytes 7-10/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute dd if=/tmp/testfile skip=7 bs=1 count=4 2>/dev/null -# ${actualData} = Execute cat /tmp/testfile6.result -# Should Be Equal ${expectedData} ${actualData} -# -## if end is greater than file length, returns whole file -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-10000 /tmp/testfile7.result -# Should contain ${result} ContentRange -# Should contain ${result} bytes 0-10/11 -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute cat /tmp/testfile -# ${actualData} = Execute cat /tmp/testfile7.result -# Should Be Equal ${expectedData} ${actualData} -# -#Incorrect values for end and start offset -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-11-10000 /tmp/testfile8.result -# Should not contain ${result} ContentRange -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute cat /tmp/testfile -# ${actualData} = Execute cat /tmp/testfile8.result -# Should Be Equal ${expectedData} ${actualData} -# -# ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=11-8 /tmp/testfile9.result -# Should not contain ${result} ContentRange -# Should contain ${result} AcceptRanges -# ${expectedData} = Execute cat /tmp/testfile -# ${actualData} = Execute cat /tmp/testfile8.result -# Should Be Equal ${expectedData} ${actualData} -# -#Zero byte file -# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result 255 -# Should contain ${result} InvalidRange -# -# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result 255 -# Should contain ${result} InvalidRange -# -# ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 -# Should contain ${result} InvalidRange -# -#Create file with user defined metadata -# Execute echo "Randomtext" > /tmp/testfile2 -# Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" -# -# ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 -# Should contain ${result} \"custom-key1\": \"custom-value1\" -# Should contain ${result} \"custom-key2\": \"custom-value2\" -# -# ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1 -# Should contain ${result} \"custom-key1\" : \"custom-value1\" -# Should contain ${result} \"custom-key2\" : \"custom-value2\" -# -#Create file with user defined metadata with gdpr enabled value in request -# Execute echo "Randomtext" > /tmp/testfile2 -# Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" -# -# ${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 -# Should contain ${result} \"custom-key2\": \"custom-value2\" -# Should not contain ${result} \"gdprEnabled\": \"true\" - - -Create file with user defined metadata size of total keys and values larger than 2 KB - Execute echo "Randomtext" > /tmp/testfile2 -# Execute dd if=/dev/zero of=/tmp/testMetadataKey bs=2000 count=1 -# Execute dd if=/dev/zero of=/tmp/testMetadataValue bs=2000 count=1 -# Execute testMetadataKey=$(printf 'k%.0s' {1..20}) -# Execute testMetadataValue=$(printf 'v%.0s' {1..20}) - Execute testMetadataKey=$(printf 'k%.0s' {1..20});testMetadataValue=$(printf 'v%.0s' {1..20});echo "$testMetadataKey=$testMetadataValue" > /tmp/test_test.txt -# ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" -# ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="$testMetadata" - ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="testKeyKeyKey=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - -# ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata '{"$(printf 'k%.0s' {1..20})"="$(printf 'v%.0s' {1..20})"}' -# ${result} = Execute AWSS3APICli with testFile put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata /tmp/test_test.txt - - Should contain ${result} Illegal user defined metadata diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index da7b2208fedb..5f50ca8be633 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -300,10 +300,8 @@ protected Map getCustomMetadataFromHeaders( if (sizeInBytes > OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) { -// throw new IllegalArgumentException("Illegal user defined metadata." + -// " Combined size cannot exceed 2KB."); - throw new OS3Exception("403", "Illegal user defined metadata." + - " Combined size cannot exceed 2KB.", 403); + throw new IllegalArgumentException("Illegal user defined metadata." + + " Combined size cannot exceed 2KB."); } customMetadata.put(mapKey, value); From 4f4357b7ec1809fdb53f274d76ef45857f878841 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 8 Sep 2022 09:51:58 -0700 Subject: [PATCH 30/35] Fix checkstyle error --- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 13 ++----------- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 5f50ca8be633..9276354060fa 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -265,24 +265,16 @@ protected Map getCustomMetadataFromHeaders( return customMetadata; } - LOG.error("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** "); - LOG.error("list all pre-filtered metadata keys in request: "); - requestHeaders.keySet().forEach(System.out::println); - LOG.error("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** "); - Set customMetadataKeys = requestHeaders.keySet().stream() .filter(k -> { if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && - !excludeMetadataFields.contains(k.substring(CUSTOM_METADATA_HEADER_PREFIX.length()))) { + !excludeMetadataFields.contains( + k.substring(CUSTOM_METADATA_HEADER_PREFIX.length()))) { return true; } return false; }) .collect(Collectors.toSet()); - LOG.error("#### #### #### #### #### #### #### #### #### #### #### #### #### #### ####"); - LOG.error("list all metadata keys in s3 request: "); - customMetadataKeys.forEach(System.out:: println); - LOG.error("#### #### #### #### #### #### #### #### #### #### #### #### #### #### ####"); long sizeInBytes = 0; if (!customMetadataKeys.isEmpty()) { @@ -302,7 +294,6 @@ protected Map getCustomMetadataFromHeaders( OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) { throw new IllegalArgumentException("Illegal user defined metadata." + " Combined size cannot exceed 2KB."); - } customMetadata.put(mapKey, value); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 7847c2cf3f1f..49d2149d93e0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -173,7 +173,7 @@ public Response put( @HeaderParam("Content-Length") long length, @QueryParam("partNumber") int partNumber, @QueryParam("uploadId") @DefaultValue("") String uploadID, - InputStream body) throws IOException, OS3Exception, IllegalArgumentException { + InputStream body) throws IOException, OS3Exception { S3GAction s3GAction = S3GAction.CREATE_KEY; boolean auditSuccess = true; From e1f911aac74927e4cbd0701737947c5092da4cef Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 8 Sep 2022 13:34:02 -0700 Subject: [PATCH 31/35] Remove debug log;remove unused import library --- .../dist/src/main/smoketest/s3/objectputget.robot | 4 ++-- .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 8 ++------ .../apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 3 --- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 77f4b9e5fd13..2f74decb617a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -161,8 +161,8 @@ Zero byte file Should contain ${result} InvalidRange Create file with user defined metadata - Execute echo "Randomtext" > /tmp/testfile2 - Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" + Execute echo "Randomtext" > /tmp/testfile2 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 Should contain ${result} \"custom-key1\": \"custom-value1\" diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 9276354060fa..56ca728a1eea 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -58,7 +58,6 @@ import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.apache.hadoop.ozone.s3.util.AuditUtils; -import org.apache.http.protocol.HTTP; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -269,7 +268,8 @@ protected Map getCustomMetadataFromHeaders( .filter(k -> { if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && !excludeMetadataFields.contains( - k.substring(CUSTOM_METADATA_HEADER_PREFIX.length()))) { + k.substring( + CUSTOM_METADATA_HEADER_PREFIX.length()))) { return true; } return false; @@ -283,12 +283,8 @@ protected Map getCustomMetadataFromHeaders( key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); List values = requestHeaders.get(key); String value = StringUtils.join(values, ","); - LOG.error("cur mapKey = " + mapKey); sizeInBytes += mapKey.getBytes(UTF_8).length; - LOG.error("cur sizeInBytes = " + sizeInBytes); - LOG.error("cur mapValue = " + value); sizeInBytes += value.getBytes(UTF_8).length; - LOG.error("cur sizeInBytes = " + sizeInBytes); if (sizeInBytes > OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 49d2149d93e0..04bc7677c48a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -210,11 +210,8 @@ public Response put( } // Normal put object -// bucket = getBucket(bucketName); Map customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders()); -// output = bucket.createKey(keyPath, length, -// replicationConfig, customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { From 8ea42b1282a88e3c5617ce0fc152ba6a1dfcf3cb Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Fri, 9 Sep 2022 11:08:47 -0700 Subject: [PATCH 32/35] Fix checkstyle error --- .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 56ca728a1eea..56a187fd004a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -268,8 +268,8 @@ protected Map getCustomMetadataFromHeaders( .filter(k -> { if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && !excludeMetadataFields.contains( - k.substring( - CUSTOM_METADATA_HEADER_PREFIX.length()))) { + k.substring( + CUSTOM_METADATA_HEADER_PREFIX.length()))) { return true; } return false; From 0025afe43a69ff8bee40e711425b2d24bf20e425 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Fri, 9 Sep 2022 16:38:22 -0700 Subject: [PATCH 33/35] Remove additional line --- .../src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 237e922763d7..3c3af1367567 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -60,7 +60,6 @@ public class OzoneKey { private ReplicationConfig replicationConfig; private Map metadata = new HashMap<>(); - /** * Constructs OzoneKey from OmKeyInfo. * From 13bd6cab80a8868dae7cc768bf445644ffd3f348 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 22 Sep 2022 16:05:47 -0700 Subject: [PATCH 34/35] Add unit test --- .../ozone/s3/endpoint/TestEndpointBase.java | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java new file mode 100644 index 000000000000..9c1f254b798e --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * Tests the s3 EndpointBase class methods. + */ +package org.apache.hadoop.ozone.s3.endpoint; + +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.junit.Assert; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; + +/** + * Test methods of the EndpointBase. + */ +public class TestEndpointBase { + + /** + * Verify s3 metadata key "gdprEnabled" can't be set up directly + * from the normal client's request, + * it should be decided on the server side. + */ + @Test + public void testFilterGDPRFromCustomMetadataHeaders() + throws OS3Exception { + MultivaluedMap s3requestHeaders + = new MultivaluedHashMap<>(); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key1", "custom-value1"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key2", "custom-value2"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + OzoneConsts.GDPR_FLAG, "true"); + + EndpointBase endpointBase = new EndpointBase() { + @Override + public void init() { } + }; + + Map filteredCustomMetadata = + endpointBase.getCustomMetadataFromHeaders(s3requestHeaders); + Assert.assertTrue(filteredCustomMetadata.containsKey("custom-key1")); + Assert.assertEquals( + "custom-value1", filteredCustomMetadata.get("custom-key1")); + Assert.assertTrue(filteredCustomMetadata.containsKey("custom-key2")); + Assert.assertEquals( + "custom-value2", filteredCustomMetadata.get("custom-key2")); + Assert.assertFalse( + filteredCustomMetadata.containsKey(OzoneConsts.GDPR_FLAG)); + } + + /** + * Verify s3 request metadata size should be smaller than 2 KB. + */ + @Test + public void testCustomMetadataHeadersSizeOverbig() { + MultivaluedMap s3requestHeaders + = new MultivaluedHashMap<>(); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key1", "custom-value1"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key2", "custom-value2"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key3", + new String(new byte[3000], StandardCharsets.UTF_8)); + + EndpointBase endpointBase = new EndpointBase() { + @Override + public void init() { } + }; + + Exception exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> endpointBase.getCustomMetadataFromHeaders(s3requestHeaders)); + Assert.assertEquals( + "Illegal user defined metadata. Combined size cannot exceed 2KB.", + exception.getMessage()); + } + +} From 867fa274b947f4e7b6e1668dbdf0217a90eab988 Mon Sep 17 00:00:00 2001 From: DaveTeng0 Date: Thu, 22 Sep 2022 17:19:41 -0700 Subject: [PATCH 35/35] Add setter for metadata attribute of OzoneKey class --- .../main/java/org/apache/hadoop/ozone/client/OzoneKey.java | 4 ++++ .../java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 3c3af1367567..72dfb26f2158 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -166,6 +166,10 @@ public Map getMetadata() { return metadata; } + public void setMetadata(Map metadata) { + this.metadata.putAll(metadata); + } + @Deprecated @JsonIgnore public ReplicationType getReplicationType() { diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 054ea4de9b5f..757431bc6bfd 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -57,7 +57,7 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, modificationTime, type, replicationFactor); this.ozoneKeyLocations = ozoneKeyLocations; this.feInfo = feInfo; - this.getMetadata().putAll(metadata); + this.setMetadata(metadata); }