From c9ebd6db969439afae3a1d6b384839853f598698 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 30 Apr 2025 17:11:04 +0800 Subject: [PATCH 01/11] HDDS-12935. Support unsigned chunked upload with STREAMING-UNSIGNED-PAYLOAD-TRAILER --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 46 ++++ .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 85 +++++++ .../ozone/s3/SignedChunksInputStream.java | 77 ++++-- .../ozone/s3/UnsignedChunksInputStream.java | 190 ++++++++++++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 56 +++-- .../s3/signature/StringToSignProducer.java | 55 +++-- .../apache/hadoop/ozone/s3/util/S3Consts.java | 7 +- .../apache/hadoop/ozone/s3/util/S3Utils.java | 77 +++++- .../ozone/s3/TestSignedChunksInputStream.java | 231 +++++++++++++----- .../s3/TestUnsignedChunkInputStream.java | 223 +++++++++++++++++ .../ozone/s3/endpoint/TestListParts.java | 3 + .../endpoint/TestMultipartUploadComplete.java | 3 + .../endpoint/TestMultipartUploadWithCopy.java | 5 + .../ozone/s3/endpoint/TestObjectGet.java | 2 + .../ozone/s3/endpoint/TestObjectPut.java | 11 + .../s3/endpoint/TestObjectTaggingDelete.java | 3 + .../s3/endpoint/TestObjectTaggingGet.java | 3 + .../s3/endpoint/TestObjectTaggingPut.java | 4 +- .../ozone/s3/endpoint/TestPartUpload.java | 7 + .../s3/endpoint/TestPartUploadWithStream.java | 3 + .../s3/endpoint/TestPermissionCheck.java | 3 + .../s3/endpoint/TestUploadWithStream.java | 2 + .../s3/metrics/TestS3GatewayMetrics.java | 3 + 23 files changed, 978 insertions(+), 121 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUnsignedChunkInputStream.java diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index cee69f0f3603..2fa4718cf7f7 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -29,6 +29,7 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.AmazonServiceException.ErrorType; +import com.amazonaws.HttpMethod; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; import com.amazonaws.services.s3.model.AccessControlList; @@ -37,6 +38,7 @@ import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.Grantee; import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; @@ -69,17 +71,22 @@ import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; import com.amazonaws.services.s3.transfer.model.UploadResult; +import com.amazonaws.util.IOUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -979,6 +986,45 @@ public void testQuotaExceeded() throws IOException { assertEquals("QuotaExceeded", ase.getErrorCode()); } + @Test + public void testPresignedUrlGet() throws IOException { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + + // Set the presigned URL to expire after one hour. + Date expiration = new java.util.Date(); + long expTimeMillis = Instant.now().plusMillis(1000 * 60 * 60) + .toEpochMilli(); + expiration.setTime(expTimeMillis); + + // Generate the presigned URL + GeneratePresignedUrlRequest generatePresignedUrlRequest = + new GeneratePresignedUrlRequest(bucketName, keyName) + .withMethod(HttpMethod.GET) + .withExpiration(expiration); + generatePresignedUrlRequest.addRequestParameter("x-custom-parameter", "custom-value"); + URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); + + // Download the object using HttpUrlConnection (since v1.1) + // Capture the response body to a byte array. + URL presignedUrl = new URL(url.toExternalForm()); + HttpURLConnection connection = (HttpURLConnection) presignedUrl.openConnection(); + connection.setRequestMethod("GET"); + // Download the result of executing the request. + try (InputStream s3is = connection.getInputStream(); + ByteArrayOutputStream bos = new ByteArrayOutputStream( + content.getBytes(StandardCharsets.UTF_8).length)) { + IOUtils.copy(s3is, bos); + assertEquals(content, bos.toString("UTF-8")); + } + } + private boolean isBucketEmpty(Bucket bucket) { ObjectListing objectListing = s3Client.listObjects(bucket.getName()); return objectListing.getObjectSummaries().isEmpty(); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 7ef1342886c6..0f89026268f9 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -25,14 +25,18 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.URL; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -62,6 +66,12 @@ import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.HttpExecuteRequest; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; @@ -69,6 +79,7 @@ import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.ListObjectsRequest; @@ -81,6 +92,10 @@ import software.amazon.awssdk.services.s3.model.Tagging; import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.utils.IoUtils; /** * This is an abstract class to test the AWS Java S3 SDK operations. @@ -324,6 +339,76 @@ public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception assertEquals(userMetadata, headObjectResponse.metadata()); } + @Test + public void testPresignedUrlGet() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName), + RequestBody.fromString(content)); + + try (S3Presigner presigner = S3Presigner.builder() + .endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get()) + .region(s3Client.serviceClientConfiguration().region()) + .credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build()) { + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(keyName) + .build(); + + GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(10)) // The URL will expire in 10 minutes. + .getObjectRequest(objectRequest) + .build(); + + PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest); + + // Download the object using HttpUrlConnection (since v1.1) + // Capture the response body to a byte array. + URL presignedUrl = presignedRequest.url(); + HttpURLConnection connection = (HttpURLConnection) presignedUrl.openConnection(); + connection.setRequestMethod("GET"); + // Download the result of executing the request. + try (InputStream s3is = connection.getInputStream(); + ByteArrayOutputStream bos = new ByteArrayOutputStream( + content.getBytes(StandardCharsets.UTF_8).length)) { + IoUtils.copy(s3is, bos); + assertEquals(content, bos.toString("UTF-8")); + } + + // Use the AWS SDK for Java SdkHttpClient class to do the download + SdkHttpRequest request = SdkHttpRequest.builder() + .method(SdkHttpMethod.GET) + .uri(presignedUrl.toURI()) + .build(); + + HttpExecuteRequest executeRequest = HttpExecuteRequest.builder() + .request(request) + .build(); + + try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create(); + ByteArrayOutputStream bos = new ByteArrayOutputStream( + content.getBytes(StandardCharsets.UTF_8).length)) { + HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); + assertTrue(response.responseBody().isPresent(), () -> "The presigned url download request " + + "should have a response body"); + response.responseBody().ifPresent( + abortableInputStream -> { + try { + IoUtils.copy(abortableInputStream, bos); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + assertEquals(content, bos.toString("UTF-8")); + } + } + } + private String getBucketName() { return getBucketName(""); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java index 725b75591307..a6af04db3e96 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java @@ -17,21 +17,56 @@ package org.apache.hadoop.ozone.s3; +import static org.apache.hadoop.ozone.s3.util.S3Utils.eol; + import java.io.IOException; import java.io.InputStream; import java.util.regex.Matcher; import java.util.regex.Pattern; /** - * Input stream implementation to read body with chunked signatures. This should also work + * Input stream implementation to read body of a signed chunked upload. This should also work * with the chunked payloads with trailer. * + *

+ * Example chunk data: + * 

+ * 10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n
+ * <65536-bytes>\r\n
+ * 400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n
+ * <1024-bytes>\r\n
+ * 0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n
+ * x-amz-checksum-crc32c:sOO8/Q==\r\n
+ * x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n
+ *
+ *

+ * For the first chunk 10000 will be read and decoded from base-16 representation to 65536, which is the size of + * the first chunk payload. Each chunk upload ends with a zero-byte final additional chunk. + * At the end, there might be a trailer checksum payload and signature, depending on whether the x-amz-content-sha256 + * header value contains "-TRAILER" suffix (e.g. STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER + * and STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER) and "x-amz-trailer" is specified (e.g. x-amz-checksum-crc32c). + *

+ * + *

+ * The logic is similar to {@link UnsignedChunksInputStream}, but there is a "chunk-signature" to parse. + *

+ * + *

* Note that there are no actual chunk signature verification taking place. The InputStream only * returns the actual chunk payload from chunked signatures format. + *

* - * See - * - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html - * - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html + * Reference: + * */ public class SignedChunksInputStream extends InputStream { @@ -46,12 +81,21 @@ public class SignedChunksInputStream extends InputStream { */ private int remainingData = 0; + /** + * Every chunked uploads (multiple chunks) contains a final zero-byte final additional + * chunk. This can be used as the end-of-file marker. + */ + private boolean isFinalChunkEncountered = false; + public SignedChunksInputStream(InputStream inputStream) { originalStream = inputStream; } @Override public int read() throws IOException { + if (isFinalChunkEncountered) { + return -1; + } if (remainingData > 0) { int curr = originalStream.read(); remainingData--; @@ -63,7 +107,10 @@ public int read() throws IOException { return curr; } else { remainingData = readContentLengthFromHeader(); - if (remainingData == -1) { + if (remainingData <= 0) { + // there is always a final zero byte chunk so we can stop reading + // if we encounter this chunk + isFinalChunkEncountered = true; return -1; } return read(); @@ -78,6 +125,8 @@ public int read(byte[] b, int off, int len) throws IOException { throw new IndexOutOfBoundsException(); } else if (len == 0) { return 0; + } else if (isFinalChunkEncountered) { + return -1; } int currentOff = off; int currentLen = len; @@ -103,7 +152,12 @@ public int read(byte[] b, int off, int len) throws IOException { } } else { remainingData = readContentLengthFromHeader(); - if (remainingData == -1) { + if (remainingData == 0) { + // there is always a final zero byte chunk so we can stop reading + // if we encounter this chunk + isFinalChunkEncountered = true; + } + if (isFinalChunkEncountered || remainingData < 0) { break; } } @@ -125,10 +179,9 @@ private int readContentLengthFromHeader() throws IOException { prev = curr; curr = next; } - // Example - // The chunk data sent: - // 10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2 - // <65536-bytes> + // Example of a single chunk data: + // 10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n + // <65536-bytes>\r\n // // 10000 will be read and decoded from base-16 representation to 65536, which is the size of // the subsequent chunk payload. @@ -145,8 +198,4 @@ private int readContentLengthFromHeader() throws IOException { throw new IOException("Invalid signature line: " + signatureLine); } } - - private boolean eol(int prev, int curr) { - return prev == 13 && curr == 10; - } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java new file mode 100644 index 000000000000..7cc44334db7b --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3; + +import static org.apache.hadoop.ozone.s3.util.S3Utils.eol; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Input stream implementation to read body of an unsigned chunked upload. + *

+ * Currently, the only valid value of x-amz-content-sha256 header to indicate + * transfer unsigned payload in multiple chunks is STREAMING-UNSIGNED-PAYLOAD-TRAILER. + * Therefore, the input stream should work with chunked payloads with checksum trailer. + * Nevertheless, this input stream also supports chunked upload without trailer. + *

+ *

+ * Example chunk data: + * 

+ * 10000\r\n
+ * <65536-bytes>\r\n
+ * 0\r\n
+ * x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n
+ *
+ *

+ *

+ * The 10000 will be read and decoded from base-16 representation to 65536, which is the size of + * the subsequent chunk payload. Each chunk upload ends with a zero-byte final additional chunk. + * At the end, there will be a trailer checksum payload + *

+ * + *

+ * The logic is similar to {@link SignedChunksInputStream}, but since it is an unsigned chunked upload + * there is no "chunk-signature" to parse. + *

+ * + *

+ * Note that there is not actual trailer checksum verification taking place. The InputStream only + * * returns the actual chunk payload from chunked signatures format. + *

+ * + * Reference: + * + */ +public class UnsignedChunksInputStream extends InputStream { + + private final InputStream originalStream; + + /** + * Size of the chunk payload. If zero, the content length should be parsed to + * retrieve the subsequent chunk payload size. + */ + private int remainingData = 0; + + /** + * Every chunked uploads (multiple chunks) contains a final zero-byte final additional + * chunk. This can be used as the end-of-file marker. + */ + private boolean isFinalChunkEncountered = false; + + public UnsignedChunksInputStream(InputStream inputStream) { + originalStream = inputStream; + } + + @Override + public int read() throws IOException { + if (isFinalChunkEncountered) { + return -1; + } + if (remainingData > 0) { + int curr = originalStream.read(); + remainingData--; + if (remainingData == 0) { + //read the "\r\n" at the end of the data section + originalStream.read(); + originalStream.read(); + } + return curr; + } else { + remainingData = readContentLengthFromHeader(); + if (remainingData <= 0) { + // since currently trailer checksum verification is not supported, we can + // stop reading after encountering the final zero-byte chunk. + isFinalChunkEncountered = true; + return -1; + } + return read(); + } + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } else if (isFinalChunkEncountered) { + return -1; + } + int currentOff = off; + int currentLen = len; + int totalReadBytes = 0; + int realReadLen = 0; + int maxReadLen = 0; + do { + if (remainingData > 0) { + // The chunk payload size has been decoded, now read the actual chunk payload + maxReadLen = Math.min(remainingData, currentLen); + realReadLen = originalStream.read(b, currentOff, maxReadLen); + if (realReadLen == -1) { + break; + } + currentOff += realReadLen; + currentLen -= realReadLen; + totalReadBytes += realReadLen; + remainingData -= realReadLen; + if (remainingData == 0) { + //read the "\r\n" at the end of the data section + originalStream.read(); + originalStream.read(); + } + } else { + remainingData = readContentLengthFromHeader(); + if (remainingData == 0) { + // there is always a final zero byte chunk so we can stop reading + // if we encounter this chunk + isFinalChunkEncountered = true; + } + if (remainingData <= 0) { + break; + } + } + } while (currentLen > 0); + return totalReadBytes > 0 ? totalReadBytes : -1; + } + + private int readContentLengthFromHeader() throws IOException { + int prev = -1; + int curr = 0; + StringBuilder buf = new StringBuilder(); + + //read everything until the next \r\n + while (!eol(prev, curr) && curr != -1) { + int next = originalStream.read(); + if (next != -1) { + buf.append((char) next); + } + prev = curr; + curr = next; + } + // Example of a single chunk data: + // 10000\r\n + // <65536-bytes>\r\n + // + // 10000 will be read and decoded from base-16 representation to 65536, which is the size of + // the subsequent chunk payload. + String readString = buf.toString().trim(); + if (readString.isEmpty()) { + return -1; + } + return Integer.parseInt(readString, 16); + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 1b11d47a166e..923e6221e71a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -57,7 +57,11 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_DIRECTIVE_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Utils.hasMultiChunksPayload; +import static org.apache.hadoop.ozone.s3.util.S3Utils.hasUnsignedPayload; import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode; +import static org.apache.hadoop.ozone.s3.util.S3Utils.validateMultiChunksUpload; +import static org.apache.hadoop.ozone.s3.util.S3Utils.validateSignatureHeader; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -126,6 +130,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.SignedChunksInputStream; +import org.apache.hadoop.ozone.s3.UnsignedChunksInputStream; import org.apache.hadoop.ozone.s3.endpoint.S3Tagging.Tag; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -303,17 +308,29 @@ public Response put( } // Normal put object - Map customMetadata = - getCustomMetadataFromHeaders(headers.getRequestHeaders()); - - if (S3Utils.hasSignedPayloadHeader(headers)) { - digestInputStream = new DigestInputStream(new SignedChunksInputStream(body), - getMessageDigestInstance()); + final String amzContentSha256Header = validateSignatureHeader(headers, keyPath); + InputStream chunkInputStream; + if (hasMultiChunksPayload(amzContentSha256Header)) { + validateMultiChunksUpload(headers, amzDecodedLength, keyPath); + if (hasUnsignedPayload(amzContentSha256Header)) { + chunkInputStream = new UnsignedChunksInputStream(body); + } else { + chunkInputStream = new SignedChunksInputStream(body); + } length = Long.parseLong(amzDecodedLength); } else { - digestInputStream = new DigestInputStream(body, getMessageDigestInstance()); + // Single chunk upload: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + // Possible x-amz-content-sha256 values + // - Actual payload checksum value: For signed payload + // - UNSIGNED-PAYLOAD: For unsigned payload + chunkInputStream = body; } + // DigestInputStream is used for ETag calculation + digestInputStream = new DigestInputStream(chunkInputStream, getMessageDigestInstance()); + + Map customMetadata = + getCustomMetadataFromHeaders(headers.getRequestHeaders()); Map tags = getTaggingFromHeaders(headers); long putLength; @@ -957,15 +974,26 @@ private Response createMultipartKey(OzoneVolume volume, String bucket, String copyHeader = null; DigestInputStream digestInputStream = null; try { - - if (S3Utils.hasSignedPayloadHeader(headers)) { - digestInputStream = new DigestInputStream(new SignedChunksInputStream(body), - getMessageDigestInstance()); - length = Long.parseLong( - headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER)); + final String amzContentSha256Header = validateSignatureHeader(headers, key); + InputStream chunkInputStream; + if (hasMultiChunksPayload(amzContentSha256Header)) { + String amzDecodedLength = headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER); + validateMultiChunksUpload(headers, amzDecodedLength, key); + if (hasUnsignedPayload(amzContentSha256Header)) { + chunkInputStream = new UnsignedChunksInputStream(body); + } else { + chunkInputStream = new SignedChunksInputStream(body); + } + length = Long.parseLong(amzDecodedLength); } else { - digestInputStream = new DigestInputStream(body, getMessageDigestInstance()); + // Single chunk upload: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + // Possible x-amz-content-sha256 values + // - Actual payload checksum value: For signed payload + // - UNSIGNED-PAYLOAD: For unsigned payload + chunkInputStream = body; } + // DigestInputStream is used for ETag calculation + digestInputStream = new DigestInputStream(chunkInputStream, getMessageDigestInstance()); copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER); String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index 5a474a024e29..ea460f62d494 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -19,7 +19,6 @@ import static java.time.temporal.ChronoUnit.SECONDS; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR; -import static org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_UNSIGNED_PAYLOAD_TRAILER; import static org.apache.hadoop.ozone.s3.util.S3Consts.UNSIGNED_PAYLOAD; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; @@ -201,31 +200,45 @@ public static String buildCanonicalRequest( validateCanonicalHeaders(canonicalHeaders.toString(), headers, unsignedPayload); - String payloadHash; - if (UNSIGNED_PAYLOAD.equals(headers.get(X_AMZ_CONTENT_SHA256)) || - STREAMING_UNSIGNED_PAYLOAD_TRAILER.equals(headers.get(X_AMZ_CONTENT_SHA256)) || - unsignedPayload) { - payloadHash = UNSIGNED_PAYLOAD; - } else { - // According to AWS Sig V4 documentation - // https://docs.aws.amazon.com/AmazonS3/latest/API/ - // sig-v4-header-based-auth.html - // Note: The x-amz-content-sha256 header is required - // for all AWS Signature Version 4 requests.(using Authorization header) - if (!headers.containsKey(X_AMZ_CONTENT_SHA256)) { - LOG.error("The request must include " + X_AMZ_CONTENT_SHA256 - + " header for signed payload"); - throw S3_AUTHINFO_CREATION_ERROR; - } - payloadHash = headers.get(X_AMZ_CONTENT_SHA256); - } - String canonicalRequest = method + NEWLINE + String payloadHash = getPayloadHash(headers, unsignedPayload); + + return method + NEWLINE + canonicalUri + NEWLINE + canonicalQueryStr + NEWLINE + canonicalHeaders + NEWLINE + signedHeaders + NEWLINE + payloadHash; - return canonicalRequest; + } + + private static String getPayloadHash(Map headers, boolean isUsingQueryParameter) + throws OS3Exception { + if (isUsingQueryParameter) { + // According to AWS Signature V4 documentation using Query Parameters + // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html + return UNSIGNED_PAYLOAD; + } + String contentSignatureHeaderValue = headers.get(X_AMZ_CONTENT_SHA256); + // According to AWS Signature V4 documentation using Authorization Header + // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + // The x-amz-content-sha256 header is required + // for all AWS Signature Version 4 requests using Authorization header. + if (contentSignatureHeaderValue == null) { + LOG.error("The request must include " + X_AMZ_CONTENT_SHA256 + + " header for signed payload"); + throw S3_AUTHINFO_CREATION_ERROR; + } + // Simply return the header value of x-amz-content-sha256 as the payload hash + // These are the possible cases: + // 1. Actual payload checksum for single chunk upload + // 2. Unsigned payloads for multiple chunks upload + // - UNSIGNED-PAYLOAD + // - STREAMING-UNSIGNED-PAYLOAD-TRAILER + // 3. Signed payloads for multiple chunks upload + // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD + // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER + // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD + // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER + return contentSignatureHeaderValue; } /** diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index 1060f2568c80..e5f49383fcc3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -32,8 +32,8 @@ public final class S3Consts { public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class"; public static final String ENCODING_TYPE = "url"; - // Constants related to Signature calculation - // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html + // Constants related to AWS Signature Version V4 calculation + // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html public static final String X_AMZ_CONTENT_SHA256 = "x-amz-content-sha256"; public static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; @@ -45,6 +45,9 @@ public final class S3Consts { public static final String STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"; + public static final String AWS_CHUNKED = "aws-chunked"; + public static final String MULTI_CHUNKS_UPLOAD_PREFIX = "STREAMING"; + // Constants related to Range Header public static final String COPY_SOURCE_IF_PREFIX = "x-amz-copy-source-if-"; public static final String COPY_SOURCE_IF_MODIFIED_SINCE = diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index a99bfca73721..6fc9b976913a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -20,15 +20,19 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; -import static org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD; -import static org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER; -import static org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_HMAC_SHA256_PAYLOAD; -import static org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_AWS4_HMAC_SHA256_PAYLOAD_TRAILER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_CHUNKED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.MULTI_CHUNKS_UPLOAD_PREFIX; +import static org.apache.hadoop.ozone.s3.util.S3Consts.STREAMING_UNSIGNED_PAYLOAD_TRAILER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.UNSIGNED_PAYLOAD; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; +import com.google.common.base.Preconditions; +import jakarta.annotation.Nonnull; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; +import java.util.Arrays; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; @@ -37,6 +41,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; /** * Utilities. @@ -131,16 +136,62 @@ public static WebApplicationException wrapOS3Exception(OS3Exception ex) { .build()); } - public static boolean hasSignedPayloadHeader(HttpHeaders headers) { - final String signingAlgorithm = headers.getHeaderString(X_AMZ_CONTENT_SHA256); - if (signingAlgorithm == null) { - return false; + public static boolean hasUnsignedPayload(@Nonnull String amzContentSha256Header) { + Preconditions.checkNotNull(amzContentSha256Header); + return amzContentSha256Header.equals(UNSIGNED_PAYLOAD) || + amzContentSha256Header.equals(STREAMING_UNSIGNED_PAYLOAD_TRAILER); + } + + public static boolean hasMultiChunksPayload(@Nonnull String amzContentSha256Header) { + Preconditions.checkNotNull(amzContentSha256Header); + // Possible values + // - STREAMING-UNSIGNED-PAYLOAD-TRAILER + // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD + // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER + // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD + // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER + return amzContentSha256Header.startsWith(MULTI_CHUNKS_UPLOAD_PREFIX); + } + + public static void validateMultiChunksUpload(HttpHeaders headers, String amzDecodedContentLength, + String resource) throws OS3Exception { + final String contentEncoding = headers.getHeaderString(HttpHeaders.CONTENT_ENCODING); + // "Content-Encoding : aws-chunked" seems to only be sent for SDK V2, so ignore if there is no + // Content-Encoding header + if (contentEncoding == null) { + return; + } + // Amazon S3 supports multiple content encoding values for example "Content-Encoding : aws-chunked,gzip" + // We are only interested on "aws-chunked" + boolean containsAwsChunked = Arrays.asList(contentEncoding.split(",")).contains(AWS_CHUNKED); + if (!containsAwsChunked) { + OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, resource); + ex.setErrorMessage("An error occurred (InvalidArgument) for multi chunks upload: " + + "The " + HttpHeaders.CONTENT_ENCODING + " header does not contain " + AWS_CHUNKED); + throw ex; + } + + if (amzDecodedContentLength == null) { + OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, resource); + ex.setErrorMessage("An error occurred (InvalidArgument) for multi chunks upload: " + + "The " + DECODED_CONTENT_LENGTH_HEADER + " header is not specified"); + throw ex; + } + } + + public static String validateSignatureHeader(HttpHeaders headers, String resource) throws OS3Exception { + String xAmzContentSha256Header = headers.getHeaderString(X_AMZ_CONTENT_SHA256); + if (xAmzContentSha256Header == null) { + OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, resource); + ex.setErrorMessage("An error occurred (InvalidArgument): " + + "The " + X_AMZ_CONTENT_SHA256 + " header is not specified"); + throw ex; } - // Handles both AWS Signature Version 4 (HMAC-256) and AWS Signature Version 4A (ECDSA-P256-SHA256) - return signingAlgorithm.equals(STREAMING_AWS4_HMAC_SHA256_PAYLOAD) || - signingAlgorithm.equals(STREAMING_AWS4_HMAC_SHA256_PAYLOAD_TRAILER) || - signingAlgorithm.equals(STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD) || - signingAlgorithm.equals(STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER); + return xAmzContentSha256Header; + } + + public static boolean eol(int prev, int curr) { + return prev == 13 && curr == 10; } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java index cf4334ab4d40..870684098863 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java @@ -27,90 +27,209 @@ import org.junit.jupiter.api.Test; /** - * Test input stream parsing with signatures. + * Test {@link SignedChunksInputStream}. */ public class TestSignedChunksInputStream { @Test - public void emptyfile() throws IOException { - InputStream is = fileContent("0;chunk-signature" - + - "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40"); - String result = IOUtils.toString(is, UTF_8); - assertEquals("", result); - - is = fileContent("0;chunk-signature" - + - "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" - + "\n"); - result = IOUtils.toString(is, UTF_8); - assertEquals("", result); + void testEmptyFile() throws IOException { + try (InputStream is = wrapContent("0;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n")) { + assertEquals("", IOUtils.toString(is, UTF_8)); + } } @Test - public void singlechunk() throws IOException { + void testEmptyFileWithTrailer() throws IOException { + try (InputStream is = wrapContent("0;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "x-amz-checksum-crc32c:sOO8/Q==\r\n" + + "x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n")) { + assertEquals("", IOUtils.toString(is, UTF_8)); + } + } + + @Test + void testEmptyFileWithoutEnd() throws IOException { + try (InputStream is = wrapContent("0;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40")) { + assertEquals("", IOUtils.toString(is, UTF_8)); + } + } + + @Test + void testSingleChunk() throws IOException { + //test simple read() + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890\r\n")) { + assertEquals("1234567890", IOUtils.toString(is, UTF_8)); + } + + //test read(byte[],int,int) + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890\r\n")) { + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890\r\n")) { + byte[] bytes = new byte[10]; + int readLength = IOUtils.read(is, bytes, 0, 10); + assertEquals(10, readLength); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + } + + @Test + void testSingleChunkWithTrailer() throws IOException { //test simple read() - InputStream is = fileContent("0A;chunk-signature" - + - "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" - + "\n1234567890\r\n"); - String result = IOUtils.toString(is, UTF_8); - assertEquals("1234567890", result); + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890\r\n" + + "0;chunk-signature=signature\r\n" + + "x-amz-checksum-crc32c:sOO8/Q==\r\n" + + "x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n")) { + assertEquals("1234567890", IOUtils.toString(is, UTF_8)); + } + + //test read(byte[],int,int) + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890\r\n" + + "0;chunk-signature=signature\r\n" + + "x-amz-checksum-crc32c:sOO8/Q==\r\n" + + "x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n")) { + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890\r\n" + + "0;chunk-signature=signature\r\n" + + "x-amz-checksum-crc32c:sOO8/Q==\r\n" + + "x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n")) { + byte[] bytes = new byte[10]; + int readLength = IOUtils.read(is, bytes, 0, 10); + assertEquals(10, readLength); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + } + @Test + void testSingleChunkWithoutEnd() throws IOException { + //test simple read() + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890")) { + assertEquals("1234567890", IOUtils.toString(is, UTF_8)); + } //test read(byte[],int,int) - is = fileContent("0A;chunk-signature" - + - "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" - + "\n1234567890\r\n"); - byte[] bytes = new byte[10]; - IOUtils.read(is, bytes, 0, 10); - assertEquals("1234567890", - new String(bytes, UTF_8)); + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890")) { + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0A;chunk-signature" + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r\n" + + "1234567890")) { + byte[] bytes = new byte[15]; + int readLength = IOUtils.read(is, bytes, 0, 15); + assertEquals(10, readLength); + assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10)); + } } @Test - public void singlechunkwithoutend() throws IOException { + void testMultiChunks() throws IOException { //test simple read() - InputStream is = fileContent("0A;chunk-signature" - + - "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" - + "\n1234567890"); - String result = IOUtils.toString(is, UTF_8); - assertEquals("1234567890", result); + try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n" + + "1234567890\r\n" + + "05;chunk-signature=signature\r\n" + + "abcde\r\n" + + "0;chunk-signature=signature\r\n")) { + String result = IOUtils.toString(is, UTF_8); + assertEquals("1234567890abcde", result); + } //test read(byte[],int,int) - is = fileContent("0A;chunk-signature" - + - "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" - + "\n1234567890"); - byte[] bytes = new byte[10]; - IOUtils.read(is, bytes, 0, 10); - assertEquals("1234567890", - new String(bytes, UTF_8)); + try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n" + + "1234567890\r\n" + + "05;chunk-signature=signature\r\n" + + "abcde\r\n" + + "0;chunk-signature=signature\r\n")) { + byte[] bytes = new byte[15]; + IOUtils.read(is, bytes, 0, 15); + assertEquals("1234567890abcde", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n" + + "1234567890\r\n" + + "05;chunk-signature=signature\r\n" + + "abcde\r\n" + + "0;chunk-signature=signature\r\n")) { + byte[] bytes = new byte[20]; + int readLength = IOUtils.read(is, bytes, 0, 20); + assertEquals(15, readLength); + assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0, 15)); + } } @Test - public void multichunks() throws IOException { + void testMultiChunksWithTrailer() throws Exception { //test simple read() - InputStream is = fileContent("0a;chunk-signature=signature\r\n" + try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n" + "1234567890\r\n" + "05;chunk-signature=signature\r\n" - + "abcde\r\n"); - String result = IOUtils.toString(is, UTF_8); - assertEquals("1234567890abcde", result); + + "abcde\r\n" + + "0;chunk-signature=signature\r\n" + + "x-amz-checksum-crc32c:sOO8/Q==\r\n" + + "x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n")) { + String result = IOUtils.toString(is, UTF_8); + assertEquals("1234567890abcde", result); + } //test read(byte[],int,int) - is = fileContent("0a;chunk-signature=signature\r\n" + try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n" + + "1234567890\r\n" + + "05;chunk-signature=signature\r\n" + + "abcde\r\n" + + "0;chunk-signature=signature\r\n" + + "x-amz-checksum-crc32c:sOO8/Q==\r\n" + + "x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n")) { + byte[] bytes = new byte[15]; + IOUtils.read(is, bytes, 0, 15); + assertEquals("1234567890abcde", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0a;chunk-signature=signature\r\n" + "1234567890\r\n" + "05;chunk-signature=signature\r\n" - + "abcde\r\n"); - byte[] bytes = new byte[15]; - IOUtils.read(is, bytes, 0, 15); - assertEquals("1234567890abcde", - new String(bytes, UTF_8)); + + "abcde\r\n" + + "0;chunk-signature=signature\r\n" + + "x-amz-checksum-crc32c:sOO8/Q==\r\n" + + "x-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n")) { + byte[] bytes = new byte[20]; + int readLength = IOUtils.read(is, bytes, 0, 20); + assertEquals(15, readLength); + assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0, 15)); + } } - private InputStream fileContent(String content) { + private InputStream wrapContent(String content) { return new SignedChunksInputStream( new ByteArrayInputStream(content.getBytes(UTF_8))); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUnsignedChunkInputStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUnsignedChunkInputStream.java new file mode 100644 index 000000000000..0f468923cfa3 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUnsignedChunkInputStream.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +/** + * Test {@link UnsignedChunksInputStream}. + */ +public class TestUnsignedChunkInputStream { + + @Test + void testEmptyFile() throws IOException { + try (InputStream is = wrapContent("0\r\n")) { + assertEquals("", IOUtils.toString(is, UTF_8)); + } + } + + @Test + void testEmptyFileWithTrailer() throws IOException { + try (InputStream is = wrapContent("0\r\n" + + "x-amz-checksum-crc64nvme:AAAAAAAAAAA=\r\n")) { + assertEquals("", IOUtils.toString(is, UTF_8)); + } + } + + @Test + public void testEmptyFileWithoutEnd() throws IOException { + try (InputStream is = wrapContent("0\r\n" + + "x-amz-checksum-crc64nvme:AAAAAAAAAAA=")) { + assertEquals("", IOUtils.toString(is, UTF_8)); + } + } + + @Test + void testSingleChunk() throws IOException { + //test simple read() + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0\r\n")) { + assertEquals("1234567890", IOUtils.toString(is, UTF_8)); + } + + //test read(byte[],int,int) + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0\r\n")) { + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0\r\n")) { + byte[] bytes = new byte[15]; + int readLength = IOUtils.read(is, bytes, 0, 15); + assertEquals(10, readLength); + assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10)); + } + } + + @Test + void testSingleChunkWithTrailer() throws IOException { + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0\r\n" + + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) { + assertEquals("1234567890", IOUtils.toString(is, UTF_8)); + } + + //test read(byte[],int,int) + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0\r\n" + + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) { + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0\r\n" + + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) { + byte[] bytes = new byte[15]; + int readLength = IOUtils.read(is, bytes, 0, 15); + assertEquals(10, readLength); + assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10)); + } + } + + @Test + void testSingleChunkWithoutEnd() throws IOException { + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0")) { + assertEquals("1234567890", IOUtils.toString(is, UTF_8)); + } + //test read(byte[],int,int) + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0")) { + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + assertEquals("1234567890", new String(bytes, UTF_8)); + } + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0A\r\n" + + "1234567890\r\n" + + "0")) { + byte[] bytes = new byte[15]; + int readLength = IOUtils.read(is, bytes, 0, 10); + assertEquals(10, readLength); + assertEquals("1234567890", new String(bytes, UTF_8).substring(0, 10)); + } + } + + @Test + void testMultiChunks() throws IOException { + //test simple read() + try (InputStream is = wrapContent("0a\r\n" + + "1234567890\r\n" + + "05\r\n" + + "abcde\r\n" + + "0\r\n")) { + String result = IOUtils.toString(is, UTF_8); + assertEquals("1234567890abcde", result); + } + + //test read(byte[],int,int) + try (InputStream is = wrapContent("0a\r\n" + + "1234567890\r\n" + + "05\r\n" + + "abcde\r\n" + + "0\r\n")) { + byte[] bytes = new byte[15]; + IOUtils.read(is, bytes, 0, 15); + assertEquals("1234567890abcde", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0a\r\n" + + "1234567890\r\n" + + "05\r\n" + + "abcde\r\n" + + "0\r\n")) { + byte[] bytes = new byte[20]; + int readLength = IOUtils.read(is, bytes, 0, 20); + assertEquals(15, readLength); + assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0, 15)); + } + } + + @Test + void testMultiChunksWithTrailer() throws IOException { + //test simple read() + try (InputStream is = wrapContent("0a\r\n" + + "1234567890\r\n" + + "05\r\n" + + "abcde\r\n" + + "0\r\n" + + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n")) { + String result = IOUtils.toString(is, UTF_8); + assertEquals("1234567890abcde", result); + } + + //test read(byte[],int,int) + try (InputStream is = wrapContent("0a\r\n" + + "1234567890\r\n" + + "05\r\n" + + "abcde\r\n" + + "0\r\n" + + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\n")) { + byte[] bytes = new byte[15]; + IOUtils.read(is, bytes, 0, 15); + assertEquals("1234567890abcde", new String(bytes, UTF_8)); + } + + //test read(byte[],int,int) with length parameter larger than the payload + try (InputStream is = wrapContent("0a\r\n" + + "1234567890\r\n" + + "05\r\n" + + "abcde\r\n" + + "0\r\n" + + "x-amz-checksum-crc64nvme:2wstOANdZ/o=\n")) { + byte[] bytes = new byte[20]; + int readLength = IOUtils.read(is, bytes, 0, 20); + assertEquals(15, readLength); + assertEquals("1234567890abcde", new String(bytes, UTF_8).substring(0, 15)); + } + } + + private InputStream wrapContent(String content) { + return new UnsignedChunksInputStream( + new ByteArrayInputStream(content.getBytes(UTF_8))); + } + +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java index 0800b81bb7e9..30be715b5305 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java @@ -19,6 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -52,6 +53,8 @@ public void setUp() throws Exception { client.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET); HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index 46a141df74b5..fde336f48079 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -64,6 +65,8 @@ public void setUp() throws Exception { when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); rest = EndpointBuilder.newObjectEndpointBuilder() .setHeaders(headers) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java index f1321820f15a..5189ddf39aaf 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_MODIFIED_SINCE; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_UNMODIFIED_SINCE; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; @@ -121,6 +122,8 @@ public static void setUp() throws Exception { HttpHeaders headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); REST.setHeaders(headers); REST.setClient(CLIENT); @@ -434,6 +437,8 @@ private void setHeaders(Map additionalHeaders) { HttpHeaders headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); additionalHeaders .forEach((k, v) -> when(headers.getHeaderString(k)).thenReturn(v)); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 9bf5f27ddd4b..3e772f8b8bf7 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -83,6 +84,7 @@ public void init() throws OS3Exception, IOException { client.getObjectStore().createS3Bucket(BUCKET_NAME); headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); rest = EndpointBuilder.newObjectEndpointBuilder() .setClient(client) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 4f22bd418447..673fb6a75635 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -30,6 +30,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_KEY_LENGTH_LIMIT; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_NUM_LIMIT; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_VALUE_LENGTH_LIMIT; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.apache.hadoop.ozone.s3.util.S3Utils.urlEncode; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -127,6 +128,7 @@ void setup() throws IOException { clientStub.getObjectStore().createS3Bucket(DEST_BUCKET_NAME); headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); // Create PutObject and setClient to OzoneClientStub objectEndpoint = EndpointBuilder.newObjectEndpointBuilder() @@ -208,6 +210,7 @@ void testPutObjectContentLengthForStreaming() @Test public void testPutObjectWithTags() throws IOException, OS3Exception { HttpHeaders headersWithTags = Mockito.mock(HttpHeaders.class); + when(headersWithTags.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); when(headersWithTags.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2"); ByteArrayInputStream body = @@ -232,6 +235,7 @@ public void testPutObjectWithOnlyTagKey() throws Exception { ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); HttpHeaders headerWithOnlyTagKey = Mockito.mock(HttpHeaders.class); + when(headerWithOnlyTagKey.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); // Try to send with only the key (no value) when(headerWithOnlyTagKey.getHeaderString(TAG_HEADER)).thenReturn("tag1"); objectEndpoint.setHeaders(headerWithOnlyTagKey); @@ -252,6 +256,7 @@ public void testPutObjectWithDuplicateTagKey() throws Exception { ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); HttpHeaders headersWithDuplicateTagKey = Mockito.mock(HttpHeaders.class); + when(headersWithDuplicateTagKey.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); when(headersWithDuplicateTagKey.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag1=value2"); objectEndpoint.setHeaders(headersWithDuplicateTagKey); try { @@ -270,6 +275,7 @@ public void testPutObjectWithLongTagKey() throws Exception { ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); HttpHeaders headersWithLongTagKey = Mockito.mock(HttpHeaders.class); + when(headersWithLongTagKey.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); String longTagKey = StringUtils.repeat('k', TAG_KEY_LENGTH_LIMIT + 1); when(headersWithLongTagKey.getHeaderString(TAG_HEADER)).thenReturn(longTagKey + "=value1"); objectEndpoint.setHeaders(headersWithLongTagKey); @@ -289,6 +295,7 @@ public void testPutObjectWithLongTagValue() throws Exception { ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); HttpHeaders headersWithLongTagValue = Mockito.mock(HttpHeaders.class); + when(headersWithLongTagValue.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); objectEndpoint.setHeaders(headersWithLongTagValue); String longTagValue = StringUtils.repeat('v', TAG_VALUE_LENGTH_LIMIT + 1); when(headersWithLongTagValue.getHeaderString(TAG_HEADER)).thenReturn("tag1=" + longTagValue); @@ -308,6 +315,7 @@ public void testPutObjectWithTooManyTags() throws Exception { ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); HttpHeaders headersWithTooManyTags = Mockito.mock(HttpHeaders.class); + when(headersWithTooManyTags.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); StringBuilder sb = new StringBuilder(); for (int i = 0; i < TAG_NUM_LIMIT + 1; i++) { sb.append(String.format("tag%d=value%d", i, i)); @@ -579,6 +587,7 @@ public void testCopyObjectMessageDigestResetDuringException() throws IOException public void testCopyObjectWithTags() throws IOException, OS3Exception { // Put object in to source bucket HttpHeaders headersForPut = Mockito.mock(HttpHeaders.class); + when(headersForPut.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); when(headersForPut.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2"); ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); @@ -600,6 +609,7 @@ public void testCopyObjectWithTags() throws IOException, OS3Exception { // Copy object without x-amz-tagging-directive (default to COPY) String destKey = "key=value/2"; HttpHeaders headersForCopy = Mockito.mock(HttpHeaders.class); + when(headersForCopy.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); when(headersForCopy.getHeaderString(COPY_SOURCE_HEADER)).thenReturn( BUCKET_NAME + "/" + urlEncode(sourceKeyName)); @@ -738,6 +748,7 @@ void testDirectoryCreationOverFile() throws IOException, OS3Exception { @Test public void testPutEmptyObject() throws IOException, OS3Exception { HttpHeaders headersWithTags = Mockito.mock(HttpHeaders.class); + when(headersWithTags.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); String emptyString = ""; ByteArrayInputStream body = new ByteArrayInputStream(emptyString.getBytes(UTF_8)); objectEndpoint.setHeaders(headersWithTags); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java index 1d3363487939..04c4bfd65c10 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java @@ -25,6 +25,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -83,6 +84,8 @@ public void init() throws OS3Exception, IOException { body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); // Create a key with object tags Mockito.when(headers.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2"); + Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); rest.put(BUCKET_NAME, KEY_WITH_TAG, CONTENT.length(), 1, null, null, null, body); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java index 8b5ffcb3b2c1..c4eb4c25ff87 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; @@ -61,6 +62,8 @@ public void init() throws OS3Exception, IOException { client.getObjectStore().createS3Bucket(BUCKET_NAME); HttpHeaders headers = Mockito.mock(HttpHeaders.class); + Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); rest = EndpointBuilder.newObjectEndpointBuilder() .setClient(client) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java index de698116f531..02b71e8772c4 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.doThrow; @@ -75,6 +76,7 @@ void setup() throws IOException, OS3Exception { clientStub.getObjectStore().createS3Bucket(BUCKET_NAME); HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); // Create PutObject and setClient to OzoneClientStub objectEndpoint = EndpointBuilder.newObjectEndpointBuilder() @@ -83,7 +85,7 @@ void setup() throws IOException, OS3Exception { .setHeaders(headers) .build(); - + ByteArrayInputStream body = new ByteArrayInputStream("".getBytes(UTF_8)); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java index 7a3faf6bd53a..4981069528a8 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -71,6 +72,8 @@ public void setUp() throws Exception { HttpHeaders headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); rest = EndpointBuilder.newObjectEndpointBuilder() .setHeaders(headers) @@ -148,6 +151,8 @@ public void testPartUploadWithIncorrectUploadID() throws Exception { public void testPartUploadStreamContentLength() throws IOException, OS3Exception { HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); ObjectEndpoint objectEndpoint = EndpointBuilder.newObjectEndpointBuilder() .setHeaders(headers) .setClient(client) @@ -206,6 +211,8 @@ public void testPartUploadMessageDigestResetDuringException() throws IOException HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java index 2e3025e0e96b..4b2d8a49efb9 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUploadWithStream.java @@ -20,6 +20,7 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -58,6 +59,8 @@ public void setUp() throws Exception { HttpHeaders headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); OzoneConfiguration conf = new OzoneConfiguration(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 0e4319d6956b..9b34b6ec86ac 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -19,6 +19,7 @@ import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -87,6 +88,8 @@ public void setup() { when(client.getObjectStore()).thenReturn(objectStore); when(client.getConfiguration()).thenReturn(conf); headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); clientProtocol = mock(ClientProtocol.class); S3GatewayMetrics.create(conf); when(client.getProxy()).thenReturn(clientProtocol); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java index 1fcb5d8d406d..02026c2ef9ce 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -69,6 +70,7 @@ public void setUp() throws Exception { client.getObjectStore().createS3Bucket(S3BUCKET); HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); OzoneConfiguration conf = new OzoneConfiguration(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java index 53a8736bddec..63465ef7552e 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.apache.hadoop.ozone.s3.util.S3Utils.urlEncode; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -97,6 +98,8 @@ public void setup() throws Exception { headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("mockSignature"); keyEndpoint.setHeaders(headers); metrics = bucketEndpoint.getMetrics(); From 8f7b6368aa7766bd231b4be17c436621ec924996 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 7 May 2025 22:04:28 +0800 Subject: [PATCH 02/11] Standardize both signed and unsigned implementation --- .../org/apache/hadoop/ozone/s3/SignedChunksInputStream.java | 2 +- .../org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java index a6af04db3e96..1e0ac0344cb4 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java @@ -157,7 +157,7 @@ public int read(byte[] b, int off, int len) throws IOException { // if we encounter this chunk isFinalChunkEncountered = true; } - if (isFinalChunkEncountered || remainingData < 0) { + if (isFinalChunkEncountered || remainingData == -1) { break; } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java index 7cc44334db7b..8e5d6b141f5f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java @@ -153,7 +153,7 @@ public int read(byte[] b, int off, int len) throws IOException { // if we encounter this chunk isFinalChunkEncountered = true; } - if (remainingData <= 0) { + if (isFinalChunkEncountered || remainingData == -1) { break; } } From 0ffee0d64766e642155f520d713d9349a3485501 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Fri, 9 May 2025 15:13:24 +0800 Subject: [PATCH 03/11] Extract duplicated methods and add some comments --- .../ozone/s3/endpoint/ObjectEndpoint.java | 100 +++++++++++------- .../apache/hadoop/ozone/s3/util/S3Utils.java | 7 ++ 2 files changed, 67 insertions(+), 40 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 923e6221e71a..84cfeb845d5a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -104,6 +104,7 @@ import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; import javax.xml.bind.DatatypeConverter; +import net.jcip.annotations.Immutable; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -308,26 +309,10 @@ public Response put( } // Normal put object - final String amzContentSha256Header = validateSignatureHeader(headers, keyPath); - InputStream chunkInputStream; - if (hasMultiChunksPayload(amzContentSha256Header)) { - validateMultiChunksUpload(headers, amzDecodedLength, keyPath); - if (hasUnsignedPayload(amzContentSha256Header)) { - chunkInputStream = new UnsignedChunksInputStream(body); - } else { - chunkInputStream = new SignedChunksInputStream(body); - } - length = Long.parseLong(amzDecodedLength); - } else { - // Single chunk upload: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - // Possible x-amz-content-sha256 values - // - Actual payload checksum value: For signed payload - // - UNSIGNED-PAYLOAD: For unsigned payload - chunkInputStream = body; - } - - // DigestInputStream is used for ETag calculation - digestInputStream = new DigestInputStream(chunkInputStream, getMessageDigestInstance()); + S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(body, + length, amzDecodedLength, keyPath); + digestInputStream = chunkInputStreamInfo.getDigestInputStream(); + length = chunkInputStreamInfo.getEffectiveLength(); Map customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders()); @@ -974,26 +959,11 @@ private Response createMultipartKey(OzoneVolume volume, String bucket, String copyHeader = null; DigestInputStream digestInputStream = null; try { - final String amzContentSha256Header = validateSignatureHeader(headers, key); - InputStream chunkInputStream; - if (hasMultiChunksPayload(amzContentSha256Header)) { - String amzDecodedLength = headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER); - validateMultiChunksUpload(headers, amzDecodedLength, key); - if (hasUnsignedPayload(amzContentSha256Header)) { - chunkInputStream = new UnsignedChunksInputStream(body); - } else { - chunkInputStream = new SignedChunksInputStream(body); - } - length = Long.parseLong(amzDecodedLength); - } else { - // Single chunk upload: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - // Possible x-amz-content-sha256 values - // - Actual payload checksum value: For signed payload - // - UNSIGNED-PAYLOAD: For unsigned payload - chunkInputStream = body; - } - // DigestInputStream is used for ETag calculation - digestInputStream = new DigestInputStream(chunkInputStream, getMessageDigestInstance()); + String amzDecodedLength = headers.getHeaderString(DECODED_CONTENT_LENGTH_HEADER); + S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo( + body, length, amzDecodedLength, key); + digestInputStream = chunkInputStreamInfo.getDigestInputStream(); + length = chunkInputStreamInfo.getEffectiveLength(); copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER); String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER); @@ -1559,4 +1529,54 @@ private int getIOBufferSize(long fileLength) { return fileLength < bufferSize ? (int) fileLength : bufferSize; } } + + /** + * Create a {@link S3ChunkInputStreamInfo} that contains the necessary information to handle + * the S3 chunk upload. + */ + private S3ChunkInputStreamInfo getS3ChunkInputStreamInfo( + InputStream body, long contentLength, String amzDecodedLength, String keyPath) throws OS3Exception { + final String amzContentSha256Header = validateSignatureHeader(headers, keyPath); + final InputStream chunkInputStream; + final long effectiveLength; + if (hasMultiChunksPayload(amzContentSha256Header)) { + validateMultiChunksUpload(headers, amzDecodedLength, keyPath); + if (hasUnsignedPayload(amzContentSha256Header)) { + chunkInputStream = new UnsignedChunksInputStream(body); + } else { + chunkInputStream = new SignedChunksInputStream(body); + } + effectiveLength = Long.parseLong(amzDecodedLength); + } else { + // Single chunk upload: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + // Possible x-amz-content-sha256 header values + // - Actual payload checksum value: For signed payload + // - UNSIGNED-PAYLOAD: For unsigned payload + chunkInputStream = body; + effectiveLength = contentLength; + } + + // DigestInputStream is used for ETag calculation + DigestInputStream digestInputStream = new DigestInputStream(chunkInputStream, getMessageDigestInstance()); + return new S3ChunkInputStreamInfo(digestInputStream, effectiveLength); + } + + @Immutable + static final class S3ChunkInputStreamInfo { + private final DigestInputStream digestInputStream; + private final long effectiveLength; + + S3ChunkInputStreamInfo(DigestInputStream digestInputStream, long effectiveLength) { + this.digestInputStream = digestInputStream; + this.effectiveLength = effectiveLength; + } + + public DigestInputStream getDigestInputStream() { + return digestInputStream; + } + + public long getEffectiveLength() { + return effectiveLength; + } + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index 6fc9b976913a..e673305c3c79 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -144,12 +144,19 @@ public static boolean hasUnsignedPayload(@Nonnull String amzContentSha256Header) public static boolean hasMultiChunksPayload(@Nonnull String amzContentSha256Header) { Preconditions.checkNotNull(amzContentSha256Header); + // Multiple chunk uploads + // - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html + // - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html // Possible values // - STREAMING-UNSIGNED-PAYLOAD-TRAILER // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER + // Currently since all the multi chunks values have x-amz-content-sha256 header value that starts + // with STREAMING, we can use this prefix to differentiates between multi chunks and single chunks upload. + // In the future if there are more multi chunks signature algorithms that has the same prefix, + // this function will be able to handle detect it. return amzContentSha256Header.startsWith(MULTI_CHUNKS_UPLOAD_PREFIX); } From 0cf22f9e90dd7e8bccb6dfaf61745155be7680b8 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 14 May 2025 22:22:00 +0800 Subject: [PATCH 04/11] Use Date#from --- .../hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 2fa4718cf7f7..a8dfb0bda8e8 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -998,10 +998,7 @@ public void testPresignedUrlGet() throws IOException { s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); // Set the presigned URL to expire after one hour. - Date expiration = new java.util.Date(); - long expTimeMillis = Instant.now().plusMillis(1000 * 60 * 60) - .toEpochMilli(); - expiration.setTime(expTimeMillis); + Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); // Generate the presigned URL GeneratePresignedUrlRequest generatePresignedUrlRequest = From f60db526d8402e5a1228f0b3bed428ebe4ab3e51 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 14 May 2025 22:38:14 +0800 Subject: [PATCH 05/11] Add message in IndexOutOfBoundsException --- .../apache/hadoop/ozone/s3/SignedChunksInputStream.java | 9 +++++---- .../hadoop/ozone/s3/UnsignedChunksInputStream.java | 9 +++++---- .../java/org/apache/hadoop/ozone/s3/util/S3Utils.java | 6 +++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java index 1e0ac0344cb4..6c2a58abf0ed 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -119,10 +120,10 @@ public int read() throws IOException { @Override public int read(byte[] b, int off, int len) throws IOException { - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); + Objects.requireNonNull(b, "b == null"); + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException("Offset=" + off + " and len=" + + len + " don't match the array length of " + b.length); } else if (len == 0) { return 0; } else if (isFinalChunkEncountered) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java index 8e5d6b141f5f..50017a7b4a92 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Objects; /** * Input stream implementation to read body of an unsigned chunked upload. @@ -115,10 +116,10 @@ public int read() throws IOException { @Override public int read(byte[] b, int off, int len) throws IOException { - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); + Objects.requireNonNull(b, "b == null"); + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException("Offset=" + off + " and len=" + + len + " don't match the array length of " + b.length); } else if (len == 0) { return 0; } else if (isFinalChunkEncountered) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index e673305c3c79..415d41487ef3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -27,12 +27,12 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.UNSIGNED_PAYLOAD; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; -import com.google.common.base.Preconditions; import jakarta.annotation.Nonnull; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; import java.util.Arrays; +import java.util.Objects; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; @@ -137,13 +137,13 @@ public static WebApplicationException wrapOS3Exception(OS3Exception ex) { } public static boolean hasUnsignedPayload(@Nonnull String amzContentSha256Header) { - Preconditions.checkNotNull(amzContentSha256Header); + Objects.requireNonNull(amzContentSha256Header); return amzContentSha256Header.equals(UNSIGNED_PAYLOAD) || amzContentSha256Header.equals(STREAMING_UNSIGNED_PAYLOAD_TRAILER); } public static boolean hasMultiChunksPayload(@Nonnull String amzContentSha256Header) { - Preconditions.checkNotNull(amzContentSha256Header); + Objects.requireNonNull(amzContentSha256Header); // Multiple chunk uploads // - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html // - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html From 590676001ab526a46909b1792e24acdd18d964ad Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 15 May 2025 09:43:23 +0800 Subject: [PATCH 06/11] Use trim and streamline aws-chunked validation --- .../main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index 415d41487ef3..d47d40ad9056 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -170,7 +170,9 @@ public static void validateMultiChunksUpload(HttpHeaders headers, String amzDeco } // Amazon S3 supports multiple content encoding values for example "Content-Encoding : aws-chunked,gzip" // We are only interested on "aws-chunked" - boolean containsAwsChunked = Arrays.asList(contentEncoding.split(",")).contains(AWS_CHUNKED); + boolean containsAwsChunked = Arrays.stream(contentEncoding.split(",")) + .map(String::trim) + .anyMatch(AWS_CHUNKED::equals); if (!containsAwsChunked) { OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, resource); ex.setErrorMessage("An error occurred (InvalidArgument) for multi chunks upload: " + From 5a7e13a1427770dd93d9af70f71a452ce3d8f3c9 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 15 May 2025 09:48:19 +0800 Subject: [PATCH 07/11] Validate x-amz-decoded-content-length regardless Content-Encoding header existence --- .../apache/hadoop/ozone/s3/util/S3Utils.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index d47d40ad9056..87a76d72e56b 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -165,19 +165,18 @@ public static void validateMultiChunksUpload(HttpHeaders headers, String amzDeco final String contentEncoding = headers.getHeaderString(HttpHeaders.CONTENT_ENCODING); // "Content-Encoding : aws-chunked" seems to only be sent for SDK V2, so ignore if there is no // Content-Encoding header - if (contentEncoding == null) { - return; - } - // Amazon S3 supports multiple content encoding values for example "Content-Encoding : aws-chunked,gzip" - // We are only interested on "aws-chunked" - boolean containsAwsChunked = Arrays.stream(contentEncoding.split(",")) - .map(String::trim) - .anyMatch(AWS_CHUNKED::equals); - if (!containsAwsChunked) { - OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, resource); - ex.setErrorMessage("An error occurred (InvalidArgument) for multi chunks upload: " + - "The " + HttpHeaders.CONTENT_ENCODING + " header does not contain " + AWS_CHUNKED); - throw ex; + if (contentEncoding != null) { + // Amazon S3 supports multiple content encoding values for example "Content-Encoding : aws-chunked,gzip" + // We are only interested on "aws-chunked" + boolean containsAwsChunked = Arrays.stream(contentEncoding.split(",")) + .map(String::trim) + .anyMatch(AWS_CHUNKED::equals); + if (!containsAwsChunked) { + OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, resource); + ex.setErrorMessage("An error occurred (InvalidArgument) for multi chunks upload: " + + "The " + HttpHeaders.CONTENT_ENCODING + " header does not contain " + AWS_CHUNKED); + throw ex; + } } if (amzDecodedContentLength == null) { From 71f62209284cfd7d4ed5c0dcf97fddfeef541b02 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 17 May 2025 13:06:56 +0800 Subject: [PATCH 08/11] Set pathStyleEnabled for S3Presigner --- .../apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index cb8377aec284..19d03210c474 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -75,6 +75,7 @@ import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; import software.amazon.awssdk.services.s3.model.CompletedPart; @@ -417,6 +418,8 @@ public void testPresignedUrlGet() throws Exception { RequestBody.fromString(content)); try (S3Presigner presigner = S3Presigner.builder() + // TODO: Find a way to retrieve the path style configuration from S3Client instead + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) .endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get()) .region(s3Client.serviceClientConfiguration().region()) .credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build()) { From ba6d710c3930d946163c9716d118a67047cb013b Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Tue, 20 May 2025 22:05:19 +0800 Subject: [PATCH 09/11] Doc fix for input stream --- .../org/apache/hadoop/ozone/s3/SignedChunksInputStream.java | 2 +- .../org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java index 6c2a58abf0ed..f3c825db4e63 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java @@ -83,7 +83,7 @@ public class SignedChunksInputStream extends InputStream { private int remainingData = 0; /** - * Every chunked uploads (multiple chunks) contains a final zero-byte final additional + * Every chunked uploads (multiple chunks) contains an additional final zero-byte * chunk. This can be used as the end-of-file marker. */ private boolean isFinalChunkEncountered = false; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java index 50017a7b4a92..3352cf2208cf 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java @@ -79,7 +79,7 @@ public class UnsignedChunksInputStream extends InputStream { private int remainingData = 0; /** - * Every chunked uploads (multiple chunks) contains a final zero-byte final additional + * Every chunked uploads (multiple chunks) contains an additional final zero-byte * chunk. This can be used as the end-of-file marker. */ private boolean isFinalChunkEncountered = false; From 26ba8ef06f62396dc01d5280e93952d42fda6f54 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Tue, 20 May 2025 22:06:15 +0800 Subject: [PATCH 10/11] Add javadoc to eol function --- .../main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index 87a76d72e56b..2dd9f497d099 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -199,6 +199,13 @@ public static String validateSignatureHeader(HttpHeaders headers, String resourc return xAmzContentSha256Header; } + /** + * Checks if the given pair of bytes represent the end-of-line sequence (\r\n). + * + * @param prev the previous byte value (should be 13 for '\r') + * @param curr the current byte value (should be 10 for '\n') + * @return true if the pair forms a CRLF sequence, false otherwise + */ public static boolean eol(int prev, int curr) { return prev == 13 && curr == 10; } From d9ccedf785abb61c25e45de0873f1be7cf0dc2db Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Tue, 20 May 2025 22:11:40 +0800 Subject: [PATCH 11/11] Fix typo --- .../org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java index 3352cf2208cf..93565b0d820e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java @@ -53,7 +53,7 @@ * *

* Note that there is not actual trailer checksum verification taking place. The InputStream only - * * returns the actual chunk payload from chunked signatures format. + * returns the actual chunk payload from chunked signatures format. *

* * Reference: