From d7af78eebb13fa6eaaf1822b86b53164df5921f7 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Wed, 29 Jun 2022 13:15:51 -0700 Subject: [PATCH 1/2] HDDS-6973. Refactor ObjectEndpoint. --- .../ozone/s3/endpoint/ObjectEndpoint.java | 52 ++++++++----------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index d917fdc113c1..f79d729a42ae 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -199,14 +199,13 @@ public Response put( "Connection", "close").build(); } - output = - bucket.createKey(keyPath, length, replicationConfig, new HashMap<>()); - if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { body = new SignedChunksInputStream(body); } + output = bucket.createKey( + keyPath, length, replicationConfig, new HashMap<>()); IOUtils.copy(body, output); getMetrics().incCreateKeySuccess(); @@ -386,7 +385,7 @@ public Response get( } } - private void addLastModifiedDate( + static void addLastModifiedDate( ResponseBuilder responseBuilder, OzoneKey key) { ZonedDateTime lastModificationTime = key.getModificationTime() @@ -594,6 +593,10 @@ public Response initializeMultipartUpload( private ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket, String storageType) throws OS3Exception { + if (StringUtils.isEmpty(storageType)) { + storageType = S3StorageType.getDefault(ozoneConfiguration).toString(); + } + ReplicationConfig clientConfiguredReplicationConfig = null; String replication = ozoneConfiguration.get(OZONE_REPLICATION); if (replication != null) { @@ -839,6 +842,15 @@ public void setHeaders(HttpHeaders headers) { this.headers = headers; } + static void copy(InputStream src, long srcKeyLen, + String destKey, OzoneBucket destBucket, + ReplicationConfig replication) throws IOException { + try(OzoneOutputStream dest = destBucket.createKey( + destKey, srcKeyLen, replication, new HashMap<>())) { + IOUtils.copy(src, dest); + } + } + private CopyObjectResponse copyObject(String copyHeader, OzoneBucket destBucket, String destkey, @@ -850,9 +862,6 @@ private CopyObjectResponse copyObject(String copyHeader, String sourceBucket = result.getLeft(); String sourceKey = result.getRight(); - OzoneInputStream sourceInputStream = null; - OzoneOutputStream destOutputStream = null; - boolean closed = false; try { // Checking whether we trying to copying to it self. @@ -883,25 +892,15 @@ private CopyObjectResponse copyObject(String copyHeader, OzoneBucket sourceOzoneBucket = getBucket(sourceBucket); - OzoneBucket destOzoneBucket = destBucket; OzoneKeyDetails sourceKeyDetails = sourceOzoneBucket.getKey(sourceKey); long sourceKeyLen = sourceKeyDetails.getDataSize(); - sourceInputStream = sourceOzoneBucket.readKey(sourceKey); - - destOutputStream = destOzoneBucket - .createKey(destkey, sourceKeyLen, replicationConfig, new HashMap<>()); - - IOUtils.copy(sourceInputStream, destOutputStream); - - // Closing here, as if we don't call close this key will not commit in - // OM, and getKey fails. - sourceInputStream.close(); - destOutputStream.close(); - closed = true; + try(OzoneInputStream src = sourceOzoneBucket.readKey(sourceKey)) { + copy(src, sourceKeyLen, destkey, destBucket, replicationConfig); + } - OzoneKeyDetails destKeyDetails = destOzoneBucket.getKey(destkey); + final OzoneKeyDetails destKeyDetails = destBucket.getKey(destkey); getMetrics().incCopyObjectSuccess(); CopyObjectResponse copyObjectResponse = new CopyObjectResponse(); @@ -918,15 +917,6 @@ private CopyObjectResponse copyObject(String copyHeader, destBucket + "/" + destkey, ex); } throw ex; - } finally { - if (!closed) { - if (sourceInputStream != null) { - sourceInputStream.close(); - } - if (destOutputStream != null) { - destOutputStream.close(); - } - } } } @@ -991,7 +981,7 @@ private static OptionalLong parseAndValidateDate(String ozoneDateStr) { } } - private boolean checkCopySourceModificationTime(Long lastModificationTime, + static boolean checkCopySourceModificationTime(Long lastModificationTime, String copySourceIfModifiedSinceStr, String copySourceIfUnmodifiedSinceStr) { long copySourceIfModifiedSince = Long.MIN_VALUE; From 31faae396371a7242eca636e1397a5092763c3bc Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Wed, 29 Jun 2022 13:54:26 -0700 Subject: [PATCH 2/2] fix checkstyle. --- .../org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index f79d729a42ae..3ef48ef1557c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -845,7 +845,7 @@ public void setHeaders(HttpHeaders headers) { static void copy(InputStream src, long srcKeyLen, String destKey, OzoneBucket destBucket, ReplicationConfig replication) throws IOException { - try(OzoneOutputStream dest = destBucket.createKey( + try (OzoneOutputStream dest = destBucket.createKey( destKey, srcKeyLen, replication, new HashMap<>())) { IOUtils.copy(src, dest); } @@ -896,7 +896,7 @@ private CopyObjectResponse copyObject(String copyHeader, OzoneKeyDetails sourceKeyDetails = sourceOzoneBucket.getKey(sourceKey); long sourceKeyLen = sourceKeyDetails.getDataSize(); - try(OzoneInputStream src = sourceOzoneBucket.readKey(sourceKey)) { + try (OzoneInputStream src = sourceOzoneBucket.readKey(sourceKey)) { copy(src, sourceKeyLen, destkey, destBucket, replicationConfig); }