From 55c2f68cbaff36ffd9b6f9154e8b641ac20e6e39 Mon Sep 17 00:00:00 2001 From: guohao1 Date: Thu, 25 May 2023 16:25:45 +0800 Subject: [PATCH 1/6] HDDS-5869. Added support for stream on S3Gateway write path --- .../ozone/s3/endpoint/ObjectEndpoint.java | 37 +++++- .../s3/endpoint/ObjectEndpointStreaming.java | 124 ++++++++++++++++++ 2 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 5c02ae2cf7c9..b0bd8ef4e84b 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -54,6 +54,7 @@ import java.util.OptionalLong; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -93,6 +94,11 @@ import org.apache.commons.lang3.tuple.Pair; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT; @@ -141,6 +147,8 @@ public class ObjectEndpoint extends EndpointBase { https://docs.aws.amazon.com/de_de/AmazonS3/latest/API/API_GetObject.html */ private Map overrideQueryParameter; private int bufferSize; + private int chunkSize; + private boolean datastreamEnabled; public ObjectEndpoint() { overrideQueryParameter = ImmutableMap.builder() @@ -161,6 +169,13 @@ public void init() { bufferSize = (int) ozoneConfiguration.getStorageSize( OZONE_S3G_CLIENT_BUFFER_SIZE_KEY, OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT, StorageUnit.BYTES); + chunkSize = (int) ozoneConfiguration.getStorageSize( + OZONE_SCM_CHUNK_SIZE_KEY, + OZONE_SCM_CHUNK_SIZE_DEFAULT, + StorageUnit.BYTES); + datastreamEnabled = ozoneConfiguration.getBoolean( + DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT); } /** @@ -203,6 +218,13 @@ public Response put( ReplicationConfig replicationConfig = getReplicationConfig(bucket, storageType); + boolean enableEC = false; + if ((replicationConfig != null && + replicationConfig.getReplicationType() == EC) || + bucket.getReplicationConfig() instanceof ECReplicationConfig) { + enableEC = true; + } + if (copyHeader != null) { //Copy object, as copy source available. s3GAction = S3GAction.COPY_OBJECT; @@ -233,11 +255,18 @@ public Response put( .equals(headers.getHeaderString("x-amz-content-sha256"))) { body = new SignedChunksInputStream(body); } + long putLength = 0; + if (datastreamEnabled && !enableEC) { + getMetrics().updatePutKeyMetadataStats(startNanos); + putLength = ObjectEndpointStreaming + .put(bucket, keyPath, length, replicationConfig, chunkSize, body); + } else { + output = getClientProtocol().createKey(volume.getName(), bucketName, + keyPath, length, replicationConfig, customMetadata); + getMetrics().updatePutKeyMetadataStats(startNanos); + putLength = IOUtils.copyLarge(body, output); + } - output = getClientProtocol().createKey(volume.getName(), bucketName, - keyPath, length, replicationConfig, customMetadata); - getMetrics().updatePutKeyMetadataStats(startNanos); - long putLength = IOUtils.copyLarge(body, output); getMetrics().incPutKeySuccessLength(putLength); return Response.ok().status(HttpStatus.SC_OK) .build(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java new file mode 100644 index 000000000000..e7436a28c216 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST; + +/** + * Key level rest endpoints for Streaming. + */ +final class ObjectEndpointStreaming { + + private static final Logger LOG = + LoggerFactory.getLogger(ObjectEndpointStreaming.class); + + private ObjectEndpointStreaming() { + } + + public static long put(OzoneBucket bucket, String keyPath, + long length, ReplicationConfig replicationConfig, + int chunkSize, InputStream body) + throws IOException, OS3Exception { + + try { + Map keyMetadata = new HashMap<>(); + return putKeyWithStream(bucket, keyPath, + length, chunkSize, replicationConfig, keyMetadata, body); + } catch (IOException ex) { + LOG.error("Exception occurred in PutObject", ex); + if (ex instanceof OMException) { + if (((OMException) ex).getResult() == + OMException.ResultCodes.NOT_A_FILE) { + OS3Exception os3Exception = S3ErrorTable.newError(INVALID_REQUEST, + keyPath); + os3Exception.setErrorMessage("An error occurred (InvalidRequest) " + + "when calling the PutObject/MPU PartUpload operation: " + + OZONE_OM_ENABLE_FILESYSTEM_PATHS + " is enabled Keys are" + + " considered as Unix Paths. Path has Violated FS Semantics " + + "which caused put operation to fail."); + throw os3Exception; + } else if ((((OMException) ex).getResult() == + OMException.ResultCodes.PERMISSION_DENIED)) { + throw S3ErrorTable.newError(S3ErrorTable.ACCESS_DENIED, keyPath); + } + } + throw ex; + } + } + + public static long putKeyWithStream(OzoneBucket bucket, + String keyPath, + long length, + int bufferSize, + ReplicationConfig replicationConfig, + Map keyMetadata, + InputStream body) + throws IOException { + long writeLen = 0; + try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath, + length, replicationConfig, keyMetadata)) { + writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length); + } + return writeLen; + } + + private static long writeToStreamOutput(OzoneDataStreamOutput streamOutput, + InputStream body, int bufferSize, + long length) + throws IOException { + byte[] buffer = new byte[bufferSize]; + ByteBuffer writeByteBuffer; + long total = 0; + do { + int realBufferSize = (int) (length - total); + if (realBufferSize > 0 && realBufferSize < bufferSize) { + buffer = new byte[realBufferSize]; + } + int nn = body.read(buffer); + if (nn == -1) { + break; + } else if (nn != bufferSize) { + byte[] subBuffer = new byte[nn]; + System.arraycopy(buffer, 0, subBuffer, 0, nn); + writeByteBuffer = ByteBuffer.wrap(subBuffer, 0, nn); + } else { + writeByteBuffer = ByteBuffer.wrap(buffer, 0, nn); + } + streamOutput.write(writeByteBuffer, 0, nn); + total += nn; + } while (total != length); + return total; + } +} \ No newline at end of file From 9bc0073a72bfae01e0150140c8d8ade8fb6c47d4 Mon Sep 17 00:00:00 2001 From: guohao1 Date: Sat, 24 Jun 2023 16:04:38 +0800 Subject: [PATCH 2/6] checkstyle --- .../apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index e7436a28c216..d294226f23c7 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.ozone.s3.endpoint; import org.apache.hadoop.hdds.client.ReplicationConfig; From 2ff641808a58a17e6a17fbc51a057549e24a1267 Mon Sep 17 00:00:00 2001 From: guohao1 Date: Sat, 24 Jun 2023 16:52:47 +0800 Subject: [PATCH 3/6] checkstyle --- .../hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index d294226f23c7..ac666548d997 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -120,4 +120,4 @@ private static long writeToStreamOutput(OzoneDataStreamOutput streamOutput, } while (total != length); return total; } -} \ No newline at end of file +} From d55fe409a1ef5a58bd8307f8c0db4bbd24945a3e Mon Sep 17 00:00:00 2001 From: guohao1 Date: Sat, 24 Jun 2023 23:09:45 +0800 Subject: [PATCH 4/6] metadata --- .../org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 3 ++- .../hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index b0bd8ef4e84b..c3994a567ab2 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -259,7 +259,8 @@ public Response put( if (datastreamEnabled && !enableEC) { getMetrics().updatePutKeyMetadataStats(startNanos); putLength = ObjectEndpointStreaming - .put(bucket, keyPath, length, replicationConfig, chunkSize, body); + .put(bucket, keyPath, length, replicationConfig, chunkSize, + customMetadata, body); } else { output = getClientProtocol().createKey(volume.getName(), bucketName, keyPath, length, replicationConfig, customMetadata); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index ac666548d997..68b43cb35c9f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.HashMap; import java.util.Map; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; @@ -48,11 +47,11 @@ private ObjectEndpointStreaming() { public static long put(OzoneBucket bucket, String keyPath, long length, ReplicationConfig replicationConfig, - int chunkSize, InputStream body) + int chunkSize, Map keyMetadata, + InputStream body) throws IOException, OS3Exception { try { - Map keyMetadata = new HashMap<>(); return putKeyWithStream(bucket, keyPath, length, chunkSize, replicationConfig, keyMetadata, body); } catch (IOException ex) { From 56083c7744d9b167dfe9695be7f75f06f83b5137 Mon Sep 17 00:00:00 2001 From: guohao1 Date: Mon, 26 Jun 2023 20:32:06 +0800 Subject: [PATCH 5/6] code review --- .../src/main/resources/ozone-default.xml | 8 +++++ .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 5 ++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 8 ++++- .../s3/endpoint/ObjectEndpointStreaming.java | 30 +++++++------------ 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f835ed31288c..d9f9dbeedade 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3151,6 +3151,14 @@ The size of the buffer which is for read block. (4KB by default). + + ozone.s3g.datastream.min.length + OZONE, S3GATEWAY + 1MB + + Use the minimum length of ozone streaming. (1MB by default). + + ssl.server.keystore.keypassword OZONE, SECURITY, MANAGEMENT diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java index 3f80087ebb15..43b6b4d7182c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java @@ -58,6 +58,11 @@ public final class S3GatewayConfigKeys { public static final String OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT = "4KB"; + public static final String OZONE_S3G_DATASTREAM_MIN_LENGTH_KEY = + "ozone.s3g.datastream.min.length"; + public static final String OZONE_S3G_DATASTREAM_MIN_LENGTH_DEFAULT = + "1MB"; + // S3G kerberos, principal config public static final String OZONE_S3G_KERBEROS_KEYTAB_FILE_KEY = "ozone.s3g.kerberos.keytab.file"; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index c3994a567ab2..f7527f98ef30 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -105,6 +105,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DATASTREAM_MIN_LENGTH_DEFAULT; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DATASTREAM_MIN_LENGTH_KEY; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ENTITY_TOO_SMALL; @@ -149,6 +151,7 @@ public class ObjectEndpoint extends EndpointBase { private int bufferSize; private int chunkSize; private boolean datastreamEnabled; + private long datastreamMinLength; public ObjectEndpoint() { overrideQueryParameter = ImmutableMap.builder() @@ -176,6 +179,9 @@ public void init() { datastreamEnabled = ozoneConfiguration.getBoolean( DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT); + datastreamMinLength = (long) ozoneConfiguration.getStorageSize( + OZONE_S3G_DATASTREAM_MIN_LENGTH_KEY, + OZONE_S3G_DATASTREAM_MIN_LENGTH_DEFAULT, StorageUnit.BYTES); } /** @@ -256,7 +262,7 @@ public Response put( body = new SignedChunksInputStream(body); } long putLength = 0; - if (datastreamEnabled && !enableEC) { + if (datastreamEnabled && !enableEC && length > datastreamMinLength) { getMetrics().updatePutKeyMetadataStats(startNanos); putLength = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, chunkSize, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index 68b43cb35c9f..cfe298446778 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -96,27 +96,17 @@ private static long writeToStreamOutput(OzoneDataStreamOutput streamOutput, InputStream body, int bufferSize, long length) throws IOException { - byte[] buffer = new byte[bufferSize]; - ByteBuffer writeByteBuffer; - long total = 0; - do { - int realBufferSize = (int) (length - total); - if (realBufferSize > 0 && realBufferSize < bufferSize) { - buffer = new byte[realBufferSize]; - } - int nn = body.read(buffer); - if (nn == -1) { + final byte[] buffer = new byte[bufferSize]; + long n = 0; + while (n < length) { + final int toRead = Math.toIntExact(Math.min(bufferSize, length - n)); + final int readLength = body.read(buffer, 0, toRead); + if (readLength == -1) { break; - } else if (nn != bufferSize) { - byte[] subBuffer = new byte[nn]; - System.arraycopy(buffer, 0, subBuffer, 0, nn); - writeByteBuffer = ByteBuffer.wrap(subBuffer, 0, nn); - } else { - writeByteBuffer = ByteBuffer.wrap(buffer, 0, nn); } - streamOutput.write(writeByteBuffer, 0, nn); - total += nn; - } while (total != length); - return total; + streamOutput.write(ByteBuffer.wrap(buffer, 0, readLength)); + n += readLength; + } + return n; } } From 99c08571fea119bae9dffbb2acd629a464b24646 Mon Sep 17 00:00:00 2001 From: guohao1 Date: Tue, 27 Jun 2023 12:06:43 +0800 Subject: [PATCH 6/6] code review --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 8 -------- .../org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java | 5 ----- .../apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 8 ++++---- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index d9f9dbeedade..f835ed31288c 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3151,14 +3151,6 @@ The size of the buffer which is for read block. (4KB by default). - - ozone.s3g.datastream.min.length - OZONE, S3GATEWAY - 1MB - - Use the minimum length of ozone streaming. (1MB by default). - - ssl.server.keystore.keypassword OZONE, SECURITY, MANAGEMENT diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java index 43b6b4d7182c..3f80087ebb15 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java @@ -58,11 +58,6 @@ public final class S3GatewayConfigKeys { public static final String OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT = "4KB"; - public static final String OZONE_S3G_DATASTREAM_MIN_LENGTH_KEY = - "ozone.s3g.datastream.min.length"; - public static final String OZONE_S3G_DATASTREAM_MIN_LENGTH_DEFAULT = - "1MB"; - // S3G kerberos, principal config public static final String OZONE_S3G_KERBEROS_KEYTAB_FILE_KEY = "ozone.s3g.kerberos.keytab.file"; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index f7527f98ef30..fe51cf77f2e1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -99,14 +99,14 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY; -import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DATASTREAM_MIN_LENGTH_DEFAULT; -import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DATASTREAM_MIN_LENGTH_KEY; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ENTITY_TOO_SMALL; @@ -180,8 +180,8 @@ public void init() { DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT); datastreamMinLength = (long) ozoneConfiguration.getStorageSize( - OZONE_S3G_DATASTREAM_MIN_LENGTH_KEY, - OZONE_S3G_DATASTREAM_MIN_LENGTH_DEFAULT, StorageUnit.BYTES); + OZONE_FS_DATASTREAM_AUTO_THRESHOLD, + OZONE_FS_DATASTREAM_AUTO_THRESHOLD_DEFAULT, StorageUnit.BYTES); } /**