From cf5d0320a49ff94731d24e1ea1bf2fbe2eea7f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Thu, 26 Nov 2020 17:05:04 +0100 Subject: [PATCH 1/2] HDDS-4185. Disable IncrementalByteBuffer by default in Ozone Client --- .../hadoop/hdds/scm/OzoneClientConfig.java | 18 ++++++++++++++++++ .../hdds/scm/storage/BlockOutputStream.java | 7 +++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index b3c774a2c22f..91ba6a02ee0a 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@ -54,6 +54,18 @@ public class OzoneClientConfig { tags = ConfigTag.CLIENT) private int streamBufferSize = 4 * 1024 * 1024; + @Config(key = "stream.buffer.increment", + defaultValue = "0", + type = ConfigType.SIZE, + description = "Buffer (defined by ozone.client.stream.buffer.size) " + + "will be incremented with this steps. If zero, the full buffer " + + "will " + + "be created at once. Setting it to a variable between 0 and " + + "ozone.client.stream.buffer.size can reduce the memory usage for " + + "very small keys, but has a performance overhead.", + tags = ConfigTag.CLIENT) + private int bufferIncrement = 0; + @Config(key = "stream.buffer.flush.delay", defaultValue = "true", description = "Default true, when call flush() and determine whether " @@ -118,6 +130,9 @@ private void validate() { Preconditions.checkState(streamBufferFlushSize > 0); Preconditions.checkState(streamBufferMaxSize > 0); + Preconditions.checkArgument(bufferIncrement < streamBufferSize, + "Buffer increment should be smaller than the size of the stream " + + "buffer"); Preconditions.checkState(streamBufferMaxSize % streamBufferFlushSize == 0, "expected max. buffer size (%s) to be a multiple of flush size (%s)", streamBufferMaxSize, streamBufferFlushSize); @@ -209,4 +224,7 @@ public void setChecksumVerify(boolean checksumVerify) { this.checksumVerify = checksumVerify; } + public int getBufferIncrement() { + return bufferIncrement; + } } diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java index e29bbe3663e0..272120bb6972 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java @@ -46,14 +46,13 @@ import org.apache.hadoop.ozone.common.ChecksumData; import org.apache.hadoop.ozone.common.ChunkBuffer; import org.apache.hadoop.ozone.common.OzoneChecksumException; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.putBlockAsync; import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.writeChunkAsync; - -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -281,7 +280,7 @@ private void doFlushOrWatchIfNeeded() throws IOException { private void allocateNewBufferIfNeeded() { if (currentBufferRemaining == 0) { - currentBuffer = bufferPool.allocateBuffer(config.getBytesPerChecksum()); + currentBuffer = bufferPool.allocateBuffer(config.getBufferIncrement()); currentBufferRemaining = currentBuffer.remaining(); } } From 9d70554eac1b885b09c90079df68683ee4ad1652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=2C=20M=C3=A1rton?= Date: Mon, 30 Nov 2020 14:57:33 +0100 Subject: [PATCH 2/2] fix default store size. Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index 91ba6a02ee0a..c2e01487067b 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@ -55,7 +55,7 @@ public class OzoneClientConfig { private int streamBufferSize = 4 * 1024 * 1024; @Config(key = "stream.buffer.increment", - defaultValue = "0", + defaultValue = "0B", type = ConfigType.SIZE, description = "Buffer (defined by ozone.client.stream.buffer.size) " + "will be incremented with this steps. If zero, the full buffer "