From b9c99f8b607b2ba41ac059654e9728c4dba58aad Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 5 Mar 2024 22:49:18 +0800 Subject: [PATCH 1/4] HDDS-10465. Change ozone.client.bytes.per.checksum default to 16KB --- .../java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java | 6 +++--- .../main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +- hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +- .../ozone/client/checksum/ECBlockChecksumComputer.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index 3042b4d847a0..fae290f43c44 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@ -153,13 +153,13 @@ public enum ChecksumCombineMode { private String checksumType = ChecksumType.CRC32.name(); @Config(key = "bytes.per.checksum", - defaultValue = "1MB", + defaultValue = "16KB", type = ConfigType.SIZE, description = "Checksum will be computed for every bytes per checksum " + "number of bytes and stored sequentially. The minimum value for " - + "this config is 16KB.", + + "this config is 8KB.", tags = ConfigTag.CLIENT) - private int bytesPerChecksum = 1024 * 1024; + private int bytesPerChecksum = 16 * 1024; @Config(key = "verify.checksum", defaultValue = "true", diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index c7867ffdcbee..2e6863da23e4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -455,7 +455,7 @@ public final class OzoneConfigKeys { "hdds.datanode.replication.work.dir"; - public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE = 16 * 1024; + public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE = 8 * 1024; public static final String OZONE_CLIENT_READ_TIMEOUT = "ozone.client.read.timeout"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index fc873f20af69..7bc165195efa 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -841,7 +841,7 @@ ozone.chunk.read.buffer.default.size - 1MB + 16KB OZONE, SCM, CONTAINER, PERFORMANCE The default read buffer size during read chunk operations when checksum diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java index 220bef71491a..fc310bbf48fb 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java @@ -126,7 +126,7 @@ private void computeCompositeCrc() throws IOException { long chunkSize = replicationConfig.getEcChunkSize(); //When EC chunk size is not a multiple of ozone.client.bytes.per.checksum - // (default = 1MB) the last checksum in an EC chunk is only generated for + // (default = 16KB) the last checksum in an EC chunk is only generated for // offset. long bytesPerCrcOffset = chunkSize % bytesPerCrc; From 1b7eb924f4cb6b67800f2feaee39da4decfa6575 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Fri, 15 Mar 2024 16:22:32 +0800 Subject: [PATCH 2/4] fix ECBlockChecksumComputer#computeCompositeCrc --- .../hadoop/ozone/client/checksum/ECBlockChecksumComputer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java index fc310bbf48fb..e15b3bfd283d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java @@ -123,7 +123,7 @@ private void computeCompositeCrc() throws IOException { long bytesPerCrc = firstChunkInfo.getChecksumData().getBytesPerChecksum(); ECReplicationConfig replicationConfig = (ECReplicationConfig) keyInfo.getReplicationConfig(); - long chunkSize = replicationConfig.getEcChunkSize(); + long chunkSize = firstChunkInfo.getLen(); //When EC chunk size is not a multiple of ozone.client.bytes.per.checksum // (default = 16KB) the last checksum in an EC chunk is only generated for From f65aed064336a4c7b725b0d137cf891ca585af49 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Fri, 15 Mar 2024 16:39:38 +0800 Subject: [PATCH 3/4] remove unused code --- .../hadoop/ozone/client/checksum/ECBlockChecksumComputer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java index e15b3bfd283d..b2c30ed9e08f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java @@ -121,8 +121,6 @@ private void computeCompositeCrc() throws IOException { // Bytes required to create a CRC long bytesPerCrc = firstChunkInfo.getChecksumData().getBytesPerChecksum(); - ECReplicationConfig replicationConfig = - (ECReplicationConfig) keyInfo.getReplicationConfig(); long chunkSize = firstChunkInfo.getLen(); //When EC chunk size is not a multiple of ozone.client.bytes.per.checksum From c7ffe5e1fb80ad02b5233cdf3036630716e93d09 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 28 Mar 2024 17:54:23 +0800 Subject: [PATCH 4/4] remove the change to ozone.chunk.read.buffer.default.size --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +- .../ozone/container/keyvalue/interfaces/ChunkManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 7bc165195efa..fc873f20af69 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -841,7 +841,7 @@ ozone.chunk.read.buffer.default.size - 16KB + 1MB OZONE, SCM, CONTAINER, PERFORMANCE The default read buffer size during read chunk operations when checksum diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java index 151c15f35676..aa2cdcb05f5b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java @@ -126,7 +126,7 @@ static int getBufferCapacityForChunkRead(ChunkInfo chunkInfo, } else { // Set buffer capacity to checksum boundary size so that each buffer // corresponds to one checksum. If checksum is NONE, then set buffer - // capacity to default (OZONE_CHUNK_READ_BUFFER_DEFAULT_SIZE_KEY = 64KB). + // capacity to default (OZONE_CHUNK_READ_BUFFER_DEFAULT_SIZE_KEY = 1MB). ChecksumData checksumData = chunkInfo.getChecksumData(); if (checksumData != null) {