From cdf670430de61e4ee53db756ad2b3e9048e5e7dd Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 16 Apr 2024 16:31:57 +0100 Subject: [PATCH 1/2] HDDS-10704. Do not fail read of EC block if the last chunk is empty --- .../hadoop/hdds/scm/storage/BlockInputStream.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index b62415395df4..c3556bb44e90 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -294,10 +294,18 @@ private static void validate(ContainerCommandResponseProto response) throw new IllegalArgumentException("Not GetBlock: response=" + response); } final GetBlockResponseProto b = response.getGetBlock(); + long blockLength = b.getBlockData().getSize(); final List chunks = b.getBlockData().getChunksList(); for (int i = 0; i < chunks.size(); i++) { final ChunkInfo c = chunks.get(i); - if (c.getLen() <= 0) { + // HDDS-10682 caused an empty chunk to get written to the end of some EC blocks. Due to this + // validation, these blocks will not be readable. In the EC case, the empty chunk is always + // the last chunk and the offset is the block length. We can safely ignore this case and not fail. + if (c.getLen() <= 0 && i == chunks.size() - 1 && c.getOffset() == blockLength) { + DatanodeBlockID blockID = b.getBlockData().getBlockID(); + LOG.warn("The last chunk is empty for container/block {}/{} with an offset of the block length. " + + "Likely due to HDDS-10682. This is safe to ignore.", blockID.getContainerID(), blockID.getLocalID()); + } else if (c.getLen() <= 0) { throw new IOException("Failed to get chunkInfo[" + i + "]: len == " + c.getLen()); } From 1b12b885c11028f1175fb7fef0d5d43d56e39033 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 16 Apr 2024 17:21:15 +0100 Subject: [PATCH 2/2] Make variable final --- .../org/apache/hadoop/hdds/scm/storage/BlockInputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index c3556bb44e90..c5487832826a 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -294,7 +294,7 @@ private static void validate(ContainerCommandResponseProto response) throw new IllegalArgumentException("Not GetBlock: response=" + response); } final GetBlockResponseProto b = response.getGetBlock(); - long blockLength = b.getBlockData().getSize(); + final long blockLength = b.getBlockData().getSize(); final List chunks = b.getBlockData().getChunksList(); for (int i = 0; i < chunks.size(); i++) { final ChunkInfo c = chunks.get(i);