From 04cbd5c2f0426c9f1012933fbe38493a83f95cd2 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 15 Sep 2017 15:13:17 -0500 Subject: [PATCH] LZ4 decompression forward compatibility --- .../segment/data/CompressedObjectStrategy.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java index a2ee783caf1d..5cea096db8ee 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java @@ -30,7 +30,6 @@ import io.druid.java.util.common.logger.Logger; import io.druid.segment.CompressedPools; import net.jpountz.lz4.LZ4Factory; -import net.jpountz.lz4.LZ4FastDecompressor; import net.jpountz.lz4.LZ4SafeDecompressor; import org.apache.commons.lang.ArrayUtils; @@ -264,7 +263,6 @@ public byte[] compress(byte[] bytes) public static class LZ4Decompressor implements Decompressor { private static final LZ4SafeDecompressor lz4Safe = LZ4Factory.fastestInstance().safeDecompressor(); - private static final LZ4FastDecompressor lz4Fast = LZ4Factory.fastestInstance().fastDecompressor(); private static final LZ4Decompressor defaultDecompressor = new LZ4Decompressor(); @Override @@ -286,8 +284,17 @@ public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) @Override public void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize) { - // lz4Fast.decompress does not modify buffer positions - lz4Fast.decompress(in, in.position(), out, out.position(), decompressedSize); + // lz4Safe.decompress does not modify buffer positions. + // Using lz4Safe API for forward-compatibility with https://github.com/druid-io/druid/pull/4762, which doesn't + // always compressed blocks of the same size. + lz4Safe.decompress( + in, + in.position(), + numBytes, + out, + out.position(), + decompressedSize + ); out.limit(out.position() + decompressedSize); } }