From ea24b886673a1e154a81d71222d130f65da7a0ed Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Dec 2022 15:32:19 -0800 Subject: [PATCH 1/2] decompressBound() tests fixed an overflow in an intermediate result on 32-bit platform. Checked that the new test catch this bug in 32-bit mode. --- lib/decompress/zstd_decompress.c | 2 +- tests/fuzzer.c | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index e95b8822fe3..57f220d0cdd 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -785,7 +785,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize frameSizeInfo.compressedSize = (size_t)(ip - ipstart); frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) ? zfh.frameContentSize - : nbBlocks * zfh.blockSizeMax; + : (unsigned long long)nbBlocks * zfh.blockSizeMax; return frameSizeInfo; } } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index e15cf0648e7..c3322852317 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -485,6 +485,61 @@ static void test_compressBound(int tnb) DISPLAYLEVEL(3, "OK \n"); } +static void test_decompressBound(int tnb) +{ + DISPLAYLEVEL(3, "test%3i : decompressBound : ", tnb); + + // Simple compression, with size : should provide size; + { const char example[] = "abcd"; + char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))]; + size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0); + CHECK_Z(cSize); + { size_t const dbSize = ZSTD_decompressBound(cBuffer, cSize); + CHECK_Z(dbSize); + CHECK_EQ(dbSize, sizeof(example)); + } } + + // Simple small compression without size : should provide 1 block size + { char cBuffer[ZSTD_COMPRESSBOUND(0)]; + ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 }; + ZSTD_inBuffer in = { NULL, 0, 0 }; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + assert(cctx); + CHECK_Z( ZSTD_initCStream(cctx, 0) ); + CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); + CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); + CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX ); + ZSTD_freeCCtx(cctx); + } + + // Attempt to overflow 32-bit intermediate multiplication result + // This requires dBound >= 4 GB, aka 2^32. + // This requires 2^32 / 2^17 = 2^15 blocks + // => create 2^15 blocks (can be empty, or just 1 byte). + { const char input[] = "a"; + size_t const nbBlocks = (1 << 15) + 1; + size_t blockNb; + size_t const outCapacity = 1 << 18; // large margin + char* const outBuffer = malloc (outCapacity); + ZSTD_outBuffer out = { outBuffer, outCapacity, 0 }; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + assert(cctx); + assert(outBuffer); + CHECK_Z( ZSTD_initCStream(cctx, 0) ); + for (blockNb=0; blockNb 0x100000000LLU /* 4 GB */ ); + ZSTD_freeCCtx(cctx); + free(outBuffer); + } + + DISPLAYLEVEL(3, "OK \n"); +} + static int basicUnitTests(U32 const seed, double compressibility) { size_t const CNBuffSize = 5 MB; @@ -533,6 +588,8 @@ static int basicUnitTests(U32 const seed, double compressibility) test_compressBound(testNb++); + test_decompressBound(testNb++); + DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++); { ZSTD_compressionParameters params; From 2f4238e47ac6dcf923dcf95f129a283cb1cfa642 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Dec 2022 15:58:25 -0800 Subject: [PATCH 2/2] make ZSTD_DECOMPRESSBOUND() compatible with input size 0 for environments with stringent compilation warnings. --- lib/zstd.h | 2 +- tests/fuzzer.c | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 065a1d652a4..09a200af20f 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -228,7 +228,7 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize) * Will produce constant value 0 if srcSize too large. */ #define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U) -#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) > ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ /* ZSTD_isError() : * Most ZSTD_* functions returning a size_t value can be tested for error, diff --git a/tests/fuzzer.c b/tests/fuzzer.c index c3322852317..9d1862077a0 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -494,10 +494,8 @@ static void test_decompressBound(int tnb) char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))]; size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0); CHECK_Z(cSize); - { size_t const dbSize = ZSTD_decompressBound(cBuffer, cSize); - CHECK_Z(dbSize); - CHECK_EQ(dbSize, sizeof(example)); - } } + CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example)); + } // Simple small compression without size : should provide 1 block size { char cBuffer[ZSTD_COMPRESSBOUND(0)];