From 96f152f708012bf4a083a72f014ec8a9e64e492a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 24 Jan 2017 13:18:50 +0100 Subject: [PATCH 1/7] improved ZSTD_compressBlock_opt_extDict_generic --- lib/compress/zstd_opt.h | 4 ++-- zlibWrapper/.gitignore | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index f071c4f3020..3bbf2eb7d0b 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -825,7 +825,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); - if (match_num > 0 && matches[match_num-1].len > sufficient_len) { + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; last_pos = cur + 1; @@ -835,7 +835,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* set prices using matches at position = cur */ for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; + best_mlen = matches[u].len; while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore index 23d2f3a66ba..6167ca4dac8 100644 --- a/zlibWrapper/.gitignore +++ b/zlibWrapper/.gitignore @@ -22,4 +22,4 @@ zwrapbench *.txt # Directories -minizip/ \ No newline at end of file +minizip/ From 6ad02e7762d790d7d45a9249cb508219aa062336 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 24 Jan 2017 15:01:46 +0100 Subject: [PATCH 2/7] JOB_NUMBER -eq 9 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6bf99f1bfa4..35b25c099f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -159,4 +159,4 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi - - sh -c "$Cmd" + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -eq 9 ]; then sh -c "$Cmd"; fi From 74d2cfdee2fb1b67b1c119420e75811832ff4c53 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 24 Jan 2017 17:42:28 +0100 Subject: [PATCH 3/7] .travis.yml: test jobs 12-15 --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35b25c099f9..9f13272c162 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,6 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" - os: linux - sudo: false - - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" os: linux sudo: false @@ -156,7 +152,12 @@ matrix: - gcc-6 - gcc-6-multilib + # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) + - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" + os: linux + sudo: false + script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi - - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -eq 9 ]; then sh -c "$Cmd"; fi + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi From 3bb010a667d87dd220d25d2fea99fea8edb54c79 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 11:19:35 +0100 Subject: [PATCH 4/7] .travis.yml: optimized order of short tests --- .travis.yml | 39 +++++++++++++++++++++------------------ tests/Makefile | 4 ++-- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f13272c162..dba6ae9aa3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,6 @@ matrix: - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - # Ubuntu 14.04 LTS Server Edition 64 bit - env: Ubu=14.04 Cmd="make aarch64test" dist: trusty sudo: required @@ -108,6 +107,23 @@ matrix: packages: - valgrind + + + # other feature branches => short tests + - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" + os: linux + sudo: false + + - env: Ubu=14.04 Cmd="make -C tests test32" + os: linux + dist: trusty + sudo: required + addons: + apt: + packages: + - libc6-dev-i386 + - gcc-multilib + - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 @@ -127,16 +143,6 @@ matrix: - g++-4.8 - g++-4.8-multilib - - env: Ubu=14.04 Cmd="make -C tests test32" - os: linux - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll" os: linux dist: trusty @@ -152,12 +158,9 @@ matrix: - gcc-6 - gcc-6-multilib - # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" - os: linux - sudo: false - script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi - - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + # dev => normal tests; other feature branches => short tests (number > 11) + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + # master => long tests, as this is the final step towards a Release + - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi diff --git a/tests/Makefile b/tests/Makefile index 15fdc77f9f6..a4282a45066 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -52,8 +52,8 @@ endif VOID = /dev/null ZSTREAM_TESTTIME = -T2mn -FUZZERTEST= -T5mn -ZSTDRTTEST= --test-large-data +FUZZERTEST ?= -T5mn +ZSTDRTTEST = --test-large-data .PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest From c4874aab4c64a75ff531c96162c9d1e5be83cc33 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 11:57:28 +0100 Subject: [PATCH 5/7] .travis.yml: different tests for "master" branch --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dba6ae9aa3d..ffe759c4a76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -161,6 +161,6 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # dev => normal tests; other feature branches => short tests (number > 11) - - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ "$TRAVIS_BRANCH" = "dev" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + - if [ "$TRAVIS_BRANCH" != "master" ] && [ "$TRAVIS_BRANCH" = "dev" ] || [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi # master => long tests, as this is the final step towards a Release - - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi + - if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi From 64fa2dbc5ece4054f88b2f2c761336ab37c0f64f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 13:02:33 +0100 Subject: [PATCH 6/7] Fixed https://github.com/facebook/zstd/issues/232 --- .travis.yml | 2 +- programs/fileio.c | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ffe759c4a76..c5a38779169 100644 --- a/.travis.yml +++ b/.travis.yml @@ -161,6 +161,6 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') # dev => normal tests; other feature branches => short tests (number > 11) - - if [ "$TRAVIS_BRANCH" != "master" ] && [ "$TRAVIS_BRANCH" = "dev" ] || [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ]; then sh -c "$Cmd"; fi + - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ] && [ "$TRAVIS_BRANCH" != "master" ]; then sh -c "$Cmd"; fi # master => long tests, as this is the final step towards a Release - if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi diff --git a/programs/fileio.c b/programs/fileio.c index a112cc0499b..9c3c2b7af69 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -361,6 +361,11 @@ static int FIO_compressFilename_srcFile(cRess_t ress, DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } + if (!UTIL_doesFileExists(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); + return 1; + } + ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ @@ -719,6 +724,10 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } + if (!UTIL_doesFileExists(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); + return 1; + } srcFile = FIO_openSrcFile(srcFileName); if (srcFile==0) return 1; From 5022a18d5161f76d709771ff67168b501607578e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 25 Jan 2017 13:11:26 +0100 Subject: [PATCH 7/7] improved #232 fix --- programs/fileio.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9c3c2b7af69..430cc07ebd8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -138,6 +138,10 @@ static FILE* FIO_openSrcFile(const char* srcFileName) f = stdin; SET_BINARY_MODE(stdin); } else { + if (!UTIL_doesFileExists(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); + return NULL; + } f = fopen(srcFileName, "rb"); if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); } @@ -361,10 +365,6 @@ static int FIO_compressFilename_srcFile(cRess_t ress, DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } - if (!UTIL_doesFileExists(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); - return 1; - } ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ @@ -724,10 +724,6 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } - if (!UTIL_doesFileExists(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); - return 1; - } srcFile = FIO_openSrcFile(srcFileName); if (srcFile==0) return 1;