From 3c215220e321b85763bb242d47184711919b951e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 25 Jan 2023 16:29:34 -0800 Subject: [PATCH 1/2] modify cli-test logic : ignore stderr message by default Previously, cli-test would, by default, check that a stderr output is strictly identical to a saved outcome. When there was no instructions on how to interpret stderr, it would default to requiring it to be empty. There are many tests cases though where stderr content doesn't matter, and we are mainly interested in the return code of the cli. For these cases, it was possible to set a .ignore document, which would instruct to ignore stderr content. This PR update the logic, to make .ignore the default. When willing to check that stderr content is empty, one must now add an empty .strict file. This will allow status message to evolve without triggering many cli-tests errors. This is especially important when some of these status include compression results, which may change as a result of compression optimizations. It also makes it easier to add new tests which only care about the CLI's return code. --- tests/cli-tests/run.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/cli-tests/run.py b/tests/cli-tests/run.py index 8b4e21a53ec..45af5124b31 100755 --- a/tests/cli-tests/run.py +++ b/tests/cli-tests/run.py @@ -395,12 +395,10 @@ def _check_output(self, out_name: str) -> None: return self._check_output_exact(out_name, read_file(exact_name), exact_name) elif os.path.exists(glob_name): return self._check_output_glob(out_name, read_file(glob_name)) - elif os.path.exists(ignore_name): + else: check_name = f"check_{out_name}" self._success[check_name] = True self._message[check_name] = f"{out_name} ignored!" - else: - return self._check_output_exact(out_name, bytes(), exact_name) def _check_stderr(self) -> None: """Checks the stderr output against the expectation.""" @@ -738,4 +736,3 @@ def setup_zstd_symlink_dir(zstd_symlink_dir: str, zstd: str) -> None: sys.exit(0) else: sys.exit(1) - From 82ca00811aa297c1f00f549facc70787fdb59d64 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Jan 2023 11:43:27 -0800 Subject: [PATCH 2/2] change logic when stderr is not console : don't update progress status but keep warnings and final operation statement. updated tests/cli-tests/ accordingly --- programs/zstdcli.c | 5 +- tests/cli-tests/basic/memlimit.sh | 14 +++--- tests/cli-tests/compression/levels.sh | 44 ++++++++--------- .../compression/levels.sh.stderr.exact | 48 +++++++++---------- tests/cli-tests/compression/multi-threaded.sh | 16 +++---- .../multi-threaded.sh.stderr.exact | 10 ++++ .../pass-through.sh.stderr.exact | 1 + .../dictionaries/dictionary-mismatch.sh | 2 +- .../dictionary-mismatch.sh.stderr.exact | 3 +- .../progress/no-progress.sh.stderr.glob | 6 +++ 10 files changed, 83 insertions(+), 66 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index ce2b96a8c32..39f8b34fe2c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1500,9 +1500,8 @@ int main(int argCount, const char* argv[]) hasStdout = outFileName && !strcmp(outFileName,stdoutmark); if (hasStdout && (g_displayLevel==2)) g_displayLevel=1; - /* when stderr is not the console, do not pollute it with status updates - * Note : the below code actually also silence more stuff, including completion report. */ - if (!UTIL_isConsole(stderr) && (g_displayLevel==2)) g_displayLevel=1; + /* when stderr is not the console, do not pollute it with progress updates (unless requested) */ + if (!UTIL_isConsole(stderr) && (progress!=FIO_ps_always)) progress=FIO_ps_never; FIO_setProgressSetting(progress); /* don't remove source files when output is stdout */; diff --git a/tests/cli-tests/basic/memlimit.sh b/tests/cli-tests/basic/memlimit.sh index 61a5fc084cc..88d734d0d9d 100755 --- a/tests/cli-tests/basic/memlimit.sh +++ b/tests/cli-tests/basic/memlimit.sh @@ -15,25 +15,25 @@ zstd --memory=32asbdf file && die "Should not allow bogus suffix" println "+ zstd --memory=hello file" zstd --memory=hello file && die "Should not allow non-numeric parameter" println "+ zstd --memory=1 file" -zstd --memory=1 file && die "Should allow numeric parameter without suffix" +zstd -q --memory=1 file && die "Should allow numeric parameter without suffix" rm file.zst println "+ zstd --memory=1K file" -zstd --memory=1K file && die "Should allow numeric parameter with expected suffix" +zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix" rm file.zst println "+ zstd --memory=1KB file" -zstd --memory=1KB file && die "Should allow numeric parameter with expected suffix" +zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix" rm file.zst println "+ zstd --memory=1KiB file" -zstd --memory=1KiB file && die "Should allow numeric parameter with expected suffix" +zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix" rm file.zst println "+ zstd --memory=1M file" -zstd --memory=1M file && die "Should allow numeric parameter with expected suffix" +zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix" rm file.zst println "+ zstd --memory=1MB file" -zstd --memory=1MB file && die "Should allow numeric parameter with expected suffix" +zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix" rm file.zst println "+ zstd --memory=1MiB file" -zstd --memory=1MiB file && die "Should allow numeric parameter with expected suffix" +zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix" rm file.zst rm file diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh index 47ac2029f7a..cc2700a3097 100755 --- a/tests/cli-tests/compression/levels.sh +++ b/tests/cli-tests/compression/levels.sh @@ -6,10 +6,10 @@ set -v datagen > file # Compress with various levels and ensure that their sizes are ordered -zstd --fast=10 file -o file-f10.zst -zstd --fast=1 file -o file-f1.zst -zstd -1 file -o file-1.zst -zstd -19 file -o file-19.zst +zstd --fast=10 file -o file-f10.zst -q +zstd --fast=1 file -o file-f1.zst -q +zstd -1 file -o file-1.zst -q +zstd -19 file -o file-19.zst -q zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst @@ -18,15 +18,15 @@ cmp_size -lt file-1.zst file-f1.zst cmp_size -lt file-f1.zst file-f10.zst # Test default levels -zstd --fast file -f +zstd --fast file -f -q cmp file.zst file-f1.zst || die "--fast is not level -1" -zstd -0 file -o file-0.zst -zstd -f file +zstd -0 file -o file-0.zst -q +zstd -f file -q cmp file.zst file-0.zst || die "Level 0 is not the default level" # Test level clamping -zstd -99 file -o file-99.zst +zstd -99 file -o file-99.zst -q cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19" zstd --fast=200000 file -c | zstd -t @@ -34,10 +34,10 @@ zstd -5000000000 -f file && die "Level too large, must fail" ||: zstd --fast=5000000000 -f file && die "Level too large, must fail" ||: # Test setting a level through the environment variable -ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst +ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q +ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q +ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q +ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level" cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level" @@ -45,18 +45,18 @@ cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level" # Test invalid environment clevel is the default level -zstd -f file -ZSTD_CLEVEL=- zstd -f file -o file-env.zst ; cmp file.zst file-env.zst -ZSTD_CLEVEL=+ zstd -f file -o file-env.zst ; cmp file.zst file-env.zst -ZSTD_CLEVEL=a zstd -f file -o file-env.zst ; cmp file.zst file-env.zst -ZSTD_CLEVEL=-a zstd -f file -o file-env.zst ; cmp file.zst file-env.zst -ZSTD_CLEVEL=+a zstd -f file -o file-env.zst ; cmp file.zst file-env.zst -ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst ; cmp file.zst file-env.zst -ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst +zstd -f file -q +ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst # Test environment clevel is overridden by command line -ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst +ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q +ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q cmp file-1.zst file-1-env.zst || die "Environment variable not overridden" cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden" diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact index 5664f854b19..c8fb79c6896 100644 --- a/tests/cli-tests/compression/levels.sh.stderr.exact +++ b/tests/cli-tests/compression/levels.sh.stderr.exact @@ -2,30 +2,31 @@ datagen > file # Compress with various levels and ensure that their sizes are ordered -zstd --fast=10 file -o file-f10.zst -zstd --fast=1 file -o file-f1.zst -zstd -1 file -o file-1.zst -zstd -19 file -o file-19.zst +zstd --fast=10 file -o file-f10.zst -q +zstd --fast=1 file -o file-f1.zst -q +zstd -1 file -o file-1.zst -q +zstd -19 file -o file-19.zst -q zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst +4 files decompressed : 262148 bytes total cmp_size -lt file-19.zst file-1.zst cmp_size -lt file-1.zst file-f1.zst cmp_size -lt file-f1.zst file-f10.zst # Test default levels -zstd --fast file -f +zstd --fast file -f -q cmp file.zst file-f1.zst || die "--fast is not level -1" -zstd -0 file -o file-0.zst -zstd -f file +zstd -0 file -o file-0.zst -q +zstd -f file -q cmp file.zst file-0.zst || die "Level 0 is not the default level" # Test level clamping -zstd -99 file -o file-99.zst -Warning : compression level higher than max, reduced to 19 +zstd -99 file -o file-99.zst -q cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19" zstd --fast=200000 file -c | zstd -t +/*stdin*\ : 65537 bytes zstd -5000000000 -f file && die "Level too large, must fail" ||: error: numeric value overflows 32-bit unsigned int @@ -33,11 +34,10 @@ zstd --fast=5000000000 -f file && die "Level too large, must fail" ||: error: numeric value overflows 32-bit unsigned int # Test setting a level through the environment variable -ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -Warning : compression level higher than max, reduced to 19 +ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q +ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q +ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q +ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level" cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level" @@ -45,25 +45,25 @@ cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level" # Test invalid environment clevel is the default level -zstd -f file -ZSTD_CLEVEL=- zstd -f file -o file-env.zst ; cmp file.zst file-env.zst +zstd -f file -q +ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value -ZSTD_CLEVEL=+ zstd -f file -o file-env.zst ; cmp file.zst file-env.zst +ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value -ZSTD_CLEVEL=a zstd -f file -o file-env.zst ; cmp file.zst file-env.zst +ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value -ZSTD_CLEVEL=-a zstd -f file -o file-env.zst ; cmp file.zst file-env.zst +ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value -ZSTD_CLEVEL=+a zstd -f file -o file-env.zst ; cmp file.zst file-env.zst +ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value -ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst ; cmp file.zst file-env.zst +ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value -ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst +ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large # Test environment clevel is overridden by command line -ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst +ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q +ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q cmp file-1.zst file-1-env.zst || die "Environment variable not overridden" cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden" diff --git a/tests/cli-tests/compression/multi-threaded.sh b/tests/cli-tests/compression/multi-threaded.sh index bd01448e2e8..17a5eb5186b 100755 --- a/tests/cli-tests/compression/multi-threaded.sh +++ b/tests/cli-tests/compression/multi-threaded.sh @@ -3,13 +3,13 @@ set -e # Test multi-threaded flags -zstd --single-thread file -f ; zstd -t file.zst -zstd -T2 -f file ; zstd -t file.zst -zstd --rsyncable -f file ; zstd -t file.zst -zstd -T0 -f file ; zstd -t file.zst -zstd -T0 --auto-threads=logical -f file ; zstd -t file.zst -zstd -T0 --auto-threads=physical -f file; zstd -t file.zst +zstd --single-thread file -f -q ; zstd -t file.zst +zstd -T2 -f file -q ; zstd -t file.zst +zstd --rsyncable -f file -q ; zstd -t file.zst +zstd -T0 -f file -q ; zstd -t file.zst +zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst +zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst # multi-thread decompression warning test -zstd -T0 -f file ; zstd -t file.zst; zstd -T0 -d file.zst -o file3 -zstd -T0 -f file ; zstd -t file.zst; zstd -T2 -d file.zst -o file4 +zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3 +zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4 diff --git a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact index 54d47d8bd70..11daff6ba63 100644 --- a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact +++ b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact @@ -1 +1,11 @@ +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes Warning : decompression does not support multi-threading +file.zst : 65537 bytes diff --git a/tests/cli-tests/decompression/pass-through.sh.stderr.exact b/tests/cli-tests/decompression/pass-through.sh.stderr.exact index f9ac13cb275..62f96ae420e 100644 --- a/tests/cli-tests/decompression/pass-through.sh.stderr.exact +++ b/tests/cli-tests/decompression/pass-through.sh.stderr.exact @@ -1,3 +1,4 @@ +file :230.00% ( 10 B => 23 B, file.zst) zstd: file: unsupported format zstd: file: unsupported format zstd: file: unsupported format diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh b/tests/cli-tests/dictionaries/dictionary-mismatch.sh index 8264ccca5a2..885cac223ad 100755 --- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh +++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh @@ -23,7 +23,7 @@ if [ false ]; then fi set -v -zstd files/0 -D dicts/0 +zstd files/0 -D dicts/0 -q zstd -t files/0.zst -D dicts/0 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: zstd -t files/0.zst && die "Must fail" ||: diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact index 0afea722e98..8896763c1bc 100644 --- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact +++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact @@ -1,5 +1,6 @@ -zstd files/0 -D dicts/0 +zstd files/0 -D dicts/0 -q zstd -t files/0.zst -D dicts/0 +files/0.zst : 1000 bytes zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: files/0.zst : Decoding error (36) : Dictionary mismatch zstd -t files/0.zst && die "Must fail" ||: diff --git a/tests/cli-tests/progress/no-progress.sh.stderr.glob b/tests/cli-tests/progress/no-progress.sh.stderr.glob index b4262517af1..d0f91122769 100644 --- a/tests/cli-tests/progress/no-progress.sh.stderr.glob +++ b/tests/cli-tests/progress/no-progress.sh.stderr.glob @@ -1,15 +1,21 @@ Tests cases where progress information should not be printed args = compress file to file +hello*hello.zst* compress pipe to pipe compress pipe to file +*stdin*hello.zst* compress file to pipe compress 2 files +2 files compressed* decompress file to file +hello.zst* decompress pipe to pipe decompress pipe to file +*stdin* decompress file to pipe decompress 2 files +2 files decompressed* args = --fake-stderr-is-console -q compress file to file