From 6e88ca24f33ee6f7086947d035076ac4f92113d6 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 3 Nov 2017 00:32:13 +0100 Subject: [PATCH 1/3] Add a utility to assist in manually testing an exercise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of having to manually enable tests, copy the examples, and clean up, this script automates all that. - Take optional exercise path argument - Express all paths in terms of exercise path - Pass further arguments through to cargo - Use arrays to simplify file/dir preservation and reset - Use trap to ensure reset occurs on ctrl-c - Express check-exercises.sh in terms of bin/test-exercise - Add debug tests to check-exercises in case of remote execution failure - Move $![deny(warnings)] insertion into test-exercise. - Give the feature to people testing the exercise on its own - Clean up properly after - Return the bitwise OR of individual exercises' return values That last one wants some commenting. If a bunch of values fail in different ways, this is of limited utility; your return code won't be 0, and Travis will fail, but you won't get the info you expected from the return codes. However, this does have some useful properties: - if a bunch of things fail in the same way, or only one thing fails, you get Cargo's return code back directly - you discard less overall information than the naïve implementation ( if [ $? != 0 ]; then return_code=1; fi ) - always returns zero or non-zero appropriately without branching --- _test/check-exercises.sh | 90 ++++++++++++++++------------------------ bin/test-exercise | 84 +++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 54 deletions(-) create mode 100755 bin/test-exercise diff --git a/_test/check-exercises.sh b/_test/check-exercises.sh index 6da744e2a..66451a134 100755 --- a/_test/check-exercises.sh +++ b/_test/check-exercises.sh @@ -1,73 +1,55 @@ #!/bin/bash +# test for existence and executability of the test-exercise script +# this depends on that +if [ ! -f "./bin/test-exercise" ]; then + echo "bin/test-exercise does not exist" + exit 1 +fi +if [ ! -x "./bin/test-exercise" ]; then + echo "bin/test-exercise does not have its executable bit set" + exit 1 +fi + # In DENYWARNINGS mode, do not set -e so that we run all tests. # This allows us to see all warnings. if [ -z "$DENYWARNINGS" ]; then set -e fi +# can't benchmark with a stable compiler; to bench, use +# $ BENCHMARK=1 rustup run nightly _test/check-exercises.sh if [ -n "$BENCHMARK" ]; then files=exercises/*/benches else files=exercises/*/tests fi -tmp=${TMPDIR:-/tmp/} -mkdir "${tmp}exercises" - -exitcode=0 - +return_code=0 # An exercise worth testing is defined here as any top level directory with # a 'tests' directory for exercise in $files; do - # This assumes that exercises are only one directory deep - # and that the primary module is named the same as the directory - directory=$(dirname "${exercise}"); - - workdir=$(mktemp -d "${tmp}${directory}.XXXXXXXXXX") - - cp -R -T $directory $workdir - - # Run in subshell to change workdir without affecting current workdir. - ( - cd $workdir - - cp example.rs src/lib.rs - - # Overwrite empty Cargo.toml if an example specific file exists - if [ -f Cargo-example.toml ]; then - cp Cargo-example.toml Cargo.toml - fi - - # Forcibly strip all "ignore" statements from the testing files - for test in tests/*.rs; do - sed -i '/\[ignore\]/d' $test - done - - # Run benchmarks instead of tests when enabled. - if [ -n "$BENCHMARK" ]; then - cargo bench - elif [ -n "$DENYWARNINGS" ]; then - sed -i -e '1i #![deny(warnings)]' src/lib.rs - - # No-run mode so we see no test output. - # Quiet mode so we see no compile output - # (such as "Compiling"/"Downloading"). - # Compiler errors will still be shown though. - # Both flags are necessary to keep things quiet. - cargo test --quiet --no-run - else - # Run the test and get the status - cargo test - fi - ) - - status=$? - - if [ $status -ne 0 ] - then - exitcode=1 - fi + # This assumes that exercises are only one directory deep + # and that the primary module is named the same as the directory + directory=$(dirname "${exercise}"); + + if [ -n "$DENYWARNINGS" ]; then + # No-run mode so we see no test output. + # Quiet mode so we see no compile output + # (such as "Compiling"/"Downloading"). + # Compiler errors will still be shown though. + # Both flags are necessary to keep things quiet. + ./bin/test-exercise $directory --quiet --no-run + return_code=$(($return_code | $?)) + else + # Run the test and get the status + # We use release mode here because, while it somewhat increases + # the compile time for all exercises, it substantially improves + # the runtime for certain exercises such as alphametics. + # Overall this should be an improvement. + ./bin/test-exercise $directory + return_code=$(($return_code | $?)) + fi done -exit $exitcode +exit $return_code diff --git a/bin/test-exercise b/bin/test-exercise new file mode 100755 index 000000000..cccc6577f --- /dev/null +++ b/bin/test-exercise @@ -0,0 +1,84 @@ +#!/bin/bash +# Test an exercise + +# which exercise are we testing right now? +# if we were passed an argument, that should be the +# exercise directory. Otherwise, assume we're in +# it currently. +if [ $# -ge 1 ]; then + exercise=$1 + # if this script is called with arguments, it will pass through + # any beyond the first to cargo. Note that we can only get a + # free default argument if no arguments at all were passed, + # so if you are in the exercise directory and want to pass any + # arguments to cargo, you need to include the local path first. + # I.e. to test in release mode: + # $ test-exercise . --release + shift 1 +else + exercise='.' +fi + +# what cargo command will we use? +if [ -n "$BENCHMARK" ]; then + command="bench" +else + command="test" +fi + +declare -a preserve_files=("src/lib.rs" "Cargo.toml" "Cargo.lock") +declare -a preserve_dirs=("tests") + +# reset instructions +reset () { + for file in ${preserve_files[@]}; do + if [ -f "$exercise/$file.orig" ]; then + mv -f "$exercise/$file.orig" "$exercise/$file" + fi + done + for dir in ${preserve_dirs[@]}; do + if [ -d "$exercise/$dir.orig" ]; then + rm -rf "$exercise/$dir" + mv "$exercise/$dir.orig" "$exercise/$dir" + fi + done +} + +# cause the reset to execute when the script exits normally or is killed +trap reset EXIT INT TERM + +# preserve the files and directories we care about +for file in ${preserve_files[@]}; do + if [ -f "$exercise/$file" ]; then + cp "$exercise/$file" "$exercise/$file.orig" + fi +done +for dir in ${preserve_dirs[@]}; do + if [ -d "$exercise/$dir" ]; then + cp -r "$exercise/$dir" "$exercise/$dir.orig" + fi +done + +# Move example files to where Cargo expects them +cp -f "$exercise/example.rs" "$exercise/src/lib.rs" +if [ -f "$exercise/Cargo-example.toml" ]; then + cp -f "$exercise/Cargo-example.toml" "$exercise/Cargo.toml" +fi + +# If deny warnings, insert a deny warnings compiler directive in the header +if [ -n "$DENYWARNINGS" ]; then + sed -i -e '1i #![deny(warnings)]' "$exercise/src/lib.rs" +fi + +# eliminate #[ignore] lines from tests +for test in "$exercise/tests/*.rs"; do + sed -i '/\[ignore\]/d' $test +done + +# run tests from within exercise directory +# (use subshell so we auto-reset to current pwd after) +( + cd $exercise + # this is the last command; its exit code is what's passed on + cargo $command "$@" +) From 9c4221ca230c46b54253e697e913da4587aa0baf Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 8 Nov 2017 02:17:41 +0100 Subject: [PATCH 2/3] Add feature: compile in release mode for exercises with a .meta flag - Compile in release mode if .meta/test-in-release-mode exists - Add note to README documenting the above feature - Add .meta/test-in-release-mode to alphametics, which is the sore spot for debug-mode tests --- README.md | 2 ++ _test/check-exercises.sh | 7 ++++++- exercises/alphametics/.meta/test-in-release-mode | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 exercises/alphametics/.meta/test-in-release-mode diff --git a/README.md b/README.md index a95143608..b2b392a00 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ Note that: - An exercise may contain `.meta/hints.md`. This is optional and will appear after the normal exercise instructions if present. Rust is different in many ways from other languages. This is a place where the differences required for Rust are explained. If it is a large change, you may want to call this out as a comment at the top of `src/lib.rs`, so the user recognises to read this section before starting. +- If the test suite is appreciably sped up by running in release mode, and there is reason to be confident that the example implementation does not contain any overflow errors, consider adding a file `.meta/test-in-release-mode`. This should contain brief comments explaining the situation. + - `README.md` may be [regenerated](https://github.com/exercism/docs/blob/master/maintaining-a-track/regenerating-exercise-readmes.md) from Exercism data. The generator will use the `description.md` from the exercise directory in the [problem-specifications repository](https://github.com/exercism/problem-specifications/tree/master/exercises), then any hints in `.meta/hints.md`, then the [Rust-specific instructions](https://github.com/exercism/rust/blob/master/config/exercise-readme-insert.md). The `## Source` section comes from the `metadata.yml` in the same directory. Convention is that the description of the source remains text and the link is both name and hyperlink of the markdown link. - Be sure to add the exercise to an appropriate place in the `config.json` file. The position in the file determines the order exercises are sent. Generate a unique UUID for the exercise. Current difficuly levels in use are 1, 4, 7 and 10. diff --git a/_test/check-exercises.sh b/_test/check-exercises.sh index 66451a134..f82423e24 100755 --- a/_test/check-exercises.sh +++ b/_test/check-exercises.sh @@ -33,6 +33,11 @@ for exercise in $files; do # and that the primary module is named the same as the directory directory=$(dirname "${exercise}"); + release="" + if [ -z "$BENCHMARK" -a -f "$directory/.meta/test-in-release-mode" ]; then + release="--release" + fi + if [ -n "$DENYWARNINGS" ]; then # No-run mode so we see no test output. # Quiet mode so we see no compile output @@ -47,7 +52,7 @@ for exercise in $files; do # the compile time for all exercises, it substantially improves # the runtime for certain exercises such as alphametics. # Overall this should be an improvement. - ./bin/test-exercise $directory + ./bin/test-exercise $directory $release return_code=$(($return_code | $?)) fi done diff --git a/exercises/alphametics/.meta/test-in-release-mode b/exercises/alphametics/.meta/test-in-release-mode new file mode 100644 index 000000000..6d830acae --- /dev/null +++ b/exercises/alphametics/.meta/test-in-release-mode @@ -0,0 +1,2 @@ +Takes a very long time to test in debug mode. +Example implementation not known to encounter overflow errors. From 0691ef36506e76f3c511511ded49ffddcf12bb79 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 8 Nov 2017 09:25:13 +0100 Subject: [PATCH 3/3] Remove extraneous ';' from script --- _test/check-exercises.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_test/check-exercises.sh b/_test/check-exercises.sh index f82423e24..2364acf50 100755 --- a/_test/check-exercises.sh +++ b/_test/check-exercises.sh @@ -31,7 +31,7 @@ return_code=0 for exercise in $files; do # This assumes that exercises are only one directory deep # and that the primary module is named the same as the directory - directory=$(dirname "${exercise}"); + directory=$(dirname "${exercise}") release="" if [ -z "$BENCHMARK" -a -f "$directory/.meta/test-in-release-mode" ]; then