From a50f991e88fc3b4126b9e6b7ade3c5cc9996b225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gillet?= Date: Sat, 26 Mar 2022 20:59:23 +0900 Subject: [PATCH 01/11] Update elm-test-rs to v2.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 960fd19..6995b66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN curl -L -o elm.gz https://github.com/elm/compiler/releases/download/0.19.1/b && chmod +x bin/elm # Install elm-test-rs -RUN curl -L -o elm-test-rs_linux.tar.gz https://github.com/mpizenberg/elm-test-rs/releases/download/v1.1/elm-test-rs_linux.tar.gz \ +RUN curl -L -o elm-test-rs_linux.tar.gz https://github.com/mpizenberg/elm-test-rs/releases/download/v2.0/elm-test-rs_linux.tar.gz \ && tar xf elm-test-rs_linux.tar.gz \ && mv elm-test-rs bin From 174e6dc28a0985615be10f590425cd3d732aabeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gillet?= Date: Wed, 30 Mar 2022 11:19:12 +0900 Subject: [PATCH 02/11] Add test_data --- .../partial_fail/elm.json | 28 +++++++++++ .../partial_fail/expected_results.json | 46 +++++++++++++++++++ .../src/LuciansLusciousLasagna.elm | 13 ++++++ .../partial_fail/tests/Tests.elm | 37 +++++++++++++++ .../lucians-luscious-lasagna/success/elm.json | 28 +++++++++++ .../success/expected_results.json | 46 +++++++++++++++++++ .../success/src/LuciansLusciousLasagna.elm | 13 ++++++ .../success/tests/Tests.elm | 37 +++++++++++++++ .../test_issues/elm.json | 28 +++++++++++ .../test_issues/expected_results.json | 38 +++++++++++++++ .../src/LuciansLusciousLasagna.elm | 13 ++++++ .../test_issues/tests/Tests.elm | 34 ++++++++++++++ test_data/two-fer/all_fail/elm.json | 28 +++++++++++ .../two-fer/all_fail/expected_results.json | 30 ++++++++++++ test_data/two-fer/all_fail/src/TwoFer.elm | 6 +++ test_data/two-fer/all_fail/tests/Tests.elm | 23 ++++++++++ test_data/two-fer/does_not_compile/elm.json | 28 +++++++++++ .../does_not_compile/expected_results.json | 5 ++ .../two-fer/does_not_compile/src/TwoFer.elm | 6 +++ .../two-fer/does_not_compile/tests/Tests.elm | 23 ++++++++++ test_data/two-fer/missing_solution/elm.json | 28 +++++++++++ .../missing_solution/expected_results.json | 5 ++ .../two-fer/missing_solution/tests/Tests.elm | 23 ++++++++++ test_data/two-fer/missing_tests/elm.json | 28 +++++++++++ .../missing_tests/expected_results.json | 5 ++ .../two-fer/missing_tests/src/TwoFer.elm | 8 ++++ test_data/two-fer/partial_fail/elm.json | 28 +++++++++++ .../partial_fail/expected_results.json | 30 ++++++++++++ test_data/two-fer/partial_fail/src/TwoFer.elm | 6 +++ .../two-fer/partial_fail/tests/Tests.elm | 23 ++++++++++ test_data/two-fer/success/elm.json | 28 +++++++++++ .../two-fer/success/expected_results.json | 30 ++++++++++++ test_data/two-fer/success/src/TwoFer.elm | 8 ++++ test_data/two-fer/success/tests/Tests.elm | 23 ++++++++++ .../test_with_non_literal_name/elm.json | 28 +++++++++++ .../expected_results.json | 5 ++ .../test_with_non_literal_name/src/TwoFer.elm | 8 ++++ .../tests/Tests.elm | 23 ++++++++++ test_data/two-fer/with_debug_message/elm.json | 28 +++++++++++ .../with_debug_message/expected_results.json | 30 ++++++++++++ .../two-fer/with_debug_message/src/TwoFer.elm | 13 ++++++ .../with_debug_message/tests/Tests.elm | 23 ++++++++++ 42 files changed, 941 insertions(+) create mode 100644 test_data/lucians-luscious-lasagna/partial_fail/elm.json create mode 100644 test_data/lucians-luscious-lasagna/partial_fail/expected_results.json create mode 100644 test_data/lucians-luscious-lasagna/partial_fail/src/LuciansLusciousLasagna.elm create mode 100644 test_data/lucians-luscious-lasagna/partial_fail/tests/Tests.elm create mode 100644 test_data/lucians-luscious-lasagna/success/elm.json create mode 100644 test_data/lucians-luscious-lasagna/success/expected_results.json create mode 100644 test_data/lucians-luscious-lasagna/success/src/LuciansLusciousLasagna.elm create mode 100644 test_data/lucians-luscious-lasagna/success/tests/Tests.elm create mode 100644 test_data/lucians-luscious-lasagna/test_issues/elm.json create mode 100644 test_data/lucians-luscious-lasagna/test_issues/expected_results.json create mode 100644 test_data/lucians-luscious-lasagna/test_issues/src/LuciansLusciousLasagna.elm create mode 100644 test_data/lucians-luscious-lasagna/test_issues/tests/Tests.elm create mode 100644 test_data/two-fer/all_fail/elm.json create mode 100644 test_data/two-fer/all_fail/expected_results.json create mode 100644 test_data/two-fer/all_fail/src/TwoFer.elm create mode 100644 test_data/two-fer/all_fail/tests/Tests.elm create mode 100644 test_data/two-fer/does_not_compile/elm.json create mode 100644 test_data/two-fer/does_not_compile/expected_results.json create mode 100644 test_data/two-fer/does_not_compile/src/TwoFer.elm create mode 100644 test_data/two-fer/does_not_compile/tests/Tests.elm create mode 100644 test_data/two-fer/missing_solution/elm.json create mode 100644 test_data/two-fer/missing_solution/expected_results.json create mode 100644 test_data/two-fer/missing_solution/tests/Tests.elm create mode 100644 test_data/two-fer/missing_tests/elm.json create mode 100644 test_data/two-fer/missing_tests/expected_results.json create mode 100644 test_data/two-fer/missing_tests/src/TwoFer.elm create mode 100644 test_data/two-fer/partial_fail/elm.json create mode 100644 test_data/two-fer/partial_fail/expected_results.json create mode 100644 test_data/two-fer/partial_fail/src/TwoFer.elm create mode 100644 test_data/two-fer/partial_fail/tests/Tests.elm create mode 100644 test_data/two-fer/success/elm.json create mode 100644 test_data/two-fer/success/expected_results.json create mode 100644 test_data/two-fer/success/src/TwoFer.elm create mode 100644 test_data/two-fer/success/tests/Tests.elm create mode 100644 test_data/two-fer/test_with_non_literal_name/elm.json create mode 100644 test_data/two-fer/test_with_non_literal_name/expected_results.json create mode 100644 test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm create mode 100644 test_data/two-fer/test_with_non_literal_name/tests/Tests.elm create mode 100644 test_data/two-fer/with_debug_message/elm.json create mode 100644 test_data/two-fer/with_debug_message/expected_results.json create mode 100644 test_data/two-fer/with_debug_message/src/TwoFer.elm create mode 100644 test_data/two-fer/with_debug_message/tests/Tests.elm diff --git a/test_data/lucians-luscious-lasagna/partial_fail/elm.json b/test_data/lucians-luscious-lasagna/partial_fail/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/lucians-luscious-lasagna/partial_fail/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/lucians-luscious-lasagna/partial_fail/expected_results.json b/test_data/lucians-luscious-lasagna/partial_fail/expected_results.json new file mode 100644 index 0000000..426c0bd --- /dev/null +++ b/test_data/lucians-luscious-lasagna/partial_fail/expected_results.json @@ -0,0 +1,46 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "The expected amount of time in the oven is 40 minutes", + "task_id": 1, + "status": "fail", + "message": "\n 40000\n ╷\n │ Expect.equal\n ╵\n 40\n\n", + "output": null, + "test_code": "expectedMinutesInOven\n |> Expect.equal 40" + }, + { + "name": "For 2 layers, the preparation time is 4 minutes", + "task_id": 2, + "status": "pass", + "message": null, + "output": null, + "test_code": "preparationTimeInMinutes 2\n |> Expect.equal 4" + }, + { + "name": "For 5 layers, the preparation time is 10 minutes", + "task_id": 2, + "status": "pass", + "message": null, + "output": null, + "test_code": "preparationTimeInMinutes 5\n |> Expect.equal 10" + }, + { + "name": "For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking", + "task_id": 3, + "status": "pass", + "message": null, + "output": null, + "test_code": "elapsedTimeInMinutes 3 10\n |> Expect.equal 16" + }, + { + "name": "For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking", + "task_id": 3, + "status": "pass", + "message": null, + "output": null, + "test_code": "elapsedTimeInMinutes 6 30\n |> Expect.equal 42" + } + ] +} diff --git a/test_data/lucians-luscious-lasagna/partial_fail/src/LuciansLusciousLasagna.elm b/test_data/lucians-luscious-lasagna/partial_fail/src/LuciansLusciousLasagna.elm new file mode 100644 index 0000000..8b5937a --- /dev/null +++ b/test_data/lucians-luscious-lasagna/partial_fail/src/LuciansLusciousLasagna.elm @@ -0,0 +1,13 @@ +module LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes) + + +expectedMinutesInOven = + 40000 + + +preparationTimeInMinutes layers = + 2 * layers + + +elapsedTimeInMinutes layers passedAlready = + passedAlready + preparationTimeInMinutes layers diff --git a/test_data/lucians-luscious-lasagna/partial_fail/tests/Tests.elm b/test_data/lucians-luscious-lasagna/partial_fail/tests/Tests.elm new file mode 100644 index 0000000..60889b3 --- /dev/null +++ b/test_data/lucians-luscious-lasagna/partial_fail/tests/Tests.elm @@ -0,0 +1,37 @@ +module Tests exposing (tests) + +import Expect +import LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes) +import Test exposing (..) + + +tests : Test +tests = + describe "LuciansLusciousLasagna" + [ describe "1" + [ test "The expected amount of time in the oven is 40 minutes" <| + \_ -> + expectedMinutesInOven + |> Expect.equal 40 + ] + , describe "2" + [ test "For 2 layers, the preparation time is 4 minutes" <| + \_ -> + preparationTimeInMinutes 2 + |> Expect.equal 4 + , test "For 5 layers, the preparation time is 10 minutes" <| + \_ -> + preparationTimeInMinutes 5 + |> Expect.equal 10 + ] + , describe "3" + [ test "For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking" <| + \_ -> + elapsedTimeInMinutes 3 10 + |> Expect.equal 16 + , test "For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking" <| + \_ -> + elapsedTimeInMinutes 6 30 + |> Expect.equal 42 + ] + ] diff --git a/test_data/lucians-luscious-lasagna/success/elm.json b/test_data/lucians-luscious-lasagna/success/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/lucians-luscious-lasagna/success/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/lucians-luscious-lasagna/success/expected_results.json b/test_data/lucians-luscious-lasagna/success/expected_results.json new file mode 100644 index 0000000..ac7ca12 --- /dev/null +++ b/test_data/lucians-luscious-lasagna/success/expected_results.json @@ -0,0 +1,46 @@ +{ + "version": 3, + "status": "pass", + "tests": [ + { + "name": "The expected amount of time in the oven is 40 minutes", + "task_id": 1, + "status": "pass", + "message": null, + "output": null, + "test_code": "expectedMinutesInOven\n |> Expect.equal 40" + }, + { + "name": "For 2 layers, the preparation time is 4 minutes", + "task_id": 2, + "status": "pass", + "message": null, + "output": null, + "test_code": "preparationTimeInMinutes 2\n |> Expect.equal 4" + }, + { + "name": "For 5 layers, the preparation time is 10 minutes", + "task_id": 2, + "status": "pass", + "message": null, + "output": null, + "test_code": "preparationTimeInMinutes 5\n |> Expect.equal 10" + }, + { + "name": "For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking", + "task_id": 3, + "status": "pass", + "message": null, + "output": null, + "test_code": "elapsedTimeInMinutes 3 10\n |> Expect.equal 16" + }, + { + "name": "For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking", + "task_id": 3, + "status": "pass", + "message": null, + "output": null, + "test_code": "elapsedTimeInMinutes 6 30\n |> Expect.equal 42" + } + ] +} diff --git a/test_data/lucians-luscious-lasagna/success/src/LuciansLusciousLasagna.elm b/test_data/lucians-luscious-lasagna/success/src/LuciansLusciousLasagna.elm new file mode 100644 index 0000000..632e5e0 --- /dev/null +++ b/test_data/lucians-luscious-lasagna/success/src/LuciansLusciousLasagna.elm @@ -0,0 +1,13 @@ +module LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes) + + +expectedMinutesInOven = + 40 + + +preparationTimeInMinutes layers = + 2 * layers + + +elapsedTimeInMinutes layers passedAlready = + passedAlready + preparationTimeInMinutes layers diff --git a/test_data/lucians-luscious-lasagna/success/tests/Tests.elm b/test_data/lucians-luscious-lasagna/success/tests/Tests.elm new file mode 100644 index 0000000..60889b3 --- /dev/null +++ b/test_data/lucians-luscious-lasagna/success/tests/Tests.elm @@ -0,0 +1,37 @@ +module Tests exposing (tests) + +import Expect +import LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes) +import Test exposing (..) + + +tests : Test +tests = + describe "LuciansLusciousLasagna" + [ describe "1" + [ test "The expected amount of time in the oven is 40 minutes" <| + \_ -> + expectedMinutesInOven + |> Expect.equal 40 + ] + , describe "2" + [ test "For 2 layers, the preparation time is 4 minutes" <| + \_ -> + preparationTimeInMinutes 2 + |> Expect.equal 4 + , test "For 5 layers, the preparation time is 10 minutes" <| + \_ -> + preparationTimeInMinutes 5 + |> Expect.equal 10 + ] + , describe "3" + [ test "For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking" <| + \_ -> + elapsedTimeInMinutes 3 10 + |> Expect.equal 16 + , test "For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking" <| + \_ -> + elapsedTimeInMinutes 6 30 + |> Expect.equal 42 + ] + ] diff --git a/test_data/lucians-luscious-lasagna/test_issues/elm.json b/test_data/lucians-luscious-lasagna/test_issues/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/lucians-luscious-lasagna/test_issues/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/lucians-luscious-lasagna/test_issues/expected_results.json b/test_data/lucians-luscious-lasagna/test_issues/expected_results.json new file mode 100644 index 0000000..8167df6 --- /dev/null +++ b/test_data/lucians-luscious-lasagna/test_issues/expected_results.json @@ -0,0 +1,38 @@ +{ + "version": 3, + "status": "pass", + "tests": [ + { + "name": "all good", + "task_id": 1, + "status": "pass", + "message": null, + "output": null, + "test_code": "expectedMinutesInOven\n |> Expect.equal 40" + }, + { + "name": "unexpected skip in concept exercise, will be ran anyway", + "task_id": 2, + "status": "pass", + "message": null, + "output": null, + "test_code": "preparationTimeInMinutes 2\n |> Expect.equal 4" + }, + { + "name": "Missing task ID", + "task_id": null, + "status": "pass", + "message": null, + "output": null, + "test_code": "elapsedTimeInMinutes 3 10\n |> Expect.equal 16" + }, + { + "name": "wrong task ID format", + "task_id": null, + "status": "pass", + "message": null, + "output": null, + "test_code": "elapsedTimeInMinutes 6 30\n |> Expect.equal 42" + } + ] +} diff --git a/test_data/lucians-luscious-lasagna/test_issues/src/LuciansLusciousLasagna.elm b/test_data/lucians-luscious-lasagna/test_issues/src/LuciansLusciousLasagna.elm new file mode 100644 index 0000000..632e5e0 --- /dev/null +++ b/test_data/lucians-luscious-lasagna/test_issues/src/LuciansLusciousLasagna.elm @@ -0,0 +1,13 @@ +module LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes) + + +expectedMinutesInOven = + 40 + + +preparationTimeInMinutes layers = + 2 * layers + + +elapsedTimeInMinutes layers passedAlready = + passedAlready + preparationTimeInMinutes layers diff --git a/test_data/lucians-luscious-lasagna/test_issues/tests/Tests.elm b/test_data/lucians-luscious-lasagna/test_issues/tests/Tests.elm new file mode 100644 index 0000000..9874610 --- /dev/null +++ b/test_data/lucians-luscious-lasagna/test_issues/tests/Tests.elm @@ -0,0 +1,34 @@ +module Tests exposing (tests) + +import Expect +import LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes) +import Test exposing (..) + + +tests : Test +tests = + describe "LuciansLusciousLasagna" + [ describe "1" + [ test "all good" <| + \_ -> + expectedMinutesInOven + |> Expect.equal 40 + ] + , describe "2" + [ skip <| + test "unexpected skip in concept exercise, will be ran anyway" <| + \_ -> + preparationTimeInMinutes 2 + |> Expect.equal 4 + ] + , test "Missing task ID" <| + \_ -> + elapsedTimeInMinutes 3 10 + |> Expect.equal 16 + , describe "#3" + [ test "wrong task ID format" <| + \_ -> + elapsedTimeInMinutes 6 30 + |> Expect.equal 42 + ] + ] diff --git a/test_data/two-fer/all_fail/elm.json b/test_data/two-fer/all_fail/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/all_fail/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/all_fail/expected_results.json b/test_data/two-fer/all_fail/expected_results.json new file mode 100644 index 0000000..f4189f2 --- /dev/null +++ b/test_data/two-fer/all_fail/expected_results.json @@ -0,0 +1,30 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "No name given", + "task_id": null, + "status": "fail", + "message": "\n ▼▼▼ ▼▼ ▼▼ ▼ \n \"No name for anyone!\"\n ╷\n │ Expect.equal\n ╵\n \"One for you, one for me.\"\n ▲ ▲▲▲▲ ▲▲▲▲▲▲▲▲ \n\n", + "output": null, + "test_code": "Expect.equal \"One for you, one for me.\" (twoFer Nothing)" + }, + { + "name": "A name given", + "task_id": null, + "status": "fail", + "message": "\n ▼▼▼ ▼▼ ▼▼▼ ▼ \n \"No name for anyone!\"\n ╷\n │ Expect.equal\n ╵\n \"One for Alice, one for me.\"\n ▲ ▲▲▲▲▲▲▲ ▲▲▲▲▲▲▲▲ \n\n", + "output": null, + "test_code": "Expect.equal \"One for Alice, one for me.\" (twoFer (Just \"Alice\"))" + }, + { + "name": "Another name given", + "task_id": null, + "status": "fail", + "message": "\n ▼▼▼ ▼▼ ▼▼▼ ▼ \n \"No name for anyone!\"\n ╷\n │ Expect.equal\n ╵\n \"One for Bob, one for me.\"\n ▲ ▲ ▲▲▲▲ ▲▲▲▲▲▲▲▲ \n\n", + "output": null, + "test_code": "Expect.equal \"One for Bob, one for me.\" (twoFer (Just \"Bob\"))" + } + ] +} diff --git a/test_data/two-fer/all_fail/src/TwoFer.elm b/test_data/two-fer/all_fail/src/TwoFer.elm new file mode 100644 index 0000000..d93c0a5 --- /dev/null +++ b/test_data/two-fer/all_fail/src/TwoFer.elm @@ -0,0 +1,6 @@ +module TwoFer exposing (twoFer) + + +twoFer : Maybe String -> String +twoFer name = + "No name for anyone!" diff --git a/test_data/two-fer/all_fail/tests/Tests.elm b/test_data/two-fer/all_fail/tests/Tests.elm new file mode 100644 index 0000000..f6c7c25 --- /dev/null +++ b/test_data/two-fer/all_fail/tests/Tests.elm @@ -0,0 +1,23 @@ +module Tests exposing (tests) + +import Expect +import String +import Test exposing (..) +import TwoFer exposing (twoFer) + + +tests : Test +tests = + describe "Two-fer" + [ test "No name given" <| + \() -> + Expect.equal "One for you, one for me." (twoFer Nothing) + , skip <| + test "A name given" <| + \() -> + Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) + , skip <| + test "Another name given" <| + \() -> + Expect.equal "One for Bob, one for me." (twoFer (Just "Bob")) + ] diff --git a/test_data/two-fer/does_not_compile/elm.json b/test_data/two-fer/does_not_compile/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/does_not_compile/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/does_not_compile/expected_results.json b/test_data/two-fer/does_not_compile/expected_results.json new file mode 100644 index 0000000..ed0d6ce --- /dev/null +++ b/test_data/two-fer/does_not_compile/expected_results.json @@ -0,0 +1,5 @@ +{ + "version": 3, + "status": "error", + "message": "\nelm-test-rs 2.0.0 for elm 0.19.1\n--------------------------------\n\n-- UNFINISHED DEFINITION -------------------------- /tmp/solution/src/TwoFer.elm\n\nI got stuck while parsing the `twoFer` definition:\n\n4| twoFer : Maybe String -> String\n5| twoFer name =\n ^\nI was expecting to see an expression next. What is it equal to?\n\nHere is a valid definition (with a type annotation) for reference:\n\n greet : String -> String\n greet name =\n \"Hello \" ++ name ++ \"!\"\n\nThe top line (called a \"type annotation\") is optional. You can leave it off if\nyou want. As you get more comfortable with Elm and as your project grows, it\nbecomes more and more valuable to add them though! They work great as\ncompiler-verified documentation, and they often improve error messages!\n\n" +} diff --git a/test_data/two-fer/does_not_compile/src/TwoFer.elm b/test_data/two-fer/does_not_compile/src/TwoFer.elm new file mode 100644 index 0000000..db9f130 --- /dev/null +++ b/test_data/two-fer/does_not_compile/src/TwoFer.elm @@ -0,0 +1,6 @@ +module TwoFer exposing (twoFer) + + +twoFer : Maybe String -> String +twoFer name = + diff --git a/test_data/two-fer/does_not_compile/tests/Tests.elm b/test_data/two-fer/does_not_compile/tests/Tests.elm new file mode 100644 index 0000000..f6c7c25 --- /dev/null +++ b/test_data/two-fer/does_not_compile/tests/Tests.elm @@ -0,0 +1,23 @@ +module Tests exposing (tests) + +import Expect +import String +import Test exposing (..) +import TwoFer exposing (twoFer) + + +tests : Test +tests = + describe "Two-fer" + [ test "No name given" <| + \() -> + Expect.equal "One for you, one for me." (twoFer Nothing) + , skip <| + test "A name given" <| + \() -> + Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) + , skip <| + test "Another name given" <| + \() -> + Expect.equal "One for Bob, one for me." (twoFer (Just "Bob")) + ] diff --git a/test_data/two-fer/missing_solution/elm.json b/test_data/two-fer/missing_solution/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/missing_solution/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/missing_solution/expected_results.json b/test_data/two-fer/missing_solution/expected_results.json new file mode 100644 index 0000000..b81afab --- /dev/null +++ b/test_data/two-fer/missing_solution/expected_results.json @@ -0,0 +1,5 @@ +{ + "version": 3, + "status": "error", + "message": "\nelm-test-rs 2.0.0 for elm 0.19.1\n--------------------------------\n\n-- MODULE NOT FOUND ------------------------------ /tmp/solution/tests/Tests.elm\n\nYou are trying to import a `TwoFer` module:\n\n6| import TwoFer exposing (twoFer)\n ^^^^^^\nI checked the \"dependencies\" and \"source-directories\" listed in your elm.json,\nbut I cannot find it! Maybe it is a typo for one of these names?\n\n Parser\n Runner\n Time\n Bitwise\n\nHint: If it is not a typo, check the \"dependencies\" and \"source-directories\" of\nyour elm.json to make sure all the packages you need are listed there!\n\n" +} diff --git a/test_data/two-fer/missing_solution/tests/Tests.elm b/test_data/two-fer/missing_solution/tests/Tests.elm new file mode 100644 index 0000000..f6c7c25 --- /dev/null +++ b/test_data/two-fer/missing_solution/tests/Tests.elm @@ -0,0 +1,23 @@ +module Tests exposing (tests) + +import Expect +import String +import Test exposing (..) +import TwoFer exposing (twoFer) + + +tests : Test +tests = + describe "Two-fer" + [ test "No name given" <| + \() -> + Expect.equal "One for you, one for me." (twoFer Nothing) + , skip <| + test "A name given" <| + \() -> + Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) + , skip <| + test "Another name given" <| + \() -> + Expect.equal "One for Bob, one for me." (twoFer (Just "Bob")) + ] diff --git a/test_data/two-fer/missing_tests/elm.json b/test_data/two-fer/missing_tests/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/missing_tests/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/missing_tests/expected_results.json b/test_data/two-fer/missing_tests/expected_results.json new file mode 100644 index 0000000..330832b --- /dev/null +++ b/test_data/two-fer/missing_tests/expected_results.json @@ -0,0 +1,5 @@ +{ + "version": 3, + "status": "error", + "message": "sed: can't read tests/Tests.elm: No such file or directory\n" +} diff --git a/test_data/two-fer/missing_tests/src/TwoFer.elm b/test_data/two-fer/missing_tests/src/TwoFer.elm new file mode 100644 index 0000000..b9df217 --- /dev/null +++ b/test_data/two-fer/missing_tests/src/TwoFer.elm @@ -0,0 +1,8 @@ +module TwoFer exposing (twoFer) + + +twoFer : Maybe String -> String +twoFer name = + "One for " + ++ Maybe.withDefault "you" name + ++ ", one for me." diff --git a/test_data/two-fer/partial_fail/elm.json b/test_data/two-fer/partial_fail/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/partial_fail/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/partial_fail/expected_results.json b/test_data/two-fer/partial_fail/expected_results.json new file mode 100644 index 0000000..facd835 --- /dev/null +++ b/test_data/two-fer/partial_fail/expected_results.json @@ -0,0 +1,30 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "No name given", + "task_id": null, + "status": "pass", + "message": null, + "output": null, + "test_code": "Expect.equal \"One for you, one for me.\" (twoFer Nothing)" + }, + { + "name": "A name given", + "task_id": null, + "status": "fail", + "message": "\n ▼▼▼ \n \"One for you, one for me.\"\n ╷\n │ Expect.equal\n ╵\n \"One for Alice, one for me.\"\n ▲▲▲▲▲ \n\n", + "output": null, + "test_code": "Expect.equal \"One for Alice, one for me.\" (twoFer (Just \"Alice\"))" + }, + { + "name": "Another name given", + "task_id": null, + "status": "fail", + "message": "\n ▼ ▼ \n \"One for you, one for me.\"\n ╷\n │ Expect.equal\n ╵\n \"One for Bob, one for me.\"\n ▲ ▲ \n\n", + "output": null, + "test_code": "Expect.equal \"One for Bob, one for me.\" (twoFer (Just \"Bob\"))" + } + ] +} diff --git a/test_data/two-fer/partial_fail/src/TwoFer.elm b/test_data/two-fer/partial_fail/src/TwoFer.elm new file mode 100644 index 0000000..568fcef --- /dev/null +++ b/test_data/two-fer/partial_fail/src/TwoFer.elm @@ -0,0 +1,6 @@ +module TwoFer exposing (twoFer) + + +twoFer : Maybe String -> String +twoFer name = + "One for you, one for me." diff --git a/test_data/two-fer/partial_fail/tests/Tests.elm b/test_data/two-fer/partial_fail/tests/Tests.elm new file mode 100644 index 0000000..f6c7c25 --- /dev/null +++ b/test_data/two-fer/partial_fail/tests/Tests.elm @@ -0,0 +1,23 @@ +module Tests exposing (tests) + +import Expect +import String +import Test exposing (..) +import TwoFer exposing (twoFer) + + +tests : Test +tests = + describe "Two-fer" + [ test "No name given" <| + \() -> + Expect.equal "One for you, one for me." (twoFer Nothing) + , skip <| + test "A name given" <| + \() -> + Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) + , skip <| + test "Another name given" <| + \() -> + Expect.equal "One for Bob, one for me." (twoFer (Just "Bob")) + ] diff --git a/test_data/two-fer/success/elm.json b/test_data/two-fer/success/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/success/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/success/expected_results.json b/test_data/two-fer/success/expected_results.json new file mode 100644 index 0000000..c1aeb90 --- /dev/null +++ b/test_data/two-fer/success/expected_results.json @@ -0,0 +1,30 @@ +{ + "version": 3, + "status": "pass", + "tests": [ + { + "name": "No name given", + "task_id": null, + "status": "pass", + "message": null, + "output": null, + "test_code": "Expect.equal \"One for you, one for me.\" (twoFer Nothing)" + }, + { + "name": "A name given", + "task_id": null, + "status": "pass", + "message": null, + "output": null, + "test_code": "Expect.equal \"One for Alice, one for me.\" (twoFer (Just \"Alice\"))" + }, + { + "name": "Another name given", + "task_id": null, + "status": "pass", + "message": null, + "output": null, + "test_code": "Expect.equal \"One for Bob, one for me.\" (twoFer (Just \"Bob\"))" + } + ] +} diff --git a/test_data/two-fer/success/src/TwoFer.elm b/test_data/two-fer/success/src/TwoFer.elm new file mode 100644 index 0000000..b9df217 --- /dev/null +++ b/test_data/two-fer/success/src/TwoFer.elm @@ -0,0 +1,8 @@ +module TwoFer exposing (twoFer) + + +twoFer : Maybe String -> String +twoFer name = + "One for " + ++ Maybe.withDefault "you" name + ++ ", one for me." diff --git a/test_data/two-fer/success/tests/Tests.elm b/test_data/two-fer/success/tests/Tests.elm new file mode 100644 index 0000000..f6c7c25 --- /dev/null +++ b/test_data/two-fer/success/tests/Tests.elm @@ -0,0 +1,23 @@ +module Tests exposing (tests) + +import Expect +import String +import Test exposing (..) +import TwoFer exposing (twoFer) + + +tests : Test +tests = + describe "Two-fer" + [ test "No name given" <| + \() -> + Expect.equal "One for you, one for me." (twoFer Nothing) + , skip <| + test "A name given" <| + \() -> + Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) + , skip <| + test "Another name given" <| + \() -> + Expect.equal "One for Bob, one for me." (twoFer (Just "Bob")) + ] diff --git a/test_data/two-fer/test_with_non_literal_name/elm.json b/test_data/two-fer/test_with_non_literal_name/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/test_with_non_literal_name/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/test_with_non_literal_name/expected_results.json b/test_data/two-fer/test_with_non_literal_name/expected_results.json new file mode 100644 index 0000000..b79b51f --- /dev/null +++ b/test_data/two-fer/test_with_non_literal_name/expected_results.json @@ -0,0 +1,5 @@ +{ + "version": 3, + "status": "error", + "message": "Number of tests doesn't match number of extracted code snippets. Please report this issue." +} diff --git a/test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm b/test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm new file mode 100644 index 0000000..b9df217 --- /dev/null +++ b/test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm @@ -0,0 +1,8 @@ +module TwoFer exposing (twoFer) + + +twoFer : Maybe String -> String +twoFer name = + "One for " + ++ Maybe.withDefault "you" name + ++ ", one for me." diff --git a/test_data/two-fer/test_with_non_literal_name/tests/Tests.elm b/test_data/two-fer/test_with_non_literal_name/tests/Tests.elm new file mode 100644 index 0000000..491975b --- /dev/null +++ b/test_data/two-fer/test_with_non_literal_name/tests/Tests.elm @@ -0,0 +1,23 @@ +module Tests exposing (tests) + +import Expect +import String +import Test exposing (..) +import TwoFer exposing (twoFer) + + +tests : Test +tests = + describe "Two-fer" + [ test "Normal test" <| + \() -> + Expect.equal "One for you, one for me." (twoFer Nothing) + , skip <| + let + testName = + "Non literal test name, code will not get extracted" + in + test testName <| + \() -> + Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) + ] diff --git a/test_data/two-fer/with_debug_message/elm.json b/test_data/two-fer/with_debug_message/elm.json new file mode 100644 index 0000000..e08a5bb --- /dev/null +++ b/test_data/two-fer/with_debug_message/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/parser": "1.1.0", + "elm/random": "1.0.0", + "elm/regex": "1.0.0", + "elm/time": "1.0.0" + }, + "indirect": {} + }, + "test-dependencies": { + "direct": { + "elm-explorations/test": "1.2.2", + "rtfeldman/elm-iso8601-date-strings": "1.1.3" + }, + "indirect": { + "elm/html": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + } +} diff --git a/test_data/two-fer/with_debug_message/expected_results.json b/test_data/two-fer/with_debug_message/expected_results.json new file mode 100644 index 0000000..b76ff93 --- /dev/null +++ b/test_data/two-fer/with_debug_message/expected_results.json @@ -0,0 +1,30 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "No log since test will pass", + "task_id": null, + "status": "pass", + "message": null, + "output": null, + "test_code": "Expect.equal \"One for you, one for me.\" (twoFer Nothing)" + }, + { + "name": "Will be logged", + "task_id": null, + "status": "fail", + "message": "\n ▼▼▼ \n \"One for you, one for me.\"\n ╷\n │ Expect.equal\n ╵\n \"One for Alice, one for me.\"\n ▲▲▲▲▲ \n\n", + "output": "Log variable: Just \"Alice\"\n\nLog constant: 42\n", + "test_code": "Expect.equal \"One for Alice, one for me.\" (twoFer (Just \"Alice\"))" + }, + { + "name": "Will be logged too", + "task_id": null, + "status": "fail", + "message": "\n ▼ ▼ \n \"One for you, one for me.\"\n ╷\n │ Expect.equal\n ╵\n \"One for Bob, one for me.\"\n ▲ ▲ \n\n", + "output": "Log variable: Just \"Bob\"\n\nLog constant: 42\n", + "test_code": "Expect.equal \"One for Bob, one for me.\" (twoFer (Just \"Bob\"))" + } + ] +} diff --git a/test_data/two-fer/with_debug_message/src/TwoFer.elm b/test_data/two-fer/with_debug_message/src/TwoFer.elm new file mode 100644 index 0000000..d4d3357 --- /dev/null +++ b/test_data/two-fer/with_debug_message/src/TwoFer.elm @@ -0,0 +1,13 @@ +module TwoFer exposing (twoFer) + + +twoFer : Maybe String -> String +twoFer name = + let + _ = + Debug.log "Log variable" name + + _ = + Debug.log "Log constant" 42 + in + "One for you, one for me." diff --git a/test_data/two-fer/with_debug_message/tests/Tests.elm b/test_data/two-fer/with_debug_message/tests/Tests.elm new file mode 100644 index 0000000..9c68c96 --- /dev/null +++ b/test_data/two-fer/with_debug_message/tests/Tests.elm @@ -0,0 +1,23 @@ +module Tests exposing (tests) + +import Expect +import String +import Test exposing (..) +import TwoFer exposing (twoFer) + + +tests : Test +tests = + describe "Two-fer" + [ test "No log since test will pass" <| + \() -> + Expect.equal "One for you, one for me." (twoFer Nothing) + , skip <| + test "Will be logged" <| + \() -> + Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) + , skip <| + test "Will be logged too" <| + \() -> + Expect.equal "One for Bob, one for me." (twoFer (Just "Bob")) + ] From feecbda8435034bd63581f95776c20e468f8b6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gillet?= Date: Wed, 30 Mar 2022 15:05:09 +0900 Subject: [PATCH 03/11] Add smoke tests Some added changes: It now runs tests in /tmp/solution in order to use --read-only docker flag. run.sh now checks for more failure types: * If test file doesn't exist * if number of tests doesn't match number of code snippets elm-test-rs v2.0 seems to return test results in random order, so I changed the jq script to merge test results and code snippets in the order of the snippets. --- .gitignore | 5 ++++ Dockerfile | 3 ++ bin/check_files.sh | 55 ++++++++++++++++++++++++++++++++++++ bin/run-in-docker.sh | 3 ++ bin/run-tests-in-docker.sh | 31 ++++++++++++++++++++ bin/run.sh | 58 ++++++++++++++++++++++---------------- bin/smoke_test.sh | 18 ++++++++++++ 7 files changed, 149 insertions(+), 24 deletions(-) create mode 100755 bin/check_files.sh create mode 100755 bin/run-tests-in-docker.sh create mode 100755 bin/smoke_test.sh diff --git a/.gitignore b/.gitignore index eca1f14..14cd73d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ .netlify elm-stuff + +# smoke test intermediate files +test_code.json +test_results.json +results.json diff --git a/Dockerfile b/Dockerfile index 6995b66..8daf160 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,10 +41,13 @@ RUN cd extract-test-code \ # Pack together things to copy to the runner container COPY bin/run.sh bin/run.sh +COPY bin/smoke_test.sh bin/smoke_test.sh +COPY bin/check_files.sh bin/check_files.sh # Lightweight runner container FROM node:lts-alpine WORKDIR /opt/test-runner +ENV PATH="/opt/test-runner/bin:${PATH}" COPY --from=builder /opt/test-runner/bin bin COPY --from=builder /opt/test-runner/cache.tar . ENTRYPOINT [ "bin/run.sh" ] diff --git a/bin/check_files.sh b/bin/check_files.sh new file mode 100755 index 0000000..50d0328 --- /dev/null +++ b/bin/check_files.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# +# This script is called by bin/smoke_test.hs +# +# It checks that for a given exercise path (given as argument) +# 1. The analyzer created an output file +# 2. There is a reference file available for comparison +# 3. Both files are identical +# +# Otherwise, it fails with an error message. +# + +set -euo pipefail + +exercise=$1 + +function installed { + cmd=$(command -v "${1}") + + [[ -n "${cmd}" ]] && [[ -f "${cmd}" ]] + return ${?} +} + +function die { + >&2 echo "Fatal: ${@}" + exit 1 +} + +function main { + if [[ ! -f "${exercise}/expected_results.json" ]]; then + echo "🔥 ${exercise}: expected expected_results.json to exist 🔥" + exit 1 + fi + + if [[ ! -f "${exercise}/results.json" ]]; then + echo "🔥 ${exercise}: expected results.json to exist on successful run 🔥" + exit 1 + fi + + jq -S . ${exercise}/expected_results.json > /tmp/expected.json + jq -S . ${exercise}/results.json > /tmp/actual.json + if ! diff /tmp/expected.json /tmp/actual.json ;then + echo "🔥 ${exercise}: expected ${exercise}/results.json to equal ${exercise}/expected_results.json on successful run 🔥" + exit 1 + fi + + echo "🏁 ${exercise}: expected files present and correct after successful run 🏁" +} + +# Check for all required dependencies +for dep in diff jq; do + installed "${dep}" || die "Missing '${dep}'" +done + +main "$@"; exit diff --git a/bin/run-in-docker.sh b/bin/run-in-docker.sh index 8313632..afb6c3e 100755 --- a/bin/run-in-docker.sh +++ b/bin/run-in-docker.sh @@ -43,6 +43,9 @@ docker build -t elm-test-runner . # run image passing the arguments mkdir -p "$OUTPUT_DIR" docker run --network none \ + --rm \ + --read-only \ --mount type=bind,src=$INPUT_DIR,dst=/solution \ --mount type=bind,src=$OUTPUT_DIR,dst=/output \ + --mount type=tmpfs,dst=/tmp \ elm-test-runner $SLUG /solution/ /output/ diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh new file mode 100755 index 0000000..7e35f63 --- /dev/null +++ b/bin/run-tests-in-docker.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Synopsis: +# Test the test-runner Docker image by running it against a predefined set of +# solutions with an expected output. +# The test-runner Docker image is built automatically. + +# Output: +# Outputs the diff of the expected test results against the actual test results +# generated by the test runner Docker image. + +# Example: +# ./bin/run-tests-in-docker.sh + +set -e # Make script exit when a command fail. +set -u # Exit on usage of undeclared variable. +# set -x # Trace what gets executed. +set -o pipefail # Catch failures in pipes. + +# build docker image +docker build --rm -t elm-test-runner . + +# run image passing the arguments +docker run \ + --rm \ + --read-only \ + --network none \ + --mount type=bind,src=$(realpath test_data),dst=/opt/test-runner/test_data \ + --mount type=tmpfs,dst=/tmp \ + --entrypoint /opt/test-runner/bin/smoke_test.sh \ + elm-test-runner diff --git a/bin/run.sh b/bin/run.sh index fee10e0..9f3448e 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -10,16 +10,13 @@ SLUG="$1" INPUT_DIR="$2" OUTPUT_DIR="$3" -# Setup a working directory -WORK_DIR=/opt/test-runner/app -cp -rp $INPUT_DIR $WORK_DIR -mkdir -p $WORK_DIR/elm-stuff && rm -rf $WORK_DIR/elm-stuff +# Setup a temporary working directory +WORK_DIR=/tmp/solution +mkdir -p $WORK_DIR && rm -rf $WORK_DIR +cp -r $INPUT_DIR $WORK_DIR tar xf cache.tar -C $WORK_DIR cd $WORK_DIR -# Add bin/ to the path to make available elm and elm-test-rs. -export PATH=/opt/test-runner/bin:${PATH} - # Use the cache in .elm/ by redefining the elm home directory. export ELM_HOME=$WORK_DIR/.elm @@ -30,7 +27,6 @@ LogLevel Error Port 4343 Listen 127.0.0.1 MaxClients 100 -StartServers 1 Filter "filter.conf" FilterDefaultDeny Yes EOT @@ -39,36 +35,50 @@ touch filter.conf export https_proxy=127.0.0.1:4343 tinyproxy -d -c proxy.conf & -# Un-skip all the skipped tests -sed -i 's/skip <|//g' tests/Tests.elm - # Temporarily disable -e mode set +e -elm-test-rs -v --report exercism --offline > $OUTPUT_DIR/results.json 2> stderr.txt +# Un-skip all the skipped tests +sed -i 's/skip <|//g' tests/Tests.elm 2> stderr.txt +STATUS=$? +cat stderr.txt +if [ $STATUS -ne 0 ]; then + jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json + echo "An error occured while replacing test in the test file. " + exit 0 +fi + +elm-test-rs -v --report exercism --offline > test_results.json 2> stderr.txt STATUS=$? cat stderr.txt # elm-test-rs will exit(0) if tests pass, exit(2) if tests fail if [ $STATUS -ne 0 ] && [ $STATUS -ne 2 ]; then - jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json - echo "Finished with error" - exit 0 + jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json + echo "An error occured while running tests with elm-test-rs" + exit 0 fi # Extract test code -cat tests/Tests.elm | node ../bin/cli.js > $OUTPUT_DIR/test_code.json 2> stderr.txt +cat tests/Tests.elm | node /opt/test-runner/bin/cli.js > test_code.json 2> stderr.txt STATUS=$? cat stderr.txt if [ $STATUS -ne 0 ]; then - echo "An error occurred while extracting the test code from the test file." - exit 0 + jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json + echo "An error occurred while extracting the test code from the test file." + exit 0 +fi + +# Check number of tests matches number of extracted code snippets +test_code_length=$(jq 'length' test_code.json) +test_result_length=$(jq '.tests | length' test_results.json) +if [ $test_code_length -ne $test_result_length ] ; then + err="Number of tests doesn't match number of extracted code snippets. Please report this issue." + jq -n --arg m "${err}" '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json + echo $err + exit 0 fi set -e -cd $OUTPUT_DIR -# Merge tests results with extracted test code. -# This rely on the fact that the order is the same in both arrays. -jq -s '[range(.[1]|length) as $i | .[0].tests[$i] + { test_code: .[1][$i].testCode }]' results.json test_code.json > concat.json -jq -s '.[0].tests = .[1] | .[0]' results.json concat.json > results_with_code.json -mv results_with_code.json results.json +# Merge tests results with extracted test code, in the extracted code order. +jq --slurpfile code test_code.json '.tests = (.tests + ($code | .[0] | to_entries | map({order: .key, name: .value.name, test_code: .value.testCode})) | group_by(.name) | map(add) | sort_by(.order) | map(del(.order)))' test_results.json > $OUTPUT_DIR/results.json echo Finished diff --git a/bin/smoke_test.sh b/bin/smoke_test.sh new file mode 100755 index 0000000..88edbe3 --- /dev/null +++ b/bin/smoke_test.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e # Make script exit when a command fail. +set -u # Exit on usage of undeclared variable. +# set -x # Trace what gets executed. +set -o pipefail # Catch failures in pipes. + +for solution in test_data/*/* ; do + echo $solution + slug=$(basename $(dirname $solution)) + solution=$(realpath $solution) + # run tests + bin/run.sh $slug $solution $solution > /dev/null + # check result + bin/check_files.sh $solution +done + +echo "Smoke test finished" From cc1a4592715b0e60a89a8a1d5cb5129732d97035 Mon Sep 17 00:00:00 2001 From: Jie Date: Thu, 31 Mar 2022 22:23:20 +0900 Subject: [PATCH 04/11] Fix comment typos Co-authored-by: Cedd Burge --- bin/check_files.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/check_files.sh b/bin/check_files.sh index 50d0328..e794283 100755 --- a/bin/check_files.sh +++ b/bin/check_files.sh @@ -1,9 +1,9 @@ #!/bin/sh # -# This script is called by bin/smoke_test.hs +# This script is called by bin/smoke_test.sh # # It checks that for a given exercise path (given as argument) -# 1. The analyzer created an output file +# 1. The test runner created an output file # 2. There is a reference file available for comparison # 3. Both files are identical # From 6b6e5445f53ee6933aaedc8c2a0de424c2892bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gillet?= Date: Thu, 31 Mar 2022 22:54:40 +0900 Subject: [PATCH 05/11] Remove overzealous tests --- test_data/two-fer/missing_solution/elm.json | 28 ------------------- .../missing_solution/expected_results.json | 5 ---- .../two-fer/missing_solution/tests/Tests.elm | 23 --------------- test_data/two-fer/missing_tests/elm.json | 28 ------------------- .../missing_tests/expected_results.json | 5 ---- .../two-fer/missing_tests/src/TwoFer.elm | 8 ------ .../test_with_non_literal_name/elm.json | 28 ------------------- .../expected_results.json | 5 ---- .../test_with_non_literal_name/src/TwoFer.elm | 8 ------ .../tests/Tests.elm | 23 --------------- 10 files changed, 161 deletions(-) delete mode 100644 test_data/two-fer/missing_solution/elm.json delete mode 100644 test_data/two-fer/missing_solution/expected_results.json delete mode 100644 test_data/two-fer/missing_solution/tests/Tests.elm delete mode 100644 test_data/two-fer/missing_tests/elm.json delete mode 100644 test_data/two-fer/missing_tests/expected_results.json delete mode 100644 test_data/two-fer/missing_tests/src/TwoFer.elm delete mode 100644 test_data/two-fer/test_with_non_literal_name/elm.json delete mode 100644 test_data/two-fer/test_with_non_literal_name/expected_results.json delete mode 100644 test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm delete mode 100644 test_data/two-fer/test_with_non_literal_name/tests/Tests.elm diff --git a/test_data/two-fer/missing_solution/elm.json b/test_data/two-fer/missing_solution/elm.json deleted file mode 100644 index e08a5bb..0000000 --- a/test_data/two-fer/missing_solution/elm.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "type": "application", - "source-directories": [ - "src" - ], - "elm-version": "0.19.1", - "dependencies": { - "direct": { - "elm/core": "1.0.5", - "elm/json": "1.1.3", - "elm/parser": "1.1.0", - "elm/random": "1.0.0", - "elm/regex": "1.0.0", - "elm/time": "1.0.0" - }, - "indirect": {} - }, - "test-dependencies": { - "direct": { - "elm-explorations/test": "1.2.2", - "rtfeldman/elm-iso8601-date-strings": "1.1.3" - }, - "indirect": { - "elm/html": "1.0.0", - "elm/virtual-dom": "1.0.2" - } - } -} diff --git a/test_data/two-fer/missing_solution/expected_results.json b/test_data/two-fer/missing_solution/expected_results.json deleted file mode 100644 index b81afab..0000000 --- a/test_data/two-fer/missing_solution/expected_results.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "version": 3, - "status": "error", - "message": "\nelm-test-rs 2.0.0 for elm 0.19.1\n--------------------------------\n\n-- MODULE NOT FOUND ------------------------------ /tmp/solution/tests/Tests.elm\n\nYou are trying to import a `TwoFer` module:\n\n6| import TwoFer exposing (twoFer)\n ^^^^^^\nI checked the \"dependencies\" and \"source-directories\" listed in your elm.json,\nbut I cannot find it! Maybe it is a typo for one of these names?\n\n Parser\n Runner\n Time\n Bitwise\n\nHint: If it is not a typo, check the \"dependencies\" and \"source-directories\" of\nyour elm.json to make sure all the packages you need are listed there!\n\n" -} diff --git a/test_data/two-fer/missing_solution/tests/Tests.elm b/test_data/two-fer/missing_solution/tests/Tests.elm deleted file mode 100644 index f6c7c25..0000000 --- a/test_data/two-fer/missing_solution/tests/Tests.elm +++ /dev/null @@ -1,23 +0,0 @@ -module Tests exposing (tests) - -import Expect -import String -import Test exposing (..) -import TwoFer exposing (twoFer) - - -tests : Test -tests = - describe "Two-fer" - [ test "No name given" <| - \() -> - Expect.equal "One for you, one for me." (twoFer Nothing) - , skip <| - test "A name given" <| - \() -> - Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) - , skip <| - test "Another name given" <| - \() -> - Expect.equal "One for Bob, one for me." (twoFer (Just "Bob")) - ] diff --git a/test_data/two-fer/missing_tests/elm.json b/test_data/two-fer/missing_tests/elm.json deleted file mode 100644 index e08a5bb..0000000 --- a/test_data/two-fer/missing_tests/elm.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "type": "application", - "source-directories": [ - "src" - ], - "elm-version": "0.19.1", - "dependencies": { - "direct": { - "elm/core": "1.0.5", - "elm/json": "1.1.3", - "elm/parser": "1.1.0", - "elm/random": "1.0.0", - "elm/regex": "1.0.0", - "elm/time": "1.0.0" - }, - "indirect": {} - }, - "test-dependencies": { - "direct": { - "elm-explorations/test": "1.2.2", - "rtfeldman/elm-iso8601-date-strings": "1.1.3" - }, - "indirect": { - "elm/html": "1.0.0", - "elm/virtual-dom": "1.0.2" - } - } -} diff --git a/test_data/two-fer/missing_tests/expected_results.json b/test_data/two-fer/missing_tests/expected_results.json deleted file mode 100644 index 330832b..0000000 --- a/test_data/two-fer/missing_tests/expected_results.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "version": 3, - "status": "error", - "message": "sed: can't read tests/Tests.elm: No such file or directory\n" -} diff --git a/test_data/two-fer/missing_tests/src/TwoFer.elm b/test_data/two-fer/missing_tests/src/TwoFer.elm deleted file mode 100644 index b9df217..0000000 --- a/test_data/two-fer/missing_tests/src/TwoFer.elm +++ /dev/null @@ -1,8 +0,0 @@ -module TwoFer exposing (twoFer) - - -twoFer : Maybe String -> String -twoFer name = - "One for " - ++ Maybe.withDefault "you" name - ++ ", one for me." diff --git a/test_data/two-fer/test_with_non_literal_name/elm.json b/test_data/two-fer/test_with_non_literal_name/elm.json deleted file mode 100644 index e08a5bb..0000000 --- a/test_data/two-fer/test_with_non_literal_name/elm.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "type": "application", - "source-directories": [ - "src" - ], - "elm-version": "0.19.1", - "dependencies": { - "direct": { - "elm/core": "1.0.5", - "elm/json": "1.1.3", - "elm/parser": "1.1.0", - "elm/random": "1.0.0", - "elm/regex": "1.0.0", - "elm/time": "1.0.0" - }, - "indirect": {} - }, - "test-dependencies": { - "direct": { - "elm-explorations/test": "1.2.2", - "rtfeldman/elm-iso8601-date-strings": "1.1.3" - }, - "indirect": { - "elm/html": "1.0.0", - "elm/virtual-dom": "1.0.2" - } - } -} diff --git a/test_data/two-fer/test_with_non_literal_name/expected_results.json b/test_data/two-fer/test_with_non_literal_name/expected_results.json deleted file mode 100644 index b79b51f..0000000 --- a/test_data/two-fer/test_with_non_literal_name/expected_results.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "version": 3, - "status": "error", - "message": "Number of tests doesn't match number of extracted code snippets. Please report this issue." -} diff --git a/test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm b/test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm deleted file mode 100644 index b9df217..0000000 --- a/test_data/two-fer/test_with_non_literal_name/src/TwoFer.elm +++ /dev/null @@ -1,8 +0,0 @@ -module TwoFer exposing (twoFer) - - -twoFer : Maybe String -> String -twoFer name = - "One for " - ++ Maybe.withDefault "you" name - ++ ", one for me." diff --git a/test_data/two-fer/test_with_non_literal_name/tests/Tests.elm b/test_data/two-fer/test_with_non_literal_name/tests/Tests.elm deleted file mode 100644 index 491975b..0000000 --- a/test_data/two-fer/test_with_non_literal_name/tests/Tests.elm +++ /dev/null @@ -1,23 +0,0 @@ -module Tests exposing (tests) - -import Expect -import String -import Test exposing (..) -import TwoFer exposing (twoFer) - - -tests : Test -tests = - describe "Two-fer" - [ test "Normal test" <| - \() -> - Expect.equal "One for you, one for me." (twoFer Nothing) - , skip <| - let - testName = - "Non literal test name, code will not get extracted" - in - test testName <| - \() -> - Expect.equal "One for Alice, one for me." (twoFer (Just "Alice")) - ] From dd62e44ac692e4b65d4ff48e8f8a090559c860de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gillet?= Date: Thu, 31 Mar 2022 23:02:38 +0900 Subject: [PATCH 06/11] Add smoke tests to CI --- .github/workflows/pullrequest.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index df364e5..4486135 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -26,3 +26,13 @@ jobs: tags: latest cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + + smoke-test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 + + - name: Run Smoke Tests in Docker + run: bin/run-tests-in-docker.sh + From 31a373d43fd23f305279fe53ed5cedd328227a67 Mon Sep 17 00:00:00 2001 From: Matthieu Pizenberg Date: Sun, 3 Apr 2022 19:02:54 +0200 Subject: [PATCH 07/11] Some rephrasing --- bin/run.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 9f3448e..57ecdaf 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -43,17 +43,18 @@ STATUS=$? cat stderr.txt if [ $STATUS -ne 0 ]; then jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json - echo "An error occured while replacing test in the test file. " + echo "An error occured while un-skipping the tests." exit 0 fi +# Run the tests elm-test-rs -v --report exercism --offline > test_results.json 2> stderr.txt STATUS=$? cat stderr.txt # elm-test-rs will exit(0) if tests pass, exit(2) if tests fail if [ $STATUS -ne 0 ] && [ $STATUS -ne 2 ]; then jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json - echo "An error occured while running tests with elm-test-rs" + echo "An error occured while running the tests." exit 0 fi @@ -63,7 +64,7 @@ STATUS=$? cat stderr.txt if [ $STATUS -ne 0 ]; then jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json - echo "An error occurred while extracting the test code from the test file." + echo "An error occurred while extracting the test code snippets." exit 0 fi @@ -71,7 +72,7 @@ fi test_code_length=$(jq 'length' test_code.json) test_result_length=$(jq '.tests | length' test_results.json) if [ $test_code_length -ne $test_result_length ] ; then - err="Number of tests doesn't match number of extracted code snippets. Please report this issue." + err="Number of tests doesn't match number of extracted code snippets. Please report this issue at https://github.com/exercism/elm-test-runner/issues." jq -n --arg m "${err}" '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json echo $err exit 0 From d60edb9258a41c7e43fcf168cbfb927c2807285d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gillet?= Date: Tue, 5 Apr 2022 00:00:17 +0900 Subject: [PATCH 08/11] Simplify and merge smoke test scripts --- bin/check_files.sh | 55 ---------------------------------------------- bin/smoke_test.sh | 22 +++++++++++++++++-- 2 files changed, 20 insertions(+), 57 deletions(-) delete mode 100755 bin/check_files.sh diff --git a/bin/check_files.sh b/bin/check_files.sh deleted file mode 100755 index e794283..0000000 --- a/bin/check_files.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# This script is called by bin/smoke_test.sh -# -# It checks that for a given exercise path (given as argument) -# 1. The test runner created an output file -# 2. There is a reference file available for comparison -# 3. Both files are identical -# -# Otherwise, it fails with an error message. -# - -set -euo pipefail - -exercise=$1 - -function installed { - cmd=$(command -v "${1}") - - [[ -n "${cmd}" ]] && [[ -f "${cmd}" ]] - return ${?} -} - -function die { - >&2 echo "Fatal: ${@}" - exit 1 -} - -function main { - if [[ ! -f "${exercise}/expected_results.json" ]]; then - echo "🔥 ${exercise}: expected expected_results.json to exist 🔥" - exit 1 - fi - - if [[ ! -f "${exercise}/results.json" ]]; then - echo "🔥 ${exercise}: expected results.json to exist on successful run 🔥" - exit 1 - fi - - jq -S . ${exercise}/expected_results.json > /tmp/expected.json - jq -S . ${exercise}/results.json > /tmp/actual.json - if ! diff /tmp/expected.json /tmp/actual.json ;then - echo "🔥 ${exercise}: expected ${exercise}/results.json to equal ${exercise}/expected_results.json on successful run 🔥" - exit 1 - fi - - echo "🏁 ${exercise}: expected files present and correct after successful run 🏁" -} - -# Check for all required dependencies -for dep in diff jq; do - installed "${dep}" || die "Missing '${dep}'" -done - -main "$@"; exit diff --git a/bin/smoke_test.sh b/bin/smoke_test.sh index 88edbe3..c119fe6 100755 --- a/bin/smoke_test.sh +++ b/bin/smoke_test.sh @@ -6,13 +6,31 @@ set -u # Exit on usage of undeclared variable. set -o pipefail # Catch failures in pipes. for solution in test_data/*/* ; do - echo $solution slug=$(basename $(dirname $solution)) solution=$(realpath $solution) + # run tests bin/run.sh $slug $solution $solution > /dev/null + # check result - bin/check_files.sh $solution + if [[ ! -f "${solution}/expected_results.json" ]]; then + echo "🔥 ${solution}: expected expected_results.json to exist 🔥" + exit 1 + fi + + if [[ ! -f "${solution}/results.json" ]]; then + echo "🔥 ${solution}: expected results.json to exist on successful run 🔥" + exit 1 + fi + + jq -S . ${solution}/expected_results.json > /tmp/expected.json + jq -S . ${solution}/results.json > /tmp/actual.json + if ! diff /tmp/expected.json /tmp/actual.json ;then + echo "🔥 ${solution}: expected results.json to equal expected_results.json on successful run 🔥" + exit 1 + fi + + echo "🏁 ${solution}: expected files present and correct after successful run 🏁" done echo "Smoke test finished" From 178a111243d9b961c9a2396153c3b5eb25232117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gillet?= Date: Tue, 5 Apr 2022 00:03:31 +0900 Subject: [PATCH 09/11] Remove deleted file from Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8daf160..0128306 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,6 @@ RUN cd extract-test-code \ # Pack together things to copy to the runner container COPY bin/run.sh bin/run.sh COPY bin/smoke_test.sh bin/smoke_test.sh -COPY bin/check_files.sh bin/check_files.sh # Lightweight runner container FROM node:lts-alpine From 5a07dddd9d69c70ee65e6f2cf4f5cb0dfdf5df44 Mon Sep 17 00:00:00 2001 From: Matthieu Pizenberg Date: Thu, 7 Apr 2022 21:53:15 +0200 Subject: [PATCH 10/11] Update elm-test-rs to v2.0.1 (unique test names) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0128306..17e223f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN curl -L -o elm.gz https://github.com/elm/compiler/releases/download/0.19.1/b && chmod +x bin/elm # Install elm-test-rs -RUN curl -L -o elm-test-rs_linux.tar.gz https://github.com/mpizenberg/elm-test-rs/releases/download/v2.0/elm-test-rs_linux.tar.gz \ +RUN curl -L -o elm-test-rs_linux.tar.gz https://github.com/mpizenberg/elm-test-rs/releases/download/v2.0.1/elm-test-rs_linux.tar.gz \ && tar xf elm-test-rs_linux.tar.gz \ && mv elm-test-rs bin From 65c242aabff50b07c158d78bc452488281146720 Mon Sep 17 00:00:00 2001 From: Matthieu Pizenberg Date: Tue, 12 Apr 2022 16:20:54 +0200 Subject: [PATCH 11/11] Update smoke tests with task ids, and elm-test-rs 2.0.1 --- .../partial_fail/expected_results.json | 10 +++++----- .../success/expected_results.json | 10 +++++----- .../test_issues/expected_results.json | 6 +++--- .../two-fer/does_not_compile/expected_results.json | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test_data/lucians-luscious-lasagna/partial_fail/expected_results.json b/test_data/lucians-luscious-lasagna/partial_fail/expected_results.json index 426c0bd..8a7ec1a 100644 --- a/test_data/lucians-luscious-lasagna/partial_fail/expected_results.json +++ b/test_data/lucians-luscious-lasagna/partial_fail/expected_results.json @@ -3,7 +3,7 @@ "status": "fail", "tests": [ { - "name": "The expected amount of time in the oven is 40 minutes", + "name": "1 > The expected amount of time in the oven is 40 minutes", "task_id": 1, "status": "fail", "message": "\n 40000\n ╷\n │ Expect.equal\n ╵\n 40\n\n", @@ -11,7 +11,7 @@ "test_code": "expectedMinutesInOven\n |> Expect.equal 40" }, { - "name": "For 2 layers, the preparation time is 4 minutes", + "name": "2 > For 2 layers, the preparation time is 4 minutes", "task_id": 2, "status": "pass", "message": null, @@ -19,7 +19,7 @@ "test_code": "preparationTimeInMinutes 2\n |> Expect.equal 4" }, { - "name": "For 5 layers, the preparation time is 10 minutes", + "name": "2 > For 5 layers, the preparation time is 10 minutes", "task_id": 2, "status": "pass", "message": null, @@ -27,7 +27,7 @@ "test_code": "preparationTimeInMinutes 5\n |> Expect.equal 10" }, { - "name": "For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking", + "name": "3 > For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking", "task_id": 3, "status": "pass", "message": null, @@ -35,7 +35,7 @@ "test_code": "elapsedTimeInMinutes 3 10\n |> Expect.equal 16" }, { - "name": "For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking", + "name": "3 > For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking", "task_id": 3, "status": "pass", "message": null, diff --git a/test_data/lucians-luscious-lasagna/success/expected_results.json b/test_data/lucians-luscious-lasagna/success/expected_results.json index ac7ca12..f84d261 100644 --- a/test_data/lucians-luscious-lasagna/success/expected_results.json +++ b/test_data/lucians-luscious-lasagna/success/expected_results.json @@ -3,7 +3,7 @@ "status": "pass", "tests": [ { - "name": "The expected amount of time in the oven is 40 minutes", + "name": "1 > The expected amount of time in the oven is 40 minutes", "task_id": 1, "status": "pass", "message": null, @@ -11,7 +11,7 @@ "test_code": "expectedMinutesInOven\n |> Expect.equal 40" }, { - "name": "For 2 layers, the preparation time is 4 minutes", + "name": "2 > For 2 layers, the preparation time is 4 minutes", "task_id": 2, "status": "pass", "message": null, @@ -19,7 +19,7 @@ "test_code": "preparationTimeInMinutes 2\n |> Expect.equal 4" }, { - "name": "For 5 layers, the preparation time is 10 minutes", + "name": "2 > For 5 layers, the preparation time is 10 minutes", "task_id": 2, "status": "pass", "message": null, @@ -27,7 +27,7 @@ "test_code": "preparationTimeInMinutes 5\n |> Expect.equal 10" }, { - "name": "For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking", + "name": "3 > For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking", "task_id": 3, "status": "pass", "message": null, @@ -35,7 +35,7 @@ "test_code": "elapsedTimeInMinutes 3 10\n |> Expect.equal 16" }, { - "name": "For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking", + "name": "3 > For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking", "task_id": 3, "status": "pass", "message": null, diff --git a/test_data/lucians-luscious-lasagna/test_issues/expected_results.json b/test_data/lucians-luscious-lasagna/test_issues/expected_results.json index 8167df6..b9f8004 100644 --- a/test_data/lucians-luscious-lasagna/test_issues/expected_results.json +++ b/test_data/lucians-luscious-lasagna/test_issues/expected_results.json @@ -3,7 +3,7 @@ "status": "pass", "tests": [ { - "name": "all good", + "name": "1 > all good", "task_id": 1, "status": "pass", "message": null, @@ -11,7 +11,7 @@ "test_code": "expectedMinutesInOven\n |> Expect.equal 40" }, { - "name": "unexpected skip in concept exercise, will be ran anyway", + "name": "2 > unexpected skip in concept exercise, will be ran anyway", "task_id": 2, "status": "pass", "message": null, @@ -27,7 +27,7 @@ "test_code": "elapsedTimeInMinutes 3 10\n |> Expect.equal 16" }, { - "name": "wrong task ID format", + "name": "#3 > wrong task ID format", "task_id": null, "status": "pass", "message": null, diff --git a/test_data/two-fer/does_not_compile/expected_results.json b/test_data/two-fer/does_not_compile/expected_results.json index ed0d6ce..bda8f62 100644 --- a/test_data/two-fer/does_not_compile/expected_results.json +++ b/test_data/two-fer/does_not_compile/expected_results.json @@ -1,5 +1,5 @@ { "version": 3, "status": "error", - "message": "\nelm-test-rs 2.0.0 for elm 0.19.1\n--------------------------------\n\n-- UNFINISHED DEFINITION -------------------------- /tmp/solution/src/TwoFer.elm\n\nI got stuck while parsing the `twoFer` definition:\n\n4| twoFer : Maybe String -> String\n5| twoFer name =\n ^\nI was expecting to see an expression next. What is it equal to?\n\nHere is a valid definition (with a type annotation) for reference:\n\n greet : String -> String\n greet name =\n \"Hello \" ++ name ++ \"!\"\n\nThe top line (called a \"type annotation\") is optional. You can leave it off if\nyou want. As you get more comfortable with Elm and as your project grows, it\nbecomes more and more valuable to add them though! They work great as\ncompiler-verified documentation, and they often improve error messages!\n\n" + "message": "\nelm-test-rs 2.0.1 for elm 0.19.1\n--------------------------------\n\n-- UNFINISHED DEFINITION -------------------------- /tmp/solution/src/TwoFer.elm\n\nI got stuck while parsing the `twoFer` definition:\n\n4| twoFer : Maybe String -> String\n5| twoFer name =\n ^\nI was expecting to see an expression next. What is it equal to?\n\nHere is a valid definition (with a type annotation) for reference:\n\n greet : String -> String\n greet name =\n \"Hello \" ++ name ++ \"!\"\n\nThe top line (called a \"type annotation\") is optional. You can leave it off if\nyou want. As you get more comfortable with Elm and as your project grows, it\nbecomes more and more valuable to add them though! They work great as\ncompiler-verified documentation, and they often improve error messages!\n\n" }