Update elm-test-rs to v2.0 and add smoke tests

.github/workflows/pullrequest.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
@@ -26,3 +26,13 @@ jobs:
           tags: latest
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
+
+  smoke-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+
+      - name: Run Smoke Tests in Docker
+        run: bin/run-tests-in-docker.sh
+

.gitignore

@@ -7,3 +7,8 @@
 .netlify
 
 elm-stuff
+
+# smoke test intermediate files
+test_code.json
+test_results.json
+results.json

Dockerfile

@@ -20,7 +20,7 @@ RUN curl -L -o elm.gz https://github.com/elm/compiler/releases/download/0.19.1/b
   && chmod +x bin/elm
 
 # Install elm-test-rs
-RUN curl -L -o elm-test-rs_linux.tar.gz https://github.com/mpizenberg/elm-test-rs/releases/download/v1.1/elm-test-rs_linux.tar.gz \
+RUN curl -L -o elm-test-rs_linux.tar.gz https://github.com/mpizenberg/elm-test-rs/releases/download/v2.0.1/elm-test-rs_linux.tar.gz \
   && tar xf elm-test-rs_linux.tar.gz \
   && mv elm-test-rs bin
 
@@ -41,10 +41,12 @@ RUN cd extract-test-code \
 
 # Pack together things to copy to the runner container
 COPY bin/run.sh bin/run.sh
+COPY bin/smoke_test.sh bin/smoke_test.sh
 
 # Lightweight runner container
 FROM node:lts-alpine
 WORKDIR /opt/test-runner
+ENV PATH="/opt/test-runner/bin:${PATH}"
 COPY --from=builder /opt/test-runner/bin bin
 COPY --from=builder /opt/test-runner/cache.tar .
 ENTRYPOINT [ "bin/run.sh" ]

bin/run-in-docker.sh

@@ -43,6 +43,9 @@ docker build -t elm-test-runner .
 # run image passing the arguments
 mkdir -p "$OUTPUT_DIR"
 docker run --network none \
+    --rm \
+    --read-only \
     --mount type=bind,src=$INPUT_DIR,dst=/solution \
     --mount type=bind,src=$OUTPUT_DIR,dst=/output \
+    --mount type=tmpfs,dst=/tmp \
     elm-test-runner $SLUG /solution/ /output/

bin/run-tests-in-docker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Synopsis:
+# Test the test-runner Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The test-runner Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+set -e # Make script exit when a command fail.
+set -u # Exit on usage of undeclared variable.
+# set -x # Trace what gets executed.
+set -o pipefail # Catch failures in pipes.
+
+# build docker image
+docker build --rm -t elm-test-runner .
+
+# run image passing the arguments
+docker run \
+    --rm \
+    --read-only \
+    --network none \
+    --mount type=bind,src=$(realpath test_data),dst=/opt/test-runner/test_data \
+    --mount type=tmpfs,dst=/tmp \
+    --entrypoint /opt/test-runner/bin/smoke_test.sh \
+    elm-test-runner

bin/run.sh

@@ -10,16 +10,13 @@ SLUG="$1"
 INPUT_DIR="$2"
 OUTPUT_DIR="$3"
 
-# Setup a working directory
-WORK_DIR=/opt/test-runner/app
-cp -rp $INPUT_DIR $WORK_DIR
-mkdir -p $WORK_DIR/elm-stuff && rm -rf $WORK_DIR/elm-stuff
+# Setup a temporary working directory
+WORK_DIR=/tmp/solution
+mkdir -p $WORK_DIR && rm -rf $WORK_DIR
+cp -r $INPUT_DIR $WORK_DIR
 tar xf cache.tar -C $WORK_DIR
 cd $WORK_DIR
 
-# Add bin/ to the path to make available elm and elm-test-rs.
-export PATH=/opt/test-runner/bin:${PATH}
-
 # Use the cache in .elm/ by redefining the elm home directory.
 export ELM_HOME=$WORK_DIR/.elm
 
@@ -30,7 +27,6 @@ LogLevel Error
 Port 4343
 Listen 127.0.0.1
 MaxClients 100
-StartServers 1
 Filter "filter.conf"
 FilterDefaultDeny Yes
 EOT
@@ -39,36 +35,51 @@ touch filter.conf
 export https_proxy=127.0.0.1:4343
 tinyproxy -d -c proxy.conf &
 
-# Un-skip all the skipped tests
-sed -i 's/skip <|//g' tests/Tests.elm
-
 # Temporarily disable -e mode
 set +e
-elm-test-rs -v --report exercism --offline > $OUTPUT_DIR/results.json 2> stderr.txt
+# Un-skip all the skipped tests
+sed -i 's/skip <|//g' tests/Tests.elm 2> stderr.txt
+STATUS=$?
+cat stderr.txt
+if [ $STATUS -ne 0 ]; then
+    jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json
+    echo "An error occured while un-skipping the tests."
+    exit 0
+fi
+
+# Run the tests
+elm-test-rs -v --report exercism --offline > test_results.json 2> stderr.txt
 STATUS=$?
 cat stderr.txt
 # elm-test-rs will exit(0) if tests pass, exit(2) if tests fail
 if [ $STATUS -ne 0 ] && [ $STATUS -ne 2 ]; then
-   jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json
-   echo "Finished with error"
-   exit 0
+    jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json
+    echo "An error occured while running the tests."
+    exit 0
 fi
 
 # Extract test code
-cat tests/Tests.elm | node ../bin/cli.js > $OUTPUT_DIR/test_code.json 2> stderr.txt
+cat tests/Tests.elm | node /opt/test-runner/bin/cli.js > test_code.json 2> stderr.txt
 STATUS=$?
 cat stderr.txt
 if [ $STATUS -ne 0 ]; then
-   echo "An error occurred while extracting the test code from the test file."
-   exit 0
+    jq -n --rawfile m stderr.txt '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json
+    echo "An error occurred while extracting the test code snippets."
+    exit 0
+fi
+
+# Check number of tests matches number of extracted code snippets
+test_code_length=$(jq 'length' test_code.json)
+test_result_length=$(jq '.tests | length' test_results.json)
+if [ $test_code_length -ne $test_result_length ] ; then
+    err="Number of tests doesn't match number of extracted code snippets. Please report this issue at https://github.com/exercism/elm-test-runner/issues."
+    jq -n --arg m "${err}" '{version: 3, status: "error", message:$m}' > $OUTPUT_DIR/results.json
+    echo $err
+    exit 0
 fi
 set -e
 
-cd $OUTPUT_DIR
-# Merge tests results with extracted test code.
-# This rely on the fact that the order is the same in both arrays.
-jq -s '[range(.[1]|length) as $i | .[0].tests[$i] + { test_code: .[1][$i].testCode }]' results.json test_code.json > concat.json
-jq -s '.[0].tests = .[1] | .[0]' results.json concat.json > results_with_code.json
-mv results_with_code.json results.json
+# Merge tests results with extracted test code, in the extracted code order.
+jq --slurpfile code test_code.json '.tests = (.tests + ($code | .[0] | to_entries | map({order: .key, name: .value.name, test_code: .value.testCode})) | group_by(.name) | map(add) | sort_by(.order) | map(del(.order)))' test_results.json > $OUTPUT_DIR/results.json
 
 echo Finished

bin/smoke_test.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e # Make script exit when a command fail.
+set -u # Exit on usage of undeclared variable.
+# set -x # Trace what gets executed.
+set -o pipefail # Catch failures in pipes.
+
+for solution in test_data/*/* ; do
+  slug=$(basename $(dirname $solution))
+  solution=$(realpath $solution)
+
+  # run tests
+  bin/run.sh $slug $solution $solution > /dev/null
+
+  # check result
+  if [[ ! -f "${solution}/expected_results.json" ]]; then
+    echo "🔥 ${solution}: expected expected_results.json to exist 🔥"
+    exit 1
+  fi
+
+  if [[ ! -f "${solution}/results.json" ]]; then
+    echo "🔥 ${solution}: expected results.json to exist on successful run 🔥"
+    exit 1
+  fi
+
+  jq -S . ${solution}/expected_results.json > /tmp/expected.json
+  jq -S . ${solution}/results.json > /tmp/actual.json
+  if ! diff /tmp/expected.json /tmp/actual.json ;then
+    echo "🔥 ${solution}: expected results.json to equal expected_results.json on successful run 🔥"
+    exit 1
+  fi
+
+  echo "🏁 ${solution}: expected files present and correct after successful run 🏁"
+done
+
+echo "Smoke test finished"

test_data/lucians-luscious-lasagna/partial_fail/elm.json

@@ -0,0 +1,28 @@
+{
+    "type": "application",
+    "source-directories": [
+        "src"
+    ],
+    "elm-version": "0.19.1",
+    "dependencies": {
+        "direct": {
+            "elm/core": "1.0.5",
+            "elm/json": "1.1.3",
+            "elm/parser": "1.1.0",
+            "elm/random": "1.0.0",
+            "elm/regex": "1.0.0",
+            "elm/time": "1.0.0"
+        },
+        "indirect": {}
+    },
+    "test-dependencies": {
+        "direct": {
+            "elm-explorations/test": "1.2.2",
+            "rtfeldman/elm-iso8601-date-strings": "1.1.3"
+        },
+        "indirect": {
+            "elm/html": "1.0.0",
+            "elm/virtual-dom": "1.0.2"
+        }
+    }
+}

test_data/lucians-luscious-lasagna/partial_fail/expected_results.json

@@ -0,0 +1,46 @@
+{
+  "version": 3,
+  "status": "fail",
+  "tests": [
+    {
+      "name": "1 > The expected amount of time in the oven is 40 minutes",
+      "task_id": 1,
+      "status": "fail",
+      "message": "\n    40000\n    ╷\n    │ Expect.equal\n    ╵\n    40\n\n",
+      "output": null,
+      "test_code": "expectedMinutesInOven\n |> Expect.equal 40"
+    },
+    {
+      "name": "2 > For 2 layers, the preparation time is 4 minutes",
+      "task_id": 2,
+      "status": "pass",
+      "message": null,
+      "output": null,
+      "test_code": "preparationTimeInMinutes 2\n |> Expect.equal 4"
+    },
+    {
+      "name": "2 > For 5 layers, the preparation time is 10 minutes",
+      "task_id": 2,
+      "status": "pass",
+      "message": null,
+      "output": null,
+      "test_code": "preparationTimeInMinutes 5\n |> Expect.equal 10"
+    },
+    {
+      "name": "3 > For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking",
+      "task_id": 3,
+      "status": "pass",
+      "message": null,
+      "output": null,
+      "test_code": "elapsedTimeInMinutes 3 10\n |> Expect.equal 16"
+    },
+    {
+      "name": "3 > For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking",
+      "task_id": 3,
+      "status": "pass",
+      "message": null,
+      "output": null,
+      "test_code": "elapsedTimeInMinutes 6 30\n |> Expect.equal 42"
+    }
+  ]
+}

test_data/lucians-luscious-lasagna/partial_fail/src/LuciansLusciousLasagna.elm

@@ -0,0 +1,13 @@
+module LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes)
+
+
+expectedMinutesInOven =
+    40000
+
+
+preparationTimeInMinutes layers =
+    2 * layers
+
+
+elapsedTimeInMinutes layers passedAlready =
+    passedAlready + preparationTimeInMinutes layers

test_data/lucians-luscious-lasagna/partial_fail/tests/Tests.elm

@@ -0,0 +1,37 @@
+module Tests exposing (tests)
+
+import Expect
+import LuciansLusciousLasagna exposing (elapsedTimeInMinutes, expectedMinutesInOven, preparationTimeInMinutes)
+import Test exposing (..)
+
+
+tests : Test
+tests =
+    describe "LuciansLusciousLasagna"
+        [ describe "1"
+            [ test "The expected amount of time in the oven is 40 minutes" <|
+                \_ ->
+                    expectedMinutesInOven
+                        |> Expect.equal 40
+            ]
+        , describe "2"
+            [ test "For 2 layers, the preparation time is 4 minutes" <|
+                \_ ->
+                    preparationTimeInMinutes 2
+                        |> Expect.equal 4
+            , test "For 5 layers, the preparation time is 10 minutes" <|
+                \_ ->
+                    preparationTimeInMinutes 5
+                        |> Expect.equal 10
+            ]
+        , describe "3"
+            [ test "For a 3-layers lasagna already in the oven for 10 minutes, you've spent 16 minutes cooking" <|
+                \_ ->
+                    elapsedTimeInMinutes 3 10
+                        |> Expect.equal 16
+            , test "For a 6-layers lasagna already in the oven for 30 minutes, you've spent 42 minutes cooking" <|
+                \_ ->
+                    elapsedTimeInMinutes 6 30
+                        |> Expect.equal 42
+            ]
+        ]

test_data/lucians-luscious-lasagna/success/elm.json

@@ -0,0 +1,28 @@
+{
+    "type": "application",
+    "source-directories": [
+        "src"
+    ],
+    "elm-version": "0.19.1",
+    "dependencies": {
+        "direct": {
+            "elm/core": "1.0.5",
+            "elm/json": "1.1.3",
+            "elm/parser": "1.1.0",
+            "elm/random": "1.0.0",
+            "elm/regex": "1.0.0",
+            "elm/time": "1.0.0"
+        },
+        "indirect": {}
+    },
+    "test-dependencies": {
+        "direct": {
+            "elm-explorations/test": "1.2.2",
+            "rtfeldman/elm-iso8601-date-strings": "1.1.3"
+        },
+        "indirect": {
+            "elm/html": "1.0.0",
+            "elm/virtual-dom": "1.0.2"
+        }
+    }
+}