Running smoke tests in docker

.github/workflows/elixir_test.yml

@@ -62,3 +62,14 @@ jobs:
 
       - name: Run dialyzer
         run: mix dialyzer
+
+  smoke-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+
+      - name: Run Smoke Test in Docker
+        run: bin/run-tests-in-docker.sh
+

Dockerfile

@@ -18,7 +18,7 @@ COPY --from=builder /etc/passwd /etc/passwd
 
 COPY --from=builder /elixir-analyzer/bin /opt/analyzer/bin
 RUN apt-get update && \
-  apt-get install bash -y
+  apt-get install bash jq -y
 
 USER appuser
 WORKDIR /opt/analyzer

bin/check_files.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+exercise=$1
+
+function installed {
+  cmd=$(command -v "${1}")
+
+  [[ -n "${cmd}" ]] && [[ -f "${cmd}" ]]
+  return ${?}
+}
+
+function die {
+  >&2 echo "Fatal: ${@}"
+  exit 1
+}
+
+function main {
+  expected_files=(/tmp/${exercise}/analysis.json ${exercise}/expected_analysis.json)
+
+  for file in ${expected_files[@]}; do
+    if [[ ! -f "${file}" ]]; then
+      echo "🔥 ${exercise}: expected ${file} to exist on successful run 🔥"
+      exit 1
+    fi
+  done
+
+  if ! diff <(jq -S . ${exercise}/expected_analysis.json) <(jq -S . /tmp/${exercise}/analysis.json); then
+    echo "🔥 ${exercise}: expected /tmp/${exercise}/analysis.json to equal ${exercise}/expected_analysis.json on successful run 🔥"
+    exit 1
+  fi
+
+  echo "🏁 ${exercise}: expected files present after successful run 🏁"
+}
+
+# Check for all required dependencies
+deps=(diff jq)
+for dep in "${deps[@]}"; do
+  installed "${dep}" || die "Missing '${dep}'"
+done
+
+main "$@"; exit

bin/run-tests-in-docker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Synopsis:
+# Test the analyzer Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The analyzer Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+set -e # Make script exit when a command fail.
+set -u # Exit on usage of undeclared variable.
+# set -x # Trace what gets executed.
+set -o pipefail # Catch failures in pipes.
+
+# build docker image
+docker build --rm -t elixir-analyzer .
+
+# run image passing the arguments
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src=$(realpath test_data),dst=/opt/analyzer/test_data \
+    --mount type=tmpfs,dst=/tmp \
+    --entrypoint /opt/analyzer/bin/smoke_test.sh \
+    elixir-analyzer

bin/smoke_test.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e # Make script exit when a command fail.
+set -u # Exit on usage of undeclared variable.
+# set -x # Trace what gets executed.
+set -o pipefail # Catch failures in pipes.
+
+for solution in test_data/*/* ; do
+  slug=$(basename $(dirname $solution))
+  mkdir -p /tmp/$solution
+  # run analysis
+  bin/run.sh $slug $solution /tmp/$solution
+  # check result
+  bin/check_files.sh $solution
+done

test_data/clock/perfect_solution/.meta/config.json

@@ -0,0 +1,28 @@
+{
+  "blurb": "Implement a clock that handles times without dates.",
+  "authors": [
+    "tejasbubane"
+  ],
+  "contributors": [
+    "angelikatyborska",
+    "Cohen-Carlisle",
+    "devonestes",
+    "nathanchere",
+    "neenjaw",
+    "parkerl",
+    "sotojuan"
+  ],
+  "files": {
+    "solution": [
+      "lib/clock.ex"
+    ],
+    "test": [
+      "test/clock_test.exs"
+    ],
+    "example": [
+      ".meta/example.ex"
+    ]
+  },
+  "source": "Pairing session with Erin Drummond",
+  "source_url": "https://twitter.com/ebdrummond"
+}

test_data/clock/perfect_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Submission analyzed. No automated suggestions found."}

test_data/lasagna/deprecated_modules/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"elixir.solution.compiler_warnings","params":{"warnings":"warning: Behaviour.defcallback/1 is deprecated. Use the @callback module attribute instead\n  test_data/lasagna/deprecated_modules/lib/lasagna.ex:4: Lasagna\n\nwarning: HashDict.new/0 is deprecated. Use maps and the Map module instead\n  test_data/lasagna/deprecated_modules/lib/lasagna.ex:7: Lasagna.expected_minutes_in_oven/0\n\nwarning: HashSet.member?/2 is deprecated. Use the MapSet module instead\n  test_data/lasagna/deprecated_modules/lib/lasagna.ex:12: Lasagna.remaining_minutes_in_oven/1\n\nwarning: HashSet.new/0 is deprecated. Use the MapSet module instead\n  test_data/lasagna/deprecated_modules/lib/lasagna.ex:12: Lasagna.remaining_minutes_in_oven/1\n\n"},"type":"actionable"}],"summary":"Check the comments for some suggestions. 📣"}

test_data/lasagna/failing_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"elixir.lasagna.function_reuse","type":"actionable"},{"comment":"elixir.solution.private_helper_functions","params":{"actual":"def public_helper(_)","expected":"defp public_helper(_)"},"type":"informative"},{"comment":"elixir.solution.todo_comment","type":"informative"}],"summary":"Check the comments for some suggestions. 📣"}

test_data/lasagna/missing_config/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Analysis was halted. Analysis skipped, not able to read solution config."}

test_data/lasagna/missing_exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Submission analyzed. No automated suggestions found."}

test_data/lasagna/perfect_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"elixir.solution.same_as_exemplar","type":"celebratory"}],"summary":"You're doing something right. 🎉"}

test_data/lasagna/wrong_config/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Analysis was halted. Analysis skipped, not able to decode solution config."}

test_data/lasagna/wrong_config2/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError"}

test_data/lasagna/wrong_exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Submission analyzed. No automated suggestions found."}

test_data/two_fer/error_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"elixir.general.parsing_error","params":{"error":"missing terminator: end (for \"do\" starting at line 1)","line":14},"type":"essential"}],"summary":"Check the comments for things to fix. 🛠"}

test_data/two_fer/imperfect_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"elixir.solution.raise_fn_clause_error","type":"actionable"},{"comment":"elixir.solution.variable_name_snake_case","params":{"actual":"_nameInPascalCase","expected":"_name_in_pascal_case"},"type":"actionable"},{"comment":"elixir.solution.module_attribute_name_snake_case","params":{"actual":"someUnusedModuleAttribute","expected":"some_unused_module_attribute"},"type":"actionable"},{"comment":"elixir.solution.module_pascal_case","params":{"actual":"My_empty_module","expected":"MyEmptyModule"},"type":"actionable"},{"comment":"elixir.solution.compiler_warnings","params":{"warnings":"warning: module attribute @someUnusedModuleAttribute was set but never used\n  test_data/two_fer/imperfect_solution/lib/two_fer.ex:2\n\n"},"type":"actionable"},{"comment":"elixir.solution.indentation","type":"informative"},{"comment":"elixir.solution.private_helper_functions","params":{"actual":"def public_helper(_)","expected":"defp public_helper(_)"},"type":"informative"}],"summary":"Check the comments for some suggestions. 📣"}

test_data/two_fer/informative_comments/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Submission analyzed. No automated suggestions found."}

test_data/two_fer/missing_example_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Submission analyzed. No automated suggestions found."}

test_data/two_fer/missing_file_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"elixir.general.file_not_found","params":{"file_name":"two_fer.ex","path":"test_data/two_fer/missing_file_solution"},"type":"essential"}],"summary":"Check the comments for things to fix. 🛠"}

test_data/two_fer/perfect_solution/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[],"summary":"Submission analyzed. No automated suggestions found."}