Report actual argv on errors

README.md

@@ -44,32 +44,31 @@ executor is quite simple; see the [specification] document for the details and t
 5. Execute the test suites from this repository:
 
    ```bash
-   python3 test-runner/wasi_test_runner.py                                                  \
-      -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
-         ./tests/c/testsuite/wasm32-wasip1                                                 \
-         ./tests/rust/testsuite/                                                           \
-      -r adapters/wasmtime.py # path to a runtime adapter
+   ./run-tests
    ```
 
-Optionally you can specify test cases to skip with the `--exclude-filter` option.
+By default, the test runner will detect available WASI runtimes from
+those available in [adapters/](adapters/), and will run tests on all
+available runtimes.  Pass `--runtime` to instead use a specific runtime.
+
+```
+./run-tests --runtime adapters/wasmtime.py
+```
+
+Running tests will invoke the WASI runtime's binary in a subprocess:
+`wasmtime` for `adapters/wasmtime.py`, `iwasm` for
+`adapters/wasm-micro-runtime.py`, and so on.  These binaries can be
+overridden by setting corresponding environment variables (`WASMTIME`,
+`IWASM`, etc):
 
-```bash
-python3 test-runner/wasi_test_runner.py                                                  \
-    -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
-       ./tests/c/testsuite/wasm32-wasip1                                                 \
-       ./tests/rust/testsuite/                                                           \
-    --exclude-filter examples/skip.json                                                  \
-    -r adapters/wasmtime.py # path to a runtime adapter
+```
+WASMTIME="wasmtime --wasm-features all" ./run-tests
 ```
 
-The default executable in the adapter used for test execution can be
-overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in
-[adapters/](adapters/), and might not work with 3rd party adapters.
+Optionally you can specify test cases to skip with the `--exclude-filter` option.
 
 ```bash
-TEST_RUNTIME_EXE="wasmtime --wasm-features all" python3 test-runner/wasi_test_runner.py                                                  \
-    -t ./tests/assemblyscript/testsuite/ \
-    -r adapters/wasmtime.py
+./run-tests --exclude-filter examples/skip.json                                                  \
 ```
 
 ## Contributing

adapters/wasm-micro-runtime.py

@@ -1,29 +1,35 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
-IWASM = shlex.split(os.getenv("TEST_RUNTIME_EXE", "iwasm"))
+IWASM = shlex.split(os.getenv("IWASM", "iwasm"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wamr"
 
-if args.version:
-    subprocess.run(IWASM + ["--version"])
-    sys.exit(0)
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = [f"--env={i}" for i in args.env]
-DIR_ARGS = [f"--dir={i}" for i in args.dir]
+def get_version() -> str:
+    # ensure no args when version is queried
+    result = subprocess.run(IWASM + ["--version"],
+                            encoding="UTF-8", capture_output=True,
+                            check=True)
+    output = result.stdout.splitlines()[0].split(" ")
+    return output[1]
 
-r = subprocess.run(IWASM + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
-sys.exit(r.returncode)
+
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + IWASM
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+    for host, guest in dirs:
+        argv += ["--map-dir", f"{host}::{guest}"]  # noqa: E231
+    argv += [test_path]
+    argv += args
+    return argv

adapters/wasmtime.py

@@ -1,30 +1,35 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
-WASMTIME = shlex.split(os.getenv("TEST_RUNTIME_EXE", "wasmtime"))
+WASMTIME = shlex.split(os.getenv("WASMTIME", "wasmtime"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wasmtime"
 
-if args.version:
+
+def get_version() -> str:
     # ensure no args when version is queried
-    subprocess.run(WASMTIME[0:1] + ["--version"])
-    sys.exit(0)
+    result = subprocess.run(WASMTIME[0:1] + ["--version"],
+                            encoding="UTF-8", capture_output=True,
+                            check=True)
+    output = result.stdout.splitlines()[0].split(" ")
+    return output[1]
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = [j for i in args.env for j in ["--env", i]]
-DIR_ARGS = [j for i in args.dir for j in ["--dir", i]]
 
-r = subprocess.run(WASMTIME + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
-sys.exit(r.returncode)
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + WASMTIME
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+    for host, guest in dirs:
+        argv += ["--dir", f"{host}::{guest}"]  # noqa: E231
+    argv += [test_path]
+    argv += args
+    return argv

adapters/wizard.py

@@ -1,30 +1,44 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
-WIZARD = shlex.split(os.getenv("TEST_RUNTIME_EXE", "wizeng.x86-64-linux"))
+WIZARD = shlex.split(os.getenv("WIZARD", "wizeng.x86-64-linux"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wizard"
 
-if args.version:
+
+def get_version() -> str:
     # ensure no args when version is queried
-    subprocess.run(WIZARD[0:1] + ["-version"])
-    sys.exit(0)
+    output = ""
+    try:
+        result = subprocess.run(WIZARD[0:1] + ["--version"],
+                                encoding="UTF-8", capture_output=True,
+                                check=False)
+        output = result.stdout;
+    except subprocess.CalledProcessError as e:
+        # https://github.com/titzer/wizard-engine/issues/483
+        if e.returncode != 3:
+            raise e
+        output = result.stdout
+    output = output.splitlines()[0].split(" ")
+    return output[1]
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = None if len(args.env) == 0 else f'--env={",".join(args.env)}'
-DIR_ARGS = None if len(args.dir) == 0 else f'--dir={",".join(args.dir)}'
 
-r = subprocess.run([arg for arg in WIZARD + [ENV_ARGS, DIR_ARGS, TEST_FILE] + PROG_ARGS if arg])
-sys.exit(r.returncode)
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + WIZARD
+    for k, v in env.items():
+        argv += [f"--env={k}={v}"]
+    for host, guest in dirs:
+        # FIXME: https://github.com/titzer/wizard-engine/issues/482
+        argv += [f"--dir={host}"]
+    argv += [test_path]
+    argv += args
+    return argv

run-tests

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+import sys
+from typing import List
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent / "test-runner"))
+
+from wasi_test_runner.harness import run_tests
+from wasi_test_runner import runtime_adapter
+
+parser = argparse.ArgumentParser(
+    description="WASI test runner"
+)
+
+parser.add_argument(
+    "-f",
+    "--exclude-filter",
+    action="append",
+    default=[],
+    help="Path to JSON file indicating tests to exclude.",
+)
+parser.add_argument(
+    "-r", "--runtime-adapter", help="Path to a runtime adapter."
+)
+parser.add_argument(
+    "--json-output-location",
+    help="JSON test result destination. If not specified, JSON output won't be generated.",
+)
+parser.add_argument(
+    "--disable-colors",
+    action="store_true",
+    default=False,
+    help="Disables color for console output reporter.",
+)
+parser.add_argument(
+    "--verbose",
+    action="store_true",
+    default=False,
+    help="Print more information about test results.",
+)
+
+def find_test_dirs():
+    root = Path(__file__).parent / "tests"
+    root = root.relative_to(Path('.').absolute(), walk_up=True)
+    test_dirs = []
+    for root, dirs, files in root.walk(on_error=print):
+        if "manifest.json" in files:
+            test_dirs.append(root)
+    return test_dirs
+
+def find_runtime_adapters(root, verbose=False):
+    print(f"Detecting WASI runtime availability:")
+    adapters = []
+    for candidate in root.glob("*.py"):
+        try:
+            adapter = runtime_adapter.RuntimeAdapter(candidate)
+            print(f"  {candidate.name}: {adapter.get_version()}")
+            adapters.append(adapter)
+        except runtime_adapter.LegacyRuntimeAdapterError:
+            print(f"  {candidate} is too old; update to new module format.")
+        except runtime_adapter.UnavailableRuntimeAdapterError:
+            print(f"  {candidate.name} unavailable; pass `--runtime {candidate}` to debug.")
+    print("")
+    if len(adapters) == 0:
+        print("Error: No WASI runtimes found")
+        sys.exit(1)
+    return adapters
+
+options = parser.parse_args()
+test_suite = find_test_dirs()
+if options.runtime_adapter:
+    try:
+        runtime_adapters = [
+            runtime_adapter.RuntimeAdapter(options.runtime_adapter)
+        ]
+    except runtime_adapter.LegacyRuntimeAdapterError as e:
+        print(f"Error: {e.adapter_path} is too old; update to new module format.")
+        sys.exit(1)
+    except runtime_adapter.UnavailableRuntimeAdapterError as e:
+        print(f"Error: failed to load {e.adapter_path}: {e.error}")
+        sys.exit(1)
+else:
+    runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")
+
+exclude_filters = [JSONTestExcludeFilter(filt) for filt in options.exclude_filter]
+
+sys.exit(run_tests(runtime_adapters, test_suite,
+                   color=not options.disable_colors,
+                   json_log_file=options.json_output_location,
+                   exclude_filters=exclude_filters))

test-runner/tests/test_test_suite.py

@@ -2,12 +2,14 @@
 
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite as ts
+from wasi_test_runner.runtime_adapter import RuntimeVersion
 
 
 def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCase:
     failures = [tc.Failure("a", "b")] if is_failed else []
     return tc.TestCase(
         name,
+        ["test-runtime-exe", name],
         tc.Config(),
         tc.Result(tc.Output(0, "", ""), is_executed, failures),
         1.0,
@@ -17,6 +19,7 @@ def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCa
 def test_test_suite_should_return_correct_count() -> None:
     suite = ts.TestSuite(
         "suite",
+        RuntimeVersion("test-runtime", "3.14"),
         10.0,
         datetime.now(),
         [