From 881af2043785a73d5f91f03a60a8e580f55798ff Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Thu, 13 Aug 2015 13:33:36 -0700 Subject: [PATCH 1/4] Write test output to test/output/name.wasm.log --- ml-proto/runtests.py | 24 ++++++++++++++++++++---- ml-proto/test/.gitignore | 1 + 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 ml-proto/test/.gitignore diff --git a/ml-proto/runtests.py b/ml-proto/runtests.py index 1a0bb389cd..59d4bd75c7 100755 --- a/ml-proto/runtests.py +++ b/ml-proto/runtests.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import os import os.path import unittest import subprocess @@ -8,14 +9,23 @@ class RunTests(unittest.TestCase): def _runTestFile(self, shortName, fileName, interpreterPath): print("\n// %s" % shortName) - exitCode = subprocess.call([interpreterPath, fileName]) + logPath = fileName.replace("/test/", "/test/output/").replace(".wasm", ".wasm.log") + exitCode = subprocess.call( + ("%s %s | tee %s") % (interpreterPath, fileName, logPath), + shell=True + ) self.assertEqual(0, exitCode, "test runner failed with exit code %i" % exitCode) +def generate_test_case(rec): + return lambda self : self._runTestFile(*rec) + + def generate_test_cases(cls, interpreterPath, files): for fileName in files: absFileName = os.path.abspath(fileName) attrName = fileName - testCase = lambda self : self._runTestFile(attrName, absFileName, interpreterPath) + rec = (attrName, absFileName, interpreterPath) + testCase = generate_test_case(rec) setattr(cls, attrName, testCase) def rebuild_interpreter(): @@ -28,7 +38,13 @@ def rebuild_interpreter(): return interpreterPath -if __name__ == "__main__": +if __name__ == "__main__": + try: + os.makedirs("test/output/") + except OSError: + pass + interpreterPath = rebuild_interpreter() - generate_test_cases(RunTests, interpreterPath, glob.glob("test/*.wasm")) + testFiles = glob.glob("test/*.wasm") + generate_test_cases(RunTests, interpreterPath, testFiles) unittest.main() \ No newline at end of file diff --git a/ml-proto/test/.gitignore b/ml-proto/test/.gitignore new file mode 100644 index 0000000000..6caf68aff4 --- /dev/null +++ b/ml-proto/test/.gitignore @@ -0,0 +1 @@ +output \ No newline at end of file From 11bfc89396afd496b0e310f11092c07a067bee02 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Fri, 14 Aug 2015 13:28:00 -0700 Subject: [PATCH 2/4] Nonsense to make the test runner work on windows --- ml-proto/runtests.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ml-proto/runtests.py b/ml-proto/runtests.py index 59d4bd75c7..186dea1a58 100755 --- a/ml-proto/runtests.py +++ b/ml-proto/runtests.py @@ -5,15 +5,17 @@ import unittest import subprocess import glob +import sys class RunTests(unittest.TestCase): def _runTestFile(self, shortName, fileName, interpreterPath): print("\n// %s" % shortName) - logPath = fileName.replace("/test/", "/test/output/").replace(".wasm", ".wasm.log") - exitCode = subprocess.call( - ("%s %s | tee %s") % (interpreterPath, fileName, logPath), - shell=True - ) + sys.stdout.flush() + logPath = fileName.replace("test/", "test/output/").replace(".wasm", ".wasm.log") + commandStr = ("%s %s | tee %s") % (interpreterPath, fileName, logPath) + print("// %s" % commandStr) + sys.stdout.flush() + exitCode = subprocess.call(commandStr, shell=True) self.assertEqual(0, exitCode, "test runner failed with exit code %i" % exitCode) def generate_test_case(rec): @@ -22,9 +24,8 @@ def generate_test_case(rec): def generate_test_cases(cls, interpreterPath, files): for fileName in files: - absFileName = os.path.abspath(fileName) attrName = fileName - rec = (attrName, absFileName, interpreterPath) + rec = (attrName, fileName, interpreterPath) testCase = generate_test_case(rec) setattr(cls, attrName, testCase) @@ -32,6 +33,7 @@ def rebuild_interpreter(): interpreterPath = os.path.abspath("src/main.native") print("// building main.native") + sys.stdout.flush() exitCode = subprocess.call(["ocamlbuild", "-libs", "bigarray", "main.native"], cwd=os.path.abspath("src")) if (exitCode != 0): raise Exception("ocamlbuild failed with exit code %i" % exitCode) From a2bd570bd26d9ca598c86cbc4ff7880d9283fff7 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Fri, 14 Aug 2015 13:44:42 -0700 Subject: [PATCH 3/4] Add support for comparing test output with expected output --- ml-proto/runtests.py | 22 +++++++++++++++++-- ml-proto/test/expected-output/fac.wasm.log | 4 ++++ ml-proto/test/expected-output/memory.wasm.log | 6 +++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 ml-proto/test/expected-output/fac.wasm.log create mode 100644 ml-proto/test/expected-output/memory.wasm.log diff --git a/ml-proto/runtests.py b/ml-proto/runtests.py index 186dea1a58..6e9fafcdd4 100755 --- a/ml-proto/runtests.py +++ b/ml-proto/runtests.py @@ -11,13 +11,31 @@ class RunTests(unittest.TestCase): def _runTestFile(self, shortName, fileName, interpreterPath): print("\n// %s" % shortName) sys.stdout.flush() + logPath = fileName.replace("test/", "test/output/").replace(".wasm", ".wasm.log") + try: + os.remove(logPath) + except OSError: + pass + commandStr = ("%s %s | tee %s") % (interpreterPath, fileName, logPath) - print("// %s" % commandStr) - sys.stdout.flush() exitCode = subprocess.call(commandStr, shell=True) self.assertEqual(0, exitCode, "test runner failed with exit code %i" % exitCode) + try: + expected = open(fileName.replace("test/", "test/expected-output/").replace(".wasm", ".wasm.log")) + except IOError: + print("// WARNING: No expected output found for this test") + return + + output = open(logPath) + + with expected: + with output: + expectedText = expected.read() + actualText = output.read() + self.assertEqual(expectedText, actualText) + def generate_test_case(rec): return lambda self : self._runTestFile(*rec) diff --git a/ml-proto/test/expected-output/fac.wasm.log b/ml-proto/test/expected-output/fac.wasm.log new file mode 100644 index 0000000000..7081230246 --- /dev/null +++ b/ml-proto/test/expected-output/fac.wasm.log @@ -0,0 +1,4 @@ +export 0 : Int64 -> Int64 +export 1 : Int64 -> Int64 +7034535277573963776 : Int64 +7034535277573963776 : Int64 diff --git a/ml-proto/test/expected-output/memory.wasm.log b/ml-proto/test/expected-output/memory.wasm.log new file mode 100644 index 0000000000..86f19b33aa --- /dev/null +++ b/ml-proto/test/expected-output/memory.wasm.log @@ -0,0 +1,6 @@ +export 0 : () -> Int32 +export 1 : () -> Int32 +export 2 : () -> Float64 +1 : Int32 +1 : Int32 +-3.10552331248e+231 : Float64 From 6e7fc8fd533c64385979ca847f26f380d4e31fb3 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Tue, 18 Aug 2015 13:28:04 -0700 Subject: [PATCH 4/4] Update expected test outputs --- ml-proto/test/expected-output/fac.wasm.log | 8 ++++---- ml-proto/test/expected-output/memory.wasm.log | 14 ++++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ml-proto/test/expected-output/fac.wasm.log b/ml-proto/test/expected-output/fac.wasm.log index 7081230246..e0b0cad50a 100644 --- a/ml-proto/test/expected-output/fac.wasm.log +++ b/ml-proto/test/expected-output/fac.wasm.log @@ -1,4 +1,4 @@ -export 0 : Int64 -> Int64 -export 1 : Int64 -> Int64 -7034535277573963776 : Int64 -7034535277573963776 : Int64 +export 0 : i64 -> i64 +export 1 : i64 -> i64 +7034535277573963776 : i64 +7034535277573963776 : i64 diff --git a/ml-proto/test/expected-output/memory.wasm.log b/ml-proto/test/expected-output/memory.wasm.log index 86f19b33aa..92d718b11b 100644 --- a/ml-proto/test/expected-output/memory.wasm.log +++ b/ml-proto/test/expected-output/memory.wasm.log @@ -1,6 +1,8 @@ -export 0 : () -> Int32 -export 1 : () -> Int32 -export 2 : () -> Float64 -1 : Int32 -1 : Int32 --3.10552331248e+231 : Float64 +export 0 : () -> i32 +export 1 : () -> i32 +export 2 : () -> i32 +export 3 : () -> f64 +1 : i32 +1 : i32 +1 : i32 +-3.10552331246e+231 : f64