diff --git a/changelog/core-runtime-unittestEnhancement.dd b/changelog/core-runtime-unittestEnhancement.dd new file mode 100644 index 0000000000..018af4b721 --- /dev/null +++ b/changelog/core-runtime-unittestEnhancement.dd @@ -0,0 +1,49 @@ +`core.runtime` now allows more fine-grained control over unittests. + +`core.runtime.extendedModuleUnitTester` property allows specifying information +about the tests run, and how to handle the result. See documentation for +`core.runtime.UnitTestResult` for details. + +`core.runtime.moduleUnitTester` (setting a unittest handler that returns bool) +will continue to be supported for legacy projects. + +----------------- +import core.runtime; +import core.stdc.stdio: printf; + +UnitTestResult customTester() +{ + UnitTestResult ret; + + // run only the tests in my package + immutable prefix = "myPackage."; + foreach (m; ModuleInfo) + { + if (m.unitTest !is null && m.name.length >= prefix.length && + m.name[0 .. prefix.length] == prefix) + { + ++ret.executed; // count unit tests run + try + { + m.unitTest(); + ++ret.passed; // count unit tests passed + } + catch(Throwable t) + { + auto msg = t.toString(); + printf("%.*s\n", cast(uint)msg.length, msg.ptr); + } + } + } + // always summarize + ret.summarize = true; + // only unit testing, don't ever run main + ret.runMain = false; +} + +version(unittest) static shared this() +{ + Runtime.extendedModuleUnitTester = &customTester; +} +----------------- + diff --git a/posix.mak b/posix.mak index a9d84e87a0..1e20f0a021 100644 --- a/posix.mak +++ b/posix.mak @@ -213,7 +213,8 @@ $(DRUNTIME): $(OBJS) $(SRCS) UT_MODULES:=$(patsubst src/%.d,$(ROOT)/unittest/%,$(SRCS)) HAS_ADDITIONAL_TESTS:=$(shell test -d test && echo 1) ifeq ($(HAS_ADDITIONAL_TESTS),1) - ADDITIONAL_TESTS:=test/init_fini test/exceptions test/coverage test/profile test/cycles test/allocations test/typeinfo test/thread + ADDITIONAL_TESTS:=test/init_fini test/exceptions test/coverage test/profile test/cycles test/allocations test/typeinfo \ + test/thread test/unittest ADDITIONAL_TESTS+=$(if $(SHARED),test/shared,) endif diff --git a/src/core/runtime.d b/src/core/runtime.d index 4655391c5f..ab995882e1 100644 --- a/src/core/runtime.d +++ b/src/core/runtime.d @@ -37,9 +37,58 @@ extern(C) int rt_init(); /// C interface for Runtime.terminate, returns 1/0 instead of bool extern(C) int rt_term(); +/** + * This type is returned by the module unit test handler to indicate testing + * results. + */ +struct UnitTestResult +{ + /** + * Number of modules which were tested + */ + size_t executed; + + /** + * Number of modules passed the unittests + */ + size_t passed; + + /** + * Should the main function be run or not? This is ignored if any tests + * failed. + */ + bool runMain; + + /** + * Should we print a summary of the results? + */ + bool summarize; + + /** + * Simple check for whether execution should continue after unit tests + * have been run. Works with legacy code that expected a bool return. + * + * Returns: + * true if execution should continue after testing is complete, false if + * not. + */ + bool opCast(T : bool)() const + { + return runMain && (executed == passed); + } + + /// Simple return code that says unit tests pass, and main should be run + enum UnitTestResult pass = UnitTestResult(0, 0, true, false); + /// Simple return code that says unit tests failed. + enum UnitTestResult fail = UnitTestResult(1, 0, false, false); +} + +/// Legacy module unit test handler +alias bool function() ModuleUnitTester; +/// Module unit test handler +alias UnitTestResult function() ExtendedModuleUnitTester; private { - alias bool function() ModuleUnitTester; alias bool function(Object) CollectHandler; alias Throwable.TraceInfo function( void* ptr ) TraceHandler; @@ -300,26 +349,39 @@ struct Runtime * value of this routine indicates to the runtime whether the tests ran * without error. * + * There are two options for handlers. The `bool` version is deprecated but + * will be kept for legacy support. Returning `true` from the handler is + * equivalent to returning `UnitTestResult.pass` from the extended version. + * Returning `false` from the handler is equivalent to returning + * `UnitTestResult.fail` from the extended version. + * + * See the documentation for `UnitTestResult` to see how you should set up + * the return structure. + * + * See the documentation for `runModuleUnitTests` for how the default + * algorithm works, or read the example below. + * * Params: - * h = The new unit tester. Set to null to use the default unit tester. + * h = The new unit tester. Set both to null to use the default unit + * tester. * * Example: * --------- - * version (unittest) shared static this() + * shared static this() * { * import core.runtime; * - * Runtime.moduleUnitTester = &customModuleUnitTester; + * Runtime.extendedModuleUnitTester = &customModuleUnitTester; * } * - * bool customModuleUnitTester() + * UnitTestResult customModuleUnitTester() * { * import std.stdio; * * writeln("Using customModuleUnitTester"); * * // Do the same thing as the default moduleUnitTester: - * size_t failed = 0; + * UnitTestResult result; * foreach (m; ModuleInfo) * { * if (m) @@ -328,45 +390,82 @@ struct Runtime * * if (fp) * { + * ++result.executed; * try * { * fp(); + * ++result.passed; * } * catch (Throwable e) * { * writeln(e); - * failed++; * } * } * } * } - * return failed == 0; + * if (result.executed != result.passed) + * { + * result.runMain = false; // don't run main + * result.summarize = true; // print failure + * } + * else + * { + * result.runMain = true; // all UT passed + * result.summarize = false; // be quiet about it. + * } + * return result; * } * --------- */ + static @property void extendedModuleUnitTester( ExtendedModuleUnitTester h ) + { + sm_extModuleUnitTester = h; + } + + /// Ditto static @property void moduleUnitTester( ModuleUnitTester h ) { sm_moduleUnitTester = h; } - /** - * Gets the current module unit tester. + * Gets the current legacy module unit tester. + * + * This property should not be used, but is supported for legacy purposes. + * + * Note that if the extended unit test handler is set, this handler will + * be ignored. * * Returns: - * The current module unit tester handler or null if none has been set. + * The current legacy module unit tester handler or null if none has been + * set. */ static @property ModuleUnitTester moduleUnitTester() { return sm_moduleUnitTester; } + /** + * Gets the current module unit tester. + * + * This handler overrides any legacy module unit tester set by the + * moduleUnitTester property. + * + * Returns: + * The current module unit tester handler or null if none has been + * set. + */ + static @property ExtendedModuleUnitTester extendedModuleUnitTester() + { + return sm_extModuleUnitTester; + } private: // NOTE: This field will only ever be set in a static ctor and should // never occur within any but the main thread, so it is safe to // make it __gshared. + __gshared ExtendedModuleUnitTester sm_extModuleUnitTester = null; __gshared ModuleUnitTester sm_moduleUnitTester = null; } @@ -440,14 +539,45 @@ extern (C) void profilegc_setlogfilename(string name); /** * This routine is called by the runtime to run module unit tests on startup. - * The user-supplied unit tester will be called if one has been supplied, + * The user-supplied unit tester will be called if one has been set, * otherwise all unit tests will be run in sequence. * + * If the extended unittest handler is registered, this function returns the + * result from that handler directly. + * + * If a legacy boolean returning custom handler is used, `false` maps to + * `UnitTestResult.fail`, and `true` maps to `UnitTestResult.pass`. This was + * the original behavior of the unit testing system. + * + * If no unittest custom handlers are registered, the following algorithm is + * executed (the behavior can be affected by the `--DRT-testmode` switch + * below): + * 1. Run all unit tests, tracking tests executed and passes. For each that + * fails, print the stack trace, and continue. + * 2. If there are no failures, set the summarize flag to false, and the + * runMain flag to true. + * 3. If there are failures, set the summarize flag to true, and the runMain + * flag to false. + * + * See the documentation for `UnitTestResult` for details on how the runtime + * treats the return value from this function. + * + * If the switch `--DRT-testmode` is passed to the executable, it can have + * one of 3 values: + * 1. "run-main": even if unit tests are run (and all pass), main is still run. + * This is currently the default. + * 2. "test-or-main": any unit tests present will cause the program to + * summarize the results and exit regardless of the result. This will be the + * default in 2.080. + * 3. "test-only", the runtime will always summarize and never run main, even + * if no tests are present. + * + * This command-line parameter does not affect custom unit test handlers. + * * Returns: - * true if execution should continue after testing is complete and false if - * not. Default behavior is to return true. + * A `UnitTestResult` struct indicating the result of running unit tests. */ -extern (C) bool runModuleUnitTests() +extern (C) UnitTestResult runModuleUnitTests() { // backtrace version( CRuntime_Glibc ) @@ -494,32 +624,60 @@ extern (C) bool runModuleUnitTests() } } - if( Runtime.sm_moduleUnitTester is null ) + if (Runtime.sm_extModuleUnitTester !is null) + return Runtime.sm_extModuleUnitTester(); + else if (Runtime.sm_moduleUnitTester !is null) + return Runtime.sm_moduleUnitTester() ? UnitTestResult.pass : UnitTestResult.fail; + UnitTestResult results; + foreach( m; ModuleInfo ) { - size_t failed = 0; - foreach( m; ModuleInfo ) + if( m ) { - if( m ) - { - auto fp = m.unitTest; + auto fp = m.unitTest; - if( fp ) + if( fp ) + { + ++results.executed; + try { - try - { - fp(); - } - catch( Throwable e ) - { - _d_print_throwable(e); - failed++; - } + fp(); + ++results.passed; + } + catch( Throwable e ) + { + _d_print_throwable(e); } } } - return failed == 0; } - return Runtime.sm_moduleUnitTester(); + import rt.config : rt_configOption; + if (results.passed != results.executed) + { + // by default, we always print a summary if there are failures. + results.summarize = true; + } + else switch (rt_configOption("testmode")) + { + case "": + // By default, run main. Switch to only doing unit tests in 2.080 + case "run-main": + results.runMain = true; + break; + case "test-only": + // Never run main, always summarize + results.summarize = true; + break; + case "test-or-main": + // only run main if there were no tests. Only summarize if we are not + // running main. + results.runMain = (results.executed == 0); + results.summarize = !results.runMain; + break; + default: + throw new Error("Unknown --DRT-testmode option: " ~ rt_configOption("testmode")); + } + + return results; } diff --git a/src/rt/dmain2.d b/src/rt/dmain2.d index 2473f2d880..4c124f17d2 100644 --- a/src/rt/dmain2.d +++ b/src/rt/dmain2.d @@ -40,6 +40,16 @@ version (NetBSD) import core.stdc.fenv; } +// not sure why we can't define this in one place, but this is to keep this +// module from importing core.runtime. +struct UnitTestResult +{ + size_t executed; + size_t passed; + bool runMain; + bool summarize; +} + extern (C) void _d_monitor_staticctor(); extern (C) void _d_monitor_staticdtor(); extern (C) void _d_critical_init(); @@ -52,7 +62,7 @@ extern (C) void rt_moduleTlsCtor(); extern (C) void rt_moduleDtor(); extern (C) void rt_moduleTlsDtor(); extern (C) void thread_joinAll(); -extern (C) bool runModuleUnitTests(); +extern (C) UnitTestResult runModuleUnitTests(); extern (C) void _d_initMonoTime(); version (OSX) @@ -472,8 +482,34 @@ extern (C) int _d_run_main(int argc, char **argv, MainFunc mainFunc) // thrown during cleanup, however, will abort the cleanup process. void runAll() { - if (rt_init() && runModuleUnitTests()) - tryExec({ result = mainFunc(args); }); + if (rt_init()) + { + auto utResult = runModuleUnitTests(); + assert(utResult.passed <= utResult.executed); + if (utResult.passed == utResult.executed) + { + if (utResult.summarize) + { + if (utResult.passed == 0) + .fprintf(.stderr, "No unittests run\n"); + else + .fprintf(.stderr, "%d unittests passed\n", + cast(int)utResult.passed); + } + if (utResult.runMain) + tryExec({ result = mainFunc(args); }); + else + result = EXIT_SUCCESS; + } + else + { + if (utResult.summarize) + .fprintf(.stderr, "%d/%d unittests FAILED\n", + cast(int)(utResult.executed - utResult.passed), + cast(int)utResult.executed); + result = EXIT_FAILURE; + } + } else result = EXIT_FAILURE; diff --git a/src/test_runner.d b/src/test_runner.d index 0fddcc2294..cdc7b3623c 100644 --- a/src/test_runner.d +++ b/src/test_runner.d @@ -8,7 +8,7 @@ ModuleInfo* getModuleInfo(string name) assert(0, "module '"~name~"' not found"); } -bool tester() +UnitTestResult tester() { return Runtime.args.length > 1 ? testModules() : testAll(); } @@ -16,9 +16,11 @@ bool tester() string mode; -bool testModules() +UnitTestResult testModules() { - bool ret = true; + UnitTestResult ret; + ret.summarize = false; + ret.runMain = false; foreach(name; Runtime.args[1..$]) { immutable pkg = ".package"; @@ -33,9 +35,11 @@ bool testModules() return ret; } -bool testAll() +UnitTestResult testAll() { - bool ret = true; + UnitTestResult ret; + ret.summarize = false; + ret.runMain = false; foreach(moduleInfo; ModuleInfo) { doTest(moduleInfo, ret); @@ -45,15 +49,17 @@ bool testAll() } -void doTest(ModuleInfo* moduleInfo, ref bool ret) +void doTest(ModuleInfo* moduleInfo, ref UnitTestResult ret) { if (auto fp = moduleInfo.unitTest) { auto name = moduleInfo.name; + ++ret.executed; try { immutable t0 = MonoTime.currTime; fp(); + ++ret.passed; printf("%.3fs PASS %.*s %.*s\n", (MonoTime.currTime - t0).total!"msecs" / 1000.0, cast(uint)mode.length, mode.ptr, @@ -66,7 +72,6 @@ void doTest(ModuleInfo* moduleInfo, ref bool ret) cast(uint)mode.length, mode.ptr, cast(uint)name.length, name.ptr, cast(uint)msg.length, msg.ptr); - ret = false; } } } @@ -79,7 +84,7 @@ shared static this() import core.runtime : dmd_coverSetMerge; dmd_coverSetMerge(true); } - Runtime.moduleUnitTester = &tester; + Runtime.extendedModuleUnitTester = &tester; debug mode = "debug"; else mode = "release"; diff --git a/test/unittest/Makefile b/test/unittest/Makefile new file mode 100644 index 0000000000..d54549d918 --- /dev/null +++ b/test/unittest/Makefile @@ -0,0 +1,44 @@ +include ../common.mak + +TESTS:=good goodn bad badn na + +DIFF:=diff +SED:=sed + +.PHONY: all clean +all: $(addprefix $(ROOT)/,$(addsuffix .done,$(TESTS))) + +$(ROOT)/good.done: HASMAIN=0 +$(ROOT)/goodn.done: HASMAIN=0 +$(ROOT)/bad.done: HASMAIN=0 +$(ROOT)/badn.done: HASMAIN=0 +$(ROOT)/na.done: HASMAIN=1 + +$(ROOT)/good.done: RETCODE=0 +$(ROOT)/goodn.done: RETCODE=0 +$(ROOT)/bad.done: RETCODE=1 +$(ROOT)/badn.done: RETCODE=1 +$(ROOT)/na.done: RETCODE=0 + +$(ROOT)/good.done: VER=PassNoPrintout +$(ROOT)/goodn.done: VER=GoodTests +$(ROOT)/bad.done: VER=FailNoPrintout +$(ROOT)/badn.done: VER=FailedTests +$(ROOT)/na.done: VER=NoTests + +$(ROOT)/good.done: TESTTEXT= +$(ROOT)/goodn.done: TESTTEXT=passed +$(ROOT)/bad.done: TESTTEXT= +$(ROOT)/badn.done: TESTTEXT=FAILED +$(ROOT)/na.done: TESTTEXT= + +$(ROOT)/%.done: customhandler.d + @echo Testing $* + $(QUIET)$(DMD) $(DFLAGS) -of$(ROOT)/tester_$(patsubst %.done,%, $(notdir $@)) customhandler.d -version=$(VER) + $(QUIET)$(TIMELIMIT)$(ROOT)/tester_$(patsubst %.done,%, $(notdir $@)) > $@ 2>&1; test $$? -eq $(RETCODE) + $(QUIET)test $(HASMAIN) -eq 0 || grep -q main $@ + $(QUIET)test $(HASMAIN) -eq 1 || ! grep -q main $@ + $(QUIET)test -z "$(TESTTEXT)" || grep -q "unittests $(TESTTEXT)" $@ + $(QUIET)test -n "$(TESTTEXT)" || ! grep -q "unittests" $@ +clean: + rm -rf $(GENERATED) diff --git a/test/unittest/customhandler.d b/test/unittest/customhandler.d new file mode 100644 index 0000000000..f5a04350d9 --- /dev/null +++ b/test/unittest/customhandler.d @@ -0,0 +1,21 @@ +import core.runtime; + +UnitTestResult customModuleUnitTester() +{ + version(GoodTests) return UnitTestResult(100, 100, false, true); + version(FailedTests) return UnitTestResult(100, 0, false, true); + version(NoTests) return UnitTestResult(0, 0, true, false); + version(FailNoPrintout) return UnitTestResult(100, 0, false, false); + version(PassNoPrintout) return UnitTestResult(100, 100, false, false); +} + +shared static this() +{ + Runtime.extendedModuleUnitTester = &customModuleUnitTester; +} + +void main() +{ + import core.stdc.stdio; + fprintf(stderr, "main\n"); +}