diff --git a/ci/test_integrationtests.sh b/ci/test_integrationtests.sh index ceff7967bf3d..916bdce0af9e 100755 --- a/ci/test_integrationtests.sh +++ b/ci/test_integrationtests.sh @@ -18,7 +18,7 @@ export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib cd build-ci/dashcore-$BUILD_TARGET set +e -./test/functional/test_runner.py --coverage --quiet --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS +./test/functional/test_runner.py --ci --coverage --quiet --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS RESULT=$? set -e diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 95e9ff45b4b5..8d7ed8fad919 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -188,6 +188,7 @@ def main(): Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') + parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') @@ -196,6 +197,7 @@ def main(): parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") + parser.add_argument('--failfast', action='store_true', help='stop execution after the first test failure') args, unknown_args = parser.parse_known_args() # args to be passed on always start with two dashes; tests are the remaining unknown args @@ -274,14 +276,27 @@ def main(): subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h']) sys.exit(0) - check_script_list(config["environment"]["SRCDIR"]) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) - run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) + run_tests( + test_list=test_list, + src_dir=config["environment"]["SRCDIR"], + build_dir=config["environment"]["BUILDDIR"], + exeext=config["environment"]["EXEEXT"], + tmpdir=tmpdir, + jobs=args.jobs, + enable_coverage=args.coverage, + args=passon_args, + failfast=args.failfast, + runs_ci=args.ci, + ) + +def run_tests(*, test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=None, failfast=False, runs_ci): + args = args or [] -def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): # Warn if dashd is already running (unix only) try: pidof_output = subprocess.check_output(["pidof", "dashd"]) @@ -318,7 +333,14 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) #Run Tests - job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) + job_queue = TestHandler( + num_tests_parallel=jobs, + tests_dir=tests_dir, + tmpdir=tmpdir, + test_list=test_list, + flags=flags, + timeout_duration=20 * 60 if runs_ci else float('inf'), # in seconds + ) time0 = time.time() test_results = [] @@ -337,6 +359,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') + if failfast: + logging.debug("Early exiting after test failure") + break + print_results(test_results, max_len_name, (int(time.time() - time0))) if coverage: @@ -351,6 +377,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove all_passed = all(map(lambda test_result: test_result.was_successful, test_results)) + # This will be a no-op unless failfast is True in which case there may be dangling + # processes which need to be killed. + job_queue.kill_and_join() + sys.exit(not all_passed) def print_results(test_results, max_len_name, runtime): @@ -376,11 +406,12 @@ class TestHandler: Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None): - assert(num_tests_parallel >= 1) + def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, timeout_duration): + assert num_tests_parallel >= 1 self.num_jobs = num_tests_parallel self.tests_dir = tests_dir self.tmpdir = tmpdir + self.timeout_duration = timeout_duration self.test_list = test_list self.flags = flags self.num_running = 0 @@ -416,7 +447,7 @@ def get_next(self): time.sleep(.5) for j in self.jobs: (name, time0, proc, log_out, log_err) = j - if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60: + if int(time.time() - time0) > self.timeout_duration: # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not # providing useful output. proc.send_signal(signal.SIGINT) @@ -436,6 +467,17 @@ def get_next(self): return TestResult(name, status, int(time.time() - time0)), stdout, stderr print('.', end='', flush=True) + def kill_and_join(self): + """Send SIGKILL to all jobs and block until all have ended.""" + procs = [i[2] for i in self.jobs] + + for proc in procs: + proc.kill() + + for proc in procs: + proc.wait() + + class TestResult(): def __init__(self, name, status, time): self.name = name @@ -461,7 +503,7 @@ def was_successful(self): return self.status != "Failed" -def check_script_list(src_dir): +def check_script_list(*, src_dir, fail_on_warn): """Check scripts directory. Check that there are no scripts in the functional tests directory which are @@ -471,10 +513,11 @@ def check_script_list(src_dir): missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) - if os.getenv('TRAVIS') == 'true': + if fail_on_warn: # On travis this warning is an error to prevent merging incomplete commits into master sys.exit(1) + class RPCCoverage(object): """ Coverage reporting utilities for test_runner.