From 430c0380fa151d4fca62e7dbc5756743c31e0d0c Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 30 Apr 2018 15:19:26 +0200 Subject: [PATCH 1/2] Merge #13105: [qa] Add --failfast option to functional test runner 58f9a0a Use --failfast when running functional tests on Travis (James O'Beirne) bf720c1 Add --failfast option to functional test runner (James O'Beirne) Pull request description: Add the option (`--failfast`) to stop the functional test runner's execution when it encounters the first failure. Also cleans up run_test's arguments list ([no more mutable default for `args`](http://docs.python-guide.org/en/latest/writing/gotchas/#mutable-default-arguments)) and call site. Tree-SHA512: e854b1b1634bf613ae8ae88e715df1460982fa68db9d785aafeb5eccf5bf324c7f20dded2ca6840ebf18a28347ecac2138d6c7592507b34939b02609ef55e1b3 --- ci/test_integrationtests.sh | 2 +- test/functional/test_runner.py | 36 ++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/ci/test_integrationtests.sh b/ci/test_integrationtests.sh index ceff7967bf3d..fb5615eaa878 100755 --- a/ci/test_integrationtests.sh +++ b/ci/test_integrationtests.sh @@ -18,7 +18,7 @@ export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib cd build-ci/dashcore-$BUILD_TARGET set +e -./test/functional/test_runner.py --coverage --quiet --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS +./test/functional/test_runner.py --coverage --quiet --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS RESULT=$? set -e diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 95e9ff45b4b5..909be19a1cf8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -196,6 +196,7 @@ def main(): parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") + parser.add_argument('--failfast', action='store_true', help='stop execution after the first test failure') args, unknown_args = parser.parse_known_args() # args to be passed on always start with two dashes; tests are the remaining unknown args @@ -279,9 +280,21 @@ def main(): if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) - run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) + run_tests( + test_list, + config["environment"]["SRCDIR"], + config["environment"]["BUILDDIR"], + config["environment"]["EXEEXT"], + tmpdir, + jobs=args.jobs, + enable_coverage=args.coverage, + args=passon_args, + failfast=args.failfast, + ) + +def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=None, failfast=False): + args = args or [] -def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): # Warn if dashd is already running (unix only) try: pidof_output = subprocess.check_output(["pidof", "dashd"]) @@ -337,6 +350,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') + if failfast: + logging.debug("Early exiting after test failure") + break + print_results(test_results, max_len_name, (int(time.time() - time0))) if coverage: @@ -351,6 +368,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove all_passed = all(map(lambda test_result: test_result.was_successful, test_results)) + # This will be a no-op unless failfast is True in which case there may be dangling + # processes which need to be killed. + job_queue.kill_and_join() + sys.exit(not all_passed) def print_results(test_results, max_len_name, runtime): @@ -436,6 +457,17 @@ def get_next(self): return TestResult(name, status, int(time.time() - time0)), stdout, stderr print('.', end='', flush=True) + def kill_and_join(self): + """Send SIGKILL to all jobs and block until all have ended.""" + procs = [i[2] for i in self.jobs] + + for proc in procs: + proc.kill() + + for proc in procs: + proc.wait() + + class TestResult(): def __init__(self, name, status, time): self.name = name From ef59dce92e3247633d17ed6ec4130b65de75f188 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 2 Nov 2018 15:04:55 -0400 Subject: [PATCH 2/2] Merge #14630: test_runner: Remove travis specific code fa43626611 test_runner: Remove travis specific code (MarcoFalke) Pull request description: The tests are no longer run on travis, but in a docker, developer machines or a windows vm. The code was essentially dead for months now. Fix that by explicitly passing in `--ci` to the test runner on our docker and appveyor windows vm. Tree-SHA512: 5d48693c03e8eb27536658ccf9ba738fe93a72abd4b72c80caac084b5b2cdffa77a1031a671eeefe70b71d63500f55917803d4be54d01849722afdccb700a9e6 --- ci/test_integrationtests.sh | 2 +- test/functional/test_runner.py | 37 ++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/ci/test_integrationtests.sh b/ci/test_integrationtests.sh index fb5615eaa878..916bdce0af9e 100755 --- a/ci/test_integrationtests.sh +++ b/ci/test_integrationtests.sh @@ -18,7 +18,7 @@ export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib cd build-ci/dashcore-$BUILD_TARGET set +e -./test/functional/test_runner.py --coverage --quiet --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS +./test/functional/test_runner.py --ci --coverage --quiet --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS RESULT=$? set -e diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 909be19a1cf8..8d7ed8fad919 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -188,6 +188,7 @@ def main(): Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') + parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') @@ -275,24 +276,25 @@ def main(): subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h']) sys.exit(0) - check_script_list(config["environment"]["SRCDIR"]) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) run_tests( - test_list, - config["environment"]["SRCDIR"], - config["environment"]["BUILDDIR"], - config["environment"]["EXEEXT"], - tmpdir, + test_list=test_list, + src_dir=config["environment"]["SRCDIR"], + build_dir=config["environment"]["BUILDDIR"], + exeext=config["environment"]["EXEEXT"], + tmpdir=tmpdir, jobs=args.jobs, enable_coverage=args.coverage, args=passon_args, failfast=args.failfast, + runs_ci=args.ci, ) -def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=None, failfast=False): +def run_tests(*, test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=None, failfast=False, runs_ci): args = args or [] # Warn if dashd is already running (unix only) @@ -331,7 +333,14 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) #Run Tests - job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) + job_queue = TestHandler( + num_tests_parallel=jobs, + tests_dir=tests_dir, + tmpdir=tmpdir, + test_list=test_list, + flags=flags, + timeout_duration=20 * 60 if runs_ci else float('inf'), # in seconds + ) time0 = time.time() test_results = [] @@ -397,11 +406,12 @@ class TestHandler: Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None): - assert(num_tests_parallel >= 1) + def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, timeout_duration): + assert num_tests_parallel >= 1 self.num_jobs = num_tests_parallel self.tests_dir = tests_dir self.tmpdir = tmpdir + self.timeout_duration = timeout_duration self.test_list = test_list self.flags = flags self.num_running = 0 @@ -437,7 +447,7 @@ def get_next(self): time.sleep(.5) for j in self.jobs: (name, time0, proc, log_out, log_err) = j - if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60: + if int(time.time() - time0) > self.timeout_duration: # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not # providing useful output. proc.send_signal(signal.SIGINT) @@ -493,7 +503,7 @@ def was_successful(self): return self.status != "Failed" -def check_script_list(src_dir): +def check_script_list(*, src_dir, fail_on_warn): """Check scripts directory. Check that there are no scripts in the functional tests directory which are @@ -503,10 +513,11 @@ def check_script_list(src_dir): missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) - if os.getenv('TRAVIS') == 'true': + if fail_on_warn: # On travis this warning is an error to prevent merging incomplete commits into master sys.exit(1) + class RPCCoverage(object): """ Coverage reporting utilities for test_runner.