From 8aa773824799b4413a42ed3dbf90946ca81e8457 Mon Sep 17 00:00:00 2001 From: JamesC Date: Fri, 25 Oct 2019 14:28:08 +0200 Subject: [PATCH 1/7] Removal of network_event_loop when NetworkThread instance is closed. The asyncio.new_event_loop() instance is now removed from the NetworkThread class during shutdown. This enables a NetworkThread instance to be restarted after being closed. The current NetworkThread class guards against an existing new_event_loop during initialization. --- test/functional/test_framework/mininode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 779863df7963..fceea4b796b2 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -480,7 +480,8 @@ def close(self, timeout=10): wait_until(lambda: not self.network_event_loop.is_running(), timeout=timeout) self.network_event_loop.close() self.join(timeout) - + # Safe to remove event loop. + NetworkThread.network_event_loop = None class P2PDataStore(P2PInterface): """A P2P data store class. From 594364afdc2c35877b8bedb9f07dd40156f5206e Mon Sep 17 00:00:00 2001 From: JamesC Date: Sat, 26 Oct 2019 16:34:42 +0200 Subject: [PATCH 2/7] Refactored BitcoinTestFramework main method into setup and shutdown. Setup and shutdown code now moved into dedicated methods. Test "success" is added as a BitcoinTestFramework member, which can be accessed outside of main. --- .../test_framework/test_framework.py | 75 +++++++++++-------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 9aff08fdc7da..9f1e104605d3 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -105,6 +105,31 @@ def __init__(self): def main(self): """Main function. This should not be overridden by the subclass test scripts.""" + self.parse_args() + + try: + self.setup() + self.run_test() + except BaseException as e: + self.success = TestStatus.FAILED + if isinstance(e, JSONRPCException): + self.log.error("JSONRPC error") + elif isinstance(e, SkipTest): + self.log.warning("Test Skipped: %s" % e.message) + self.success = TestStatus.SKIPPED + elif isinstance(e, AssertionError): + self.log.error("Assertion failed") + elif isinstance(e, KeyError): + self.log.error("Key error") + elif isinstance(e, Exception): + self.log.error("Unexpected exception caught during testing") + elif isinstance(e, KeyboardInterrupt): + self.log.warning("Exiting after keyboard interrupt") + finally: + exit_code = self.shutdown() + sys.exit(exit_code) + + def parse_args(self): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") @@ -135,6 +160,9 @@ def main(self): self.add_options(parser) self.options = parser.parse_args() + def setup(self): + """Call this method to startup the test-framework object with options set.""" + PortSeed.n = self.options.port_seed check_json_precision() @@ -181,33 +209,20 @@ def main(self): self.network_thread = NetworkThread() self.network_thread.start() - success = TestStatus.FAILED + if self.options.usecli: + if not self.supports_cli: + raise SkipTest("--usecli specified but test does not support using CLI") + self.skip_if_no_cli() + self.skip_test_if_missing_module() + self.setup_chain() + self.setup_network() - try: - if self.options.usecli: - if not self.supports_cli: - raise SkipTest("--usecli specified but test does not support using CLI") - self.skip_if_no_cli() - self.skip_test_if_missing_module() - self.setup_chain() - self.setup_network() - self.run_test() - success = TestStatus.PASSED - except JSONRPCException: - self.log.exception("JSONRPC error") - except SkipTest as e: - self.log.warning("Test Skipped: %s" % e.message) - success = TestStatus.SKIPPED - except AssertionError: - self.log.exception("Assertion failed") - except KeyError: - self.log.exception("Key error") - except Exception: - self.log.exception("Unexpected exception caught during testing") - except KeyboardInterrupt: - self.log.warning("Exiting after keyboard interrupt") - - if success == TestStatus.FAILED and self.options.pdbonfailure: + self.success = TestStatus.PASSED + + def shutdown(self): + """Call this method to shutdown the test-framework object.""" + + if self.success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() @@ -225,7 +240,7 @@ def main(self): should_clean_up = ( not self.options.nocleanup and not self.options.noshutdown and - success != TestStatus.FAILED and + self.success != TestStatus.FAILED and not self.options.perf ) if should_clean_up: @@ -238,10 +253,10 @@ def main(self): self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False - if success == TestStatus.PASSED: + if self.success == TestStatus.PASSED: self.log.info("Tests successful") exit_code = TEST_EXIT_PASSED - elif success == TestStatus.SKIPPED: + elif self.success == TestStatus.SKIPPED: self.log.info("Test skipped") exit_code = TEST_EXIT_SKIPPED else: @@ -251,7 +266,7 @@ def main(self): logging.shutdown() if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) - sys.exit(exit_code) + return exit_code # Methods to override in subclass test scripts. def set_test_params(self): From 3e55f72f107fd8c5ec28ef758c296d850a675822 Mon Sep 17 00:00:00 2001 From: JamesC Date: Sat, 26 Oct 2019 16:02:23 +0200 Subject: [PATCH 3/7] Added removal of logging handlers. In order for BitcoinTestFramework to correctly restart after shutdown, the previous logging handlers need to be removed, or else logging will continue in the previous temp directory. "Flush" ensures buffers are emptied, and "close" ensures file handler close logging file. --- test/functional/test_framework/test_framework.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 9f1e104605d3..8b7533929a3d 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -263,9 +263,13 @@ def shutdown(self): self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) exit_code = TEST_EXIT_FAILED - logging.shutdown() if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) + + for h in list(self.log.handlers): + h.flush() + h.close() + self.log.removeHandler(h) return exit_code # Methods to override in subclass test scripts. From 2c6fd517ed688343f306af81f725d022168b5f26 Mon Sep 17 00:00:00 2001 From: JamesC Date: Sat, 26 Oct 2019 16:03:21 +0200 Subject: [PATCH 4/7] Clear TestNode objects after shutdown. TestNode objects need to be removed during shutdown, as setup_nodes does not remove previous TestNode objects from previous test runs during setup. --- test/functional/test_framework/test_framework.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8b7533929a3d..b1f6f57195b0 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -270,6 +270,8 @@ def shutdown(self): h.flush() h.close() self.log.removeHandler(h) + + self.nodes.clear() return exit_code # Methods to override in subclass test scripts. From 2aa26a6712fe44fecd26cfcc3dd0a8cd7f20ae1a Mon Sep 17 00:00:00 2001 From: JamesC Date: Sat, 26 Oct 2019 16:00:02 +0200 Subject: [PATCH 5/7] Moved assert num_nodes is set into main(). This allows a BitcoinTestFramework child class to set test parameters in an overridden setup() rather than in an overridden set_test_params(). --- test/functional/test_framework/test_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index b1f6f57195b0..6f327793a522 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -100,11 +100,11 @@ def __init__(self): self.bind_to_localhost_only = True self.set_test_params() - assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" - def main(self): """Main function. This should not be overridden by the subclass test scripts.""" + assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" + self.parse_args() try: From e3455c744f3e0e95bc64810e1e516df6cd0e4091 Mon Sep 17 00:00:00 2001 From: JamesC Date: Thu, 3 Oct 2019 17:58:30 -0400 Subject: [PATCH 6/7] Added TestWrapper class. A BitcoinTestFramework child class which can be imported by an external user or project. TestWrapper.setup() initiates an underlying BitcoinTestFramework object with bitcoind subprocesses, rpc interfaces and test logging. TestWrapper.shutdown() safely tears down the BitcoinTestFramework object. --- .../functional/test_framework/test_wrapper.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 test/functional/test_framework/test_wrapper.py diff --git a/test/functional/test_framework/test_wrapper.py b/test/functional/test_framework/test_wrapper.py new file mode 100644 index 000000000000..e52bf8cd7d50 --- /dev/null +++ b/test/functional/test_framework/test_wrapper.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Wrapper Class for BitcoinTestFramework. + +The TestWrapper class extends the BitcoinTestFramework +rpc & daemon process management functionality to external +python environments. + +It is a singleton class, which ensures that users only +start a single TestWrapper at a time.""" + +import argparse +from os import getpid +from os.path import abspath, join + +from test_framework.test_framework import BitcoinTestFramework + +class TestWrapper: + + class __TestWrapper(BitcoinTestFramework): + + def set_test_params(self): + pass + + def run_test(self): + pass + + def setup(self, **kwargs): + + if self.running: + print("TestWrapper is already running!") + return + + self.setup_clean_chain = kwargs.get('setup_clean_chain',True) + self.num_nodes = kwargs.get('num_nodes', 1) + self.network_thread = kwargs.get('network_thread', None) + self.rpc_timeout = kwargs.get('rpc_timeout', 60) + self.supports_cli = kwargs.get('supports_cli', False) + self.bind_to_localhost_only = kwargs.get('bind_to_localhost_only', True) + + self.options = argparse.Namespace + self.options.nocleanup = kwargs.get('nocleanup', False) + self.options.noshutdown = kwargs.get('noshutdown', False) + self.options.cachedir = kwargs.get('cachedir', abspath(join(__file__ ,"../../../..") + "/test/cache")) + self.options.tmpdir = kwargs.get('tmpdir', None) + self.options.loglevel = kwargs.get('loglevel', 'INFO') + self.options.trace_rpc = kwargs.get('trace_rpc', False) + self.options.port_seed = kwargs.get('port_seed', getpid()) + self.options.coveragedir = kwargs.get('coveragedir', None) + self.options.configfile = kwargs.get('configfile', abspath(join(__file__ ,"../../../..") + "/test/config.ini")) + self.options.pdbonfailure = kwargs.get('pdbonfailure', False) + self.options.usecli = kwargs.get('usecli', False) + self.options.perf = kwargs.get('perf', False) + self.options.randomseed = kwargs.get('randomseed', None) + + self.options.bitcoind = kwargs.get('bitcoind', abspath(join(__file__ ,"../../../..") + "/src/bitcoind")) + self.options.bitcoincli = kwargs.get('bitcoincli', None) + + super().setup() + self.running = True + + def shutdown(self): + if not self.running: + print("TestWrapper is not running!") + else: + super().shutdown() + self.running = False + + instance = None + + def __new__(cls): + if not TestWrapper.instance: + TestWrapper.instance = TestWrapper.__TestWrapper() + TestWrapper.instance.running = False + return TestWrapper.instance + + def __getattr__(self, name): + return getattr(self.instance, name) + + def __setattr__(self, name, value): + return setattr(self.instance, name, value) From bf4fc418b4822b3c0983035d7cbd23a146ff4993 Mon Sep 17 00:00:00 2001 From: JamesC Date: Thu, 24 Oct 2019 20:33:06 +0200 Subject: [PATCH 7/7] Added documentation for test_wrapper submodule. --- doc/test-wrapper.md | 122 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 doc/test-wrapper.md diff --git a/doc/test-wrapper.md b/doc/test-wrapper.md new file mode 100644 index 000000000000..ab57d87f1349 --- /dev/null +++ b/doc/test-wrapper.md @@ -0,0 +1,122 @@ +Test Wrapper for Interactive Environments +========================================= + +This document describes the usage of the `TestWrapper` submodule in the `test_framework` module of the functional test framework. + +The TestWrapper submodule extends the `BitcoinTestFramework` functionality to external interactive environments for prototyping and educational purposes. Just like `BitcoinTestFramework`, the TestWrapper allows the user to: + +* Manage regtest bitcoind subprocesses. +* Access RPC interfaces of these bitcoind instances. +* Log events to functional test logging utility. + +The `TestWrapper` can be useful in interactive environments such as the Python3 command-line interpreter or [Jupyter](https://jupyter.org/) notebooks running a Python3 kernel, where is is necessary to extend the object lifetime of the underlying `BitcoinTestFramework` between user inputs. + +## 1. Requirements + +* Python3 +* `bitcoind` built in the same bitcoin repository as the TestWrapper. + +## 2. Importing TestWrapper from the Bitcoin Core repository + +We can import the TestWrapper by adding the path of the Bitcoin Core `test_framework` module to the beginning of the PATH variable, and then importing the `TestWrapper` class from the `test_wrapper` sub-package. + +``` +>>> import sys +>>> sys.path.insert(0, "/path/to/bitcoin/test/functional/test_framework") +>>> from test_framework.test_wrapper import TestWrapper +``` + +The following TestWrapper methods manage the lifetime of the underlying bitcoind processes and logging utilities. + +* `TestWrapper.setup()` +* `TestWrapper.shutdown()` + +The TestWrapper inherits all BitcoinTestFramework members and methods, such as: +* `TestWrapper.nodes[index].rpc_method()` +* `TestWrapper.log.info("Custom log message")` + +The following sections demonstrate how to initialize, run and shutdown a TestWrapper object in an interactive Python3 environment. + +## 3. Initializing a TestWrapper object + +``` +>>> test = TestWrapper() +>>> test.setup("num_nodes"=2) +20XX-XX-24TXX:XX:XX.XXXXXXX TestFramework (INFO): Initializing test directory /path/to/bitcoin_func_test_XXXXXXX +``` +The TestWrapper supports all functional test parameters of the Bitcoin TestFramework class. The full set of argument keywords which can be used to initialize the TestWrapper can be found [here](../test/functional/test_framework/test_wrapper.py). + +**Note: Running multiple instances of TestWrapper is not allowed.** +This also ensures that logging remains consolidated in the same temporary folder. If you need more bitcoind nodes than set by default (1), simply increase the `num_nodes` parameter during setup. + +``` +>>> test2 = TestWrapper() +>>> test2.setup() +TestWrapper is already running! +``` + +## 4. Interacting with the TestWrapper + +Unlike the BitcoinTestFramework class, the TestWrapper keeps the underlying Bitcoind subprocesses (nodes) and logging utilities running, until the user explicitly shuts down the TestWrapper object. + +During the time between the `setup` and `shutdown` calls, all `bitcoind` node processes and BitcoinTestFramework convenience methods can be accessed interactively. + +**Example: Mining a regtest chain** + +By default, the TestWrapper nodes are initialized with a clean chain. This means that each node has at block height 0 after initialization of the TestWrapper. + +``` +>>> test.nodes[0].getblockchaininfo()["blocks"] +0 +``` + +We now generate 101 regtest blocks, and send these to a wallet address owned by the first node. + +``` +>>> address = test.nodes[0].getnewaddress() +>>> test.nodes[0].generatetoaddress(101, address) +['2b98dd0044aae6f1cca7f88a0acf366a4bfe053c7f7b00da3c0d115f03d67efb', ... +``` +Since the two nodes are each initialized to establish a connection to the other during `setup`, the second node will receive the newly mined blocks after they propagate. + +``` +>>> test.nodes[1].getblockchaininfo()["blocks"] +101 +``` +The block rewards of the first block are now spendable by the wallet of the first node. + +``` +>>> test.nodes[0].getbalance() +Decimal('50.00000000') +``` + +We can also log custom events to the logger. + +``` +>>> TestWrapper.log.info("Successfully mined regtest chain!") +``` + +**Note: Please also consider the functional test [readme](../test/functional/README.md), which provides an overview of the test-framework**. Modules such as [key.py](../test/functional/test_framework/key.py), [script.py](../test/functional/test_framework/script.py) and [messages.py](../test/functional/test_framework/messages.py) are especially useful in constructing objects which can be passed to the bitcoind nodes managed by a running TestWrapper object. + +## 5. Shutting the TestWrapper down + +Shutting down the TestWrapper will safely tear down all running bitcoind instances and remove all temporary data and logging directories. + +``` +>>> test.shutdown() +20XX-XX-24TXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-24TXX:XX:XX.XXXXXXX TestFramework (INFO): Cleaning up /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-24TXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` +To prevent the logs from being removed after a shutdown, simply set the `TestWrapper.options.nocleanup` member to `True`. +``` +>>> test.options.nocleanup = True +>>> test.shutdown() +20XX-XX-24TXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-24TXX:XX:XX.XXXXXXX TestFramework (INFO): Not cleaning up dir /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-24TXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` + +The following utility consolidates logs from the bitcoind nodes and the underlying BitcoinTestFramework: + +* `/path/to/bitcoin/test/functional/combine_logs.py '/path/to/bitcoin_func_test_XXXXXXX'` \ No newline at end of file