diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 82a14af9da8c..87265fd23531 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -7,15 +7,14 @@ Version 1 compact blocks are pre-segwit (txids) Version 2 compact blocks are post-segwit (wtxids) """ -from decimal import Decimal import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, NODE_WITNESS, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex +from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_tx, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256 from test_framework.mininode import mininode_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_bip9_status, satoshi_round, sync_blocks, wait_until +from test_framework.util import assert_equal, get_bip9_status, wait_until # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): @@ -76,8 +75,6 @@ def request_headers_and_sync(self, locator, hashstop=0): wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock) self.clear_block_announcement() - # Block until a block announcement for a particular block hash is - # received. def wait_for_block_announcement(self, block_hash, timeout=30): def received_hash(): return (block_hash in self.announced_blockhashes) @@ -94,34 +91,123 @@ def send_await_disconnect(self, message, timeout=30): class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - # Node0 = pre-segwit, node1 = segwit-aware - self.num_nodes = 2 - # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). - # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] - self.utxos = [] + self.num_nodes = 1 def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def build_block_on_tip(self, node, segwit=False): + def build_block_on_tip(self, node): + """Builds a witness block on the tip.""" height = node.getblockcount() tip = node.getbestblockhash() mtp = node.getblockheader(tip)['mediantime'] block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) block.nVersion = 4 - if segwit: - add_witness_commitment(block) + add_witness_commitment(block) block.solve() return block - # Create 10 more anyone-can-spend utxo's for testing. - def make_utxos(self): - # Doesn't matter which node we use, just use node0. + def build_block_with_transactions(self, node, utxo, num_transactions): + """Builds a witness block with num_transactions on the tip. + + The transactions are built as a chain based on utxo.""" + block = self.build_block_on_tip(node) + + for i in range(num_transactions): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) + tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) + tx.rehash() + utxo = [tx.sha256, 0, tx.vout[0].nValue] + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() + add_witness_commitment(block) + block.solve() + return block + + def check_announcement_of_new_block(self, peer, predicate): + """Generate a block on the node and check that it is announced to peer.""" + peer.clear_block_announcement() + block_hash = int(self.nodes[0].generate(1)[0], 16) + peer.wait_for_block_announcement(block_hash, timeout=30) + assert peer.block_announced + + with mininode_lock: + assert predicate(peer), ( + "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( + block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) + + def request_cb_announcements(self, peer): + """Send a getheaders and a sendcmpct to the node.""" + tip = self.nodes[0].getbestblockhash() + peer.get_headers(locator=[int(tip, 16)], hashstop=0) + + msg = msg_sendcmpct() + msg.version = peer.version + msg.announce = True + peer.send_and_ping(msg) + + def run_test(self): + # Setup the p2p connections + self.segwit_peer_1 = self.nodes[0].add_p2p_connection(TestP2PConn()) + self.segwit_peer_1.version = 2 + self.segwit_peer_2 = self.nodes[0].add_p2p_connection(TestP2PConn()) + self.segwit_peer_2.version = 2 + self.legacy_peer = self.nodes[0].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) + self.legacy_peer.version = 1 + + # Construct UTXOs for use in later tests. + self.make_utxos(self.segwit_peer_1) + + assert_equal(get_bip9_status(self.nodes[0], "segwit")["status"], 'active') + + self.log.info("Testing SENDCMPCT p2p message... ") + self.test_sendcmpct(self.segwit_peer_1) + self.test_sendcmpct(self.segwit_peer_2) + self.test_sendcmpct_legacy(self.legacy_peer) + + self.log.info("Testing compactblock construction...") + self.test_compactblock_construction(self.legacy_peer) + self.test_compactblock_construction(self.segwit_peer_1) + + self.log.info("Testing compactblock requests...") + self.test_compactblock_requests(self.segwit_peer_1) + + self.log.info("Testing getblocktxn requests...") + self.test_getblocktxn_requests(self.segwit_peer_1) + + self.log.info("Testing getblocktxn handler...") + self.test_getblocktxn_handler(self.segwit_peer_1) + self.test_getblocktxn_handler(self.legacy_peer) + + self.log.info("Testing compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.segwit_peer_1) + self.test_compactblocks_not_at_tip(self.legacy_peer) + + self.log.info("Testing handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.segwit_peer_1) + + self.log.info("Testing reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.segwit_peer_1, self.segwit_peer_2) + + self.log.info("Testing end-to-end block relay...") + self.test_end_to_end_block_relay([self.segwit_peer_1, self.legacy_peer]) + + self.log.info("Testing handling of invalid compact blocks...") + self.test_invalid_tx_in_compactblock(self.segwit_peer_1) + self.test_invalid_tx_in_compactblock(self.legacy_peer) + + self.log.info("Testing invalid index in cmpctblock message...") + self.test_invalid_cmpctblock_message(self.segwit_peer_1) + + def make_utxos(self, peer): + """ Create 10 anyone-can-spend UTXOs for testing and send balance to bech32 output.""" block = self.build_block_on_tip(self.nodes[0]) - self.test_node.send_and_ping(msg_block(block)) + peer.send_and_ping(msg_witness_block(block)) assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 - self.nodes[0].generate(100) + address = self.nodes[0].getnewaddress(address_type="bech32") + self.nodes[0].generatetoaddress(100, address) total_value = block.vtx[0].vout[0].nValue out_value = total_value // 10 @@ -134,206 +220,175 @@ def make_utxos(self): block2 = self.build_block_on_tip(self.nodes[0]) block2.vtx.append(tx) block2.hashMerkleRoot = block2.calc_merkle_root() + add_witness_commitment(block2) block2.solve() - self.test_node.send_and_ping(msg_block(block2)) + peer.send_and_ping(msg_witness_block(block2)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) - self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) - return - - # Test "sendcmpct" (between peers preferring the same version): - # - No compact block announcements unless sendcmpct is sent. - # - If sendcmpct is sent with version > preferred_version, the message is ignored. - # - If sendcmpct is sent with boolean 0, then block announcements are not - # made with compact blocks. - # - If sendcmpct is then sent with boolean 1, then new block announcements - # are made with compact blocks. - # If old_node is passed in, request compact blocks with version=preferred-1 - # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + self.utxos = [[tx.sha256, i, out_value] for i in range(10)] + + def test_sendcmpct(self, peer, legacy_peer=None): + """Test sendcmpct between peers preferring the same version + + - No compact block announcements unless sendcmpct is sent. + - If sendcmpct is sent with version > preferred_version, the message is ignored. + - If sendcmpct is sent with boolean 0, then block announcements are not + made with compact blocks. + - If sendcmpct is then sent with boolean 1, then new block announcements + are made with compact blocks.""" + # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): - return (len(test_node.last_sendcmpct) > 0) + return (len(peer.last_sendcmpct) > 0) wait_until(received_sendcmpct, timeout=30, lock=mininode_lock) with mininode_lock: - # Check that the first version received is the preferred one - assert_equal(test_node.last_sendcmpct[0].version, preferred_version) - # And that we receive versions down to 1. - assert_equal(test_node.last_sendcmpct[-1].version, 1) - test_node.last_sendcmpct = [] - - tip = int(node.getbestblockhash(), 16) + # Check that we receive version 2 first. + assert_equal(peer.last_sendcmpct[0].version, 2) + # And that we receive version 1. + assert_equal(peer.last_sendcmpct[-1].version, 1) + peer.last_sendcmpct = [] - def check_announcement_of_new_block(node, peer, predicate): - peer.clear_block_announcement() - block_hash = int(node.generate(1)[0], 16) - peer.wait_for_block_announcement(block_hash, timeout=30) - assert peer.block_announced - - with mininode_lock: - assert predicate(peer), ( - "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( - block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) + tip = int(self.nodes[0].getbestblockhash(), 16) # We shouldn't get any block announcements via cmpctblock yet. - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" not in p.last_message) # Try one more time, this time after requesting headers. - test_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) + peer.request_headers_and_sync(locator=[tip]) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) # Test a few ways of using sendcmpct that should NOT # result in compact block announcements. # Before each test, sync the headers chain. - test_node.request_headers_and_sync(locator=[tip]) + peer.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with too-high version sendcmpct = msg_sendcmpct() - sendcmpct.version = preferred_version + 1 + sendcmpct.version = 3 sendcmpct.announce = True - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + peer.send_and_ping(sendcmpct) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. - test_node.request_headers_and_sync(locator=[tip]) + peer.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with valid version, but announce=False - sendcmpct.version = preferred_version + sendcmpct.version = 2 sendcmpct.announce = False - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + peer.send_and_ping(sendcmpct) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. - test_node.request_headers_and_sync(locator=[tip]) + peer.request_headers_and_sync(locator=[tip]) # Finally, try a SENDCMPCT message with announce=True - sendcmpct.version = preferred_version + sendcmpct.version = 2 sendcmpct.announce = True - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + peer.send_and_ping(sendcmpct) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" in p.last_message) # Try one more time (no headers sync should be needed!) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" in p.last_message) # Try one more time, after turning on sendheaders - test_node.send_and_ping(msg_sendheaders()) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + peer.send_and_ping(msg_sendheaders()) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" in p.last_message) - # Try one more time, after sending a version-1, announce=false message. - sendcmpct.version = preferred_version - 1 + # Try one more time, after sending version 1, announce=false message. + sendcmpct.version = 1 sendcmpct.announce = False - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + peer.send_and_ping(sendcmpct) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" in p.last_message) # Now turn off announcements - sendcmpct.version = preferred_version + sendcmpct.version = 2 sendcmpct.announce = False - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) - - if old_node is not None: - # Verify that a peer using an older protocol version can receive - # announcements from this node. - sendcmpct.version = preferred_version - 1 - sendcmpct.announce = True - old_node.send_and_ping(sendcmpct) - # Header sync - old_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message) - - # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. - def test_invalid_cmpctblock_message(self): - self.nodes[0].generate(101) - block = self.build_block_on_tip(self.nodes[0]) + peer.send_and_ping(sendcmpct) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) - cmpct_block = P2PHeaderAndShortIDs() - cmpct_block.header = CBlockHeader(block) - cmpct_block.prefilled_txn_length = 1 - # This index will be too high - prefilled_txn = PrefilledTransaction(1, block.vtx[0]) - cmpct_block.prefilled_txn = [prefilled_txn] - self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + def test_sendcmpct_legacy(self, peer): + """Verify that a peer using an older protocol version can receive announcements from this node.""" + sendcmpct = msg_sendcmpct() + sendcmpct.version = 1 + sendcmpct.announce = True + peer.send_and_ping(sendcmpct) + # Header sync + tip = int(self.nodes[0].getbestblockhash(), 16) + peer.request_headers_and_sync(locator=[tip]) + self.check_announcement_of_new_block(peer, lambda p: "cmpctblock" in p.last_message) + + def test_compactblock_construction(self, peer): + """Compare the generated shortids to what we expect based on BIP 152, given bitcoind's choice of nonce.""" - # Compare the generated shortids to what we expect based on BIP 152, given - # bitcoind's choice of nonce. - def test_compactblock_construction(self, node, test_node, version, use_witness_address): # Generate a bunch of transactions. - node.generate(101) num_transactions = 25 - address = node.getnewaddress() - if use_witness_address: - # Want at least one segwit spend, so move all funds to - # a witness address. - address = node.getnewaddress(address_type='bech32') - value_to_send = node.getbalance() - node.sendtoaddress(address, satoshi_round(value_to_send - Decimal(0.1))) - node.generate(1) + address = self.nodes[0].getnewaddress() segwit_tx_generated = False for i in range(num_transactions): - txid = node.sendtoaddress(address, 0.1) - hex_tx = node.gettransaction(txid)["hex"] + txid = self.nodes[0].sendtoaddress(address, 0.1) + hex_tx = self.nodes[0].gettransaction(txid)["hex"] tx = FromHex(CTransaction(), hex_tx) if not tx.wit.is_null(): segwit_tx_generated = True - if use_witness_address: - assert segwit_tx_generated # check that our test is not broken + # check that our test is not broken + assert segwit_tx_generated # Wait until we've seen the block announcement for the resulting tip - tip = int(node.getbestblockhash(), 16) - test_node.wait_for_block_announcement(tip) + tip = int(self.nodes[0].getbestblockhash(), 16) + peer.wait_for_block_announcement(tip) # Make sure we will receive a fast-announce compact block - self.request_cb_announcements(test_node, node, version) + self.request_cb_announcements(peer) # Now mine a block, and look at the resulting compact block. - test_node.clear_block_announcement() - block_hash = int(node.generate(1)[0], 16) + peer.clear_block_announcement() + block_hash = int(self.nodes[0].generate(1)[0], 16) # Store the raw block in our internal format. - block = FromHex(CBlock(), node.getblock("%064x" % block_hash, False)) + block = FromHex(CBlock(), self.nodes[0].getblock("%064x" % block_hash, False)) for tx in block.vtx: tx.calc_sha256() block.rehash() # Wait until the block was announced (via compact blocks) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + wait_until(peer.received_block_announcement, timeout=30, lock=mininode_lock) # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert "cmpctblock" in test_node.last_message + assert "cmpctblock" in peer.last_message # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + header_and_shortids = HeaderAndShortIDs(peer.last_message["cmpctblock"].header_and_shortids) + self.check_compactblock_construction_from_block(peer.version, header_and_shortids, block_hash, block) # Now fetch the compact block using a normal non-announce getdata with mininode_lock: - test_node.clear_block_announcement() + peer.clear_block_announcement() inv = CInv(4, block_hash) # 4 == "CompactBlock" - test_node.send_message(msg_getdata([inv])) + peer.send_message(msg_getdata([inv])) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + wait_until(peer.received_block_announcement, timeout=30, lock=mininode_lock) # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert "cmpctblock" in test_node.last_message + assert "cmpctblock" in peer.last_message # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + header_and_shortids = HeaderAndShortIDs(peer.last_message["cmpctblock"].header_and_shortids) + self.check_compactblock_construction_from_block(peer.version, header_and_shortids, block_hash, block) def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): - # Check that we got the right block! + """Check that the compact block sent to the peer is correct.""" header_and_shortids.header.calc_sha256() assert_equal(header_and_shortids.header.sha256, block_hash) - # Make sure the prefilled_txn appears to have included the coinbase + # Make sure the prefilled_txn includes the coinbase assert len(header_and_shortids.prefilled_txn) >= 1 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) # Check that all prefilled_txn entries match what's in the block. + # We expect only the coinbase to be prefilled since the node sent all txs to the peer earlier. for entry in header_and_shortids.prefilled_txn: entry.tx.calc_sha256() # This checks the non-witness parts of the tx agree @@ -369,30 +424,29 @@ def check_compactblock_construction_from_block(self, version, header_and_shortid header_and_shortids.shortids.pop(0) index += 1 - # Test that bitcoind requests compact blocks when we announce new blocks - # via header or inv, and that responding to getblocktxn causes the block - # to be successfully reconstructed. - # Post-segwit: upgraded nodes would only make this request of cb-version-2, - # NODE_WITNESS peers. Unupgraded nodes would still make this request of - # any cb-version-1-supporting peer. - def test_compactblock_requests(self, node, test_node, version, segwit): + def test_compactblock_requests(self, peer): + """Test compactblock requests: + + - bitcoind requests compact blocks when we announce new blocks via header or inv + - responding to getblocktxn causes the block to be successfully reconstructed.""" + # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: - block = self.build_block_on_tip(node, segwit=segwit) + block = self.build_block_on_tip(self.nodes[0]) with mininode_lock: - test_node.last_message.pop("getdata", None) + peer.last_message.pop("getdata", None) if announce == "inv": - test_node.send_message(msg_inv([CInv(2, block.sha256)])) - wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) - test_node.send_header_for_blocks([block]) + peer.send_message(msg_inv([CInv(2, block.sha256)])) + wait_until(lambda: "getheaders" in peer.last_message, timeout=30, lock=mininode_lock) + peer.send_header_for_blocks([block]) else: - test_node.send_header_for_blocks([block]) - wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert_equal(test_node.last_message["getdata"].inv[0].type, 4) - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) + peer.send_header_for_blocks([block]) + wait_until(lambda: "getdata" in peer.last_message, timeout=30, lock=mininode_lock) + assert_equal(len(peer.last_message["getdata"].inv), 1) + assert_equal(peer.last_message["getdata"].inv[0].type, 4) # type 4 is compact block + assert_equal(peer.last_message["getdata"].inv[0].hash, block.sha256) # Send back a compactblock message that omits the coinbase comp_block = HeaderAndShortIDs() @@ -400,48 +454,28 @@ def test_compactblock_requests(self, node, test_node, version, segwit): comp_block.nonce = 0 [k0, k1] = comp_block.get_siphash_keys() coinbase_hash = block.vtx[0].sha256 - if version == 2: - coinbase_hash = block.vtx[0].calc_sha256(True) + coinbase_hash = block.vtx[0].calc_sha256(True) comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)] - test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) + peer.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert "getblocktxn" in test_node.last_message - absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() + assert "getblocktxn" in peer.last_message + absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request # Send the coinbase, and verify that the tip advances. - if version == 2: - msg = msg_witness_blocktxn() - else: - msg = msg_blocktxn() + msg = msg_witness_blocktxn() msg.block_transactions.blockhash = block.sha256 msg.block_transactions.transactions = [block.vtx[0]] - test_node.send_and_ping(msg) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - # Create a chain of transactions from given utxo, and add to a new block. - def build_block_with_transactions(self, node, utxo, num_transactions): - block = self.build_block_on_tip(node) - - for i in range(num_transactions): - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) - tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) - tx.rehash() - utxo = [tx.sha256, 0, tx.vout[0].nValue] - block.vtx.append(tx) + peer.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) - block.hashMerkleRoot = block.calc_merkle_root() - block.solve() - return block + def test_getblocktxn_requests(self, peer): + """Test getblocktxs requests - # Test that we only receive getblocktxn requests for transactions that the - # node needs, and that responding to them causes the block to be - # reconstructed. - def test_getblocktxn_requests(self, node, test_node, version): - with_witness = (version == 2) + The peer should only receive getblocktxn requests for transactions that the + node needs. Responding to them causes the block to be reconstructed.""" def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) @@ -459,291 +493,215 @@ def test_tip_after_message(node, peer, msg, tip): # that we receive getblocktxn messages back. utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 5) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + block = self.build_block_with_transactions(self.nodes[0], utxo, 5) comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, use_witness=with_witness) + comp_block.initialize_from_block(block) - test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) + test_getblocktxn_response(comp_block, peer, [1, 2, 3, 4, 5]) - msg_bt = msg_blocktxn() - if with_witness: - msg_bt = msg_witness_blocktxn() # serialize with witnesses + msg_bt = msg_witness_blocktxn() # serialize with witnesses msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) - test_tip_after_message(node, test_node, msg_bt, block.sha256) + test_tip_after_message(self.nodes[0], peer, msg_bt, block.sha256) utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 5) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + block = self.build_block_with_transactions(self.nodes[0], utxo, 5) # Now try interspersing the prefilled transactions - comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness) - test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5]) + test_getblocktxn_response(comp_block, peer, [2, 3, 4]) msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) - test_tip_after_message(node, test_node, msg_bt, block.sha256) + test_tip_after_message(self.nodes[0], peer, msg_bt, block.sha256) # Now try giving one transaction ahead of time. utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 5) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - test_node.send_and_ping(msg_tx(block.vtx[1])) - assert block.vtx[1].hash in node.getrawmempool() + block = self.build_block_with_transactions(self.nodes[0], utxo, 5) + peer.send_and_ping(msg_tx(block.vtx[1])) + assert block.vtx[1].hash in self.nodes[0].getrawmempool() # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. - comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness) - test_getblocktxn_response(comp_block, test_node, [5]) + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4]) + test_getblocktxn_response(comp_block, peer, [5]) msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) - test_tip_after_message(node, test_node, msg_bt, block.sha256) + test_tip_after_message(self.nodes[0], peer, msg_bt, block.sha256) # Now provide all transactions to the node before the block is # announced and verify reconstruction happens immediately. utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 10) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + block = self.build_block_with_transactions(self.nodes[0], utxo, 10) for tx in block.vtx[1:]: - test_node.send_message(msg_tx(tx)) - test_node.sync_with_ping() + peer.send_message(msg_tx(tx)) + peer.sync_with_ping() # Make sure all transactions were accepted. - mempool = node.getrawmempool() + mempool = self.nodes[0].getrawmempool() for tx in block.vtx[1:]: assert tx.hash in mempool # Clear out last request. with mininode_lock: - test_node.last_message.pop("getblocktxn", None) + peer.last_message.pop("getblocktxn", None) # Send compact block - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) - test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) + comp_block.initialize_from_block(block, prefill_list=[0]) + test_tip_after_message(self.nodes[0], peer, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert "getblocktxn" not in test_node.last_message - - # Incorrectly responding to a getblocktxn shouldn't cause the block to be - # permanently failed. - def test_incorrect_blocktxn_response(self, node, test_node, version): - if (len(self.utxos) == 0): - self.make_utxos() - utxo = self.utxos.pop(0) - - block = self.build_block_with_transactions(node, utxo, 10) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - # Relay the first 5 transactions from the block in advance - for tx in block.vtx[1:6]: - test_node.send_message(msg_tx(tx)) - test_node.sync_with_ping() - # Make sure all transactions were accepted. - mempool = node.getrawmempool() - for tx in block.vtx[1:6]: - assert tx.hash in mempool - - # Send compact block - comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) - test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - absolute_indexes = [] - with mininode_lock: - assert "getblocktxn" in test_node.last_message - absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() - assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) - - # Now give an incorrect response. - # Note that it's possible for bitcoind to be smart enough to know we're - # lying, since it could check to see if the shortid matches what we're - # sending, and eg disconnect us for misbehavior. If that behavior - # change was made, we could just modify this test by having a - # different peer provide the block further down, so that we're still - # verifying that the block isn't marked bad permanently. This is good - # enough for now. - msg = msg_blocktxn() - if version == 2: - msg = msg_witness_blocktxn() - msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) - test_node.send_and_ping(msg) - - # Tip should not have updated - assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) + assert "getblocktxn" not in peer.last_message - # We should receive a getdata request - wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) + def test_getblocktxn_handler(self, peer): + """Test that bitcoind will not send blocktxn responses for old blocks. - # Deliver the block - if version == 2: - test_node.send_and_ping(msg_witness_block(block)) - else: - test_node.send_and_ping(msg_block(block)) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - def test_getblocktxn_handler(self, node, test_node, version): - # bitcoind will not send blocktxn responses for blocks whose height is - # more than 10 blocks deep. + 10 blocks is the max depth for requesting blocktxns.""" MAX_GETBLOCKTXN_DEPTH = 10 - chain_height = node.getblockcount() + chain_height = self.nodes[0].getblockcount() current_height = chain_height while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): - block_hash = node.getblockhash(current_height) - block = FromHex(CBlock(), node.getblock(block_hash, False)) + block_hash = self.nodes[0].getblockhash(current_height) + block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False)) msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) num_to_request = random.randint(1, len(block.vtx)) msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) - test_node.send_message(msg) - wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock) + peer.send_message(msg) + wait_until(lambda: "blocktxn" in peer.last_message, timeout=10, lock=mininode_lock) [tx.calc_sha256() for tx in block.vtx] with mininode_lock: - assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) + assert_equal(peer.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) all_indices = msg.block_txn_request.to_absolute() for index in all_indices: - tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) + tx = peer.last_message["blocktxn"].block_transactions.transactions.pop(0) tx.calc_sha256() assert_equal(tx.sha256, block.vtx[index].sha256) - if version == 1: + if peer.version == 1: # Witnesses should have been stripped assert tx.wit.is_null() else: # Check that the witness matches assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) - test_node.last_message.pop("blocktxn", None) + peer.last_message.pop("blocktxn", None) current_height -= 1 # Next request should send a full block response, as we're past the # allowed depth for a blocktxn response. - block_hash = node.getblockhash(current_height) + block_hash = self.nodes[0].getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) with mininode_lock: - test_node.last_message.pop("block", None) - test_node.last_message.pop("blocktxn", None) - test_node.send_and_ping(msg) + peer.last_message.pop("block", None) + peer.last_message.pop("blocktxn", None) + peer.send_and_ping(msg) with mininode_lock: - test_node.last_message["block"].block.calc_sha256() - assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) - assert "blocktxn" not in test_node.last_message + peer.last_message["block"].block.calc_sha256() + assert_equal(peer.last_message["block"].block.sha256, int(block_hash, 16)) + assert "blocktxn" not in peer.last_message + + def test_compactblocks_not_at_tip(self, peer): + """Test that bitcoind will not send compact blocks for old blocks. - def test_compactblocks_not_at_tip(self, node, test_node): - # Test that requesting old compactblocks doesn't work. + 5 blocks is the max depth for requesting compact blocks.""" MAX_CMPCTBLOCK_DEPTH = 5 new_blocks = [] for i in range(MAX_CMPCTBLOCK_DEPTH + 1): - test_node.clear_block_announcement() - new_blocks.append(node.generate(1)[0]) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) - - test_node.clear_block_announcement() - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) - - test_node.clear_block_announcement() - node.generate(1) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) - test_node.clear_block_announcement() + peer.clear_block_announcement() + new_blocks.append(self.nodes[0].generate(1)[0]) + wait_until(peer.received_block_announcement, timeout=30, lock=mininode_lock) + + peer.clear_block_announcement() + peer.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + wait_until(lambda: "cmpctblock" in peer.last_message, timeout=30, lock=mininode_lock) + + peer.clear_block_announcement() + self.nodes[0].generate(1) + wait_until(peer.received_block_announcement, timeout=30, lock=mininode_lock) + peer.clear_block_announcement() with mininode_lock: - test_node.last_message.pop("block", None) - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) + peer.last_message.pop("block", None) + peer.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + wait_until(lambda: "block" in peer.last_message, timeout=30, lock=mininode_lock) with mininode_lock: - test_node.last_message["block"].block.calc_sha256() - assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) + peer.last_message["block"].block.calc_sha256() + assert_equal(peer.last_message["block"].block.sha256, int(new_blocks[0], 16)) # Generate an old compactblock, and verify that it's not accepted. - cur_height = node.getblockcount() - hashPrevBlock = int(node.getblockhash(cur_height - 5), 16) - block = self.build_block_on_tip(node) + cur_height = self.nodes[0].getblockcount() + hashPrevBlock = int(self.nodes[0].getblockhash(cur_height - 5), 16) + block = self.build_block_on_tip(self.nodes[0]) block.hashPrevBlock = hashPrevBlock + add_witness_commitment(block) block.solve() comp_block = HeaderAndShortIDs() comp_block.initialize_from_block(block) - test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + peer.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - tips = node.getchaintips() - found = False - for x in tips: - if x["hash"] == block.hash: - assert_equal(x["status"], "headers-only") - found = True - break - assert found + # The block header should be stored as a chain tip. + tips = self.nodes[0].getchaintips() + tip = next(x for x in tips if x['hash'] == block.hash) + assert_equal(tip['status'], 'headers-only') # Requesting this block via getblocktxn should silently fail # (to avoid fingerprinting attacks). msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) with mininode_lock: - test_node.last_message.pop("blocktxn", None) - test_node.send_and_ping(msg) + peer.last_message.pop("blocktxn", None) + peer.send_and_ping(msg) with mininode_lock: - assert "blocktxn" not in test_node.last_message - - def activate_segwit(self, node): - node.generate(144 * 3) - assert_equal(get_bip9_status(node, "segwit")["status"], 'active') + assert "blocktxn" not in peer.last_message - def test_end_to_end_block_relay(self, node, listeners): + def test_incorrect_blocktxn_response(self, peer): + """Test that blocks aren't permanently failed if an incorrect response to a getblocktxn is received.""" utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 10) - - [l.clear_block_announcement() for l in listeners] - - # ToHex() won't serialize with witness, but this block has no witnesses - # anyway. TODO: repeat this test with witness tx's to a segwit node. - node.submitblock(ToHex(block)) + block = self.build_block_with_transactions(self.nodes[0], utxo, 10) + # Relay the first 5 transactions from the block in advance + for tx in block.vtx[1:6]: + peer.send_message(msg_tx(tx)) + peer.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:6]: + assert tx.hash in mempool - for l in listeners: - wait_until(lambda: l.received_block_announcement(), timeout=30, lock=mininode_lock) + # Send compact block + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0]) + peer.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + absolute_indexes = [] with mininode_lock: - for l in listeners: - assert "cmpctblock" in l.last_message - l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() - assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) - - # Test that we don't get disconnected if we relay a compact block with valid header, - # but invalid transactions. - def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): - assert len(self.utxos) - utxo = self.utxos[0] - - block = self.build_block_with_transactions(node, utxo, 5) - del block.vtx[3] - block.hashMerkleRoot = block.calc_merkle_root() - if use_segwit: - # If we're testing with segwit, also drop the coinbase witness, - # but include the witness commitment. - add_witness_commitment(block) - block.vtx[0].wit.vtxinwit = [] - block.solve() + assert "getblocktxn" in peer.last_message + absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() + assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) - # Now send the compact block with all transactions prefilled, and - # verify that we don't get disconnected. - comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit) - msg = msg_cmpctblock(comp_block.to_p2p()) - test_node.send_and_ping(msg) + # Now give an incorrect response. + # Note that it's possible for bitcoind to be smart enough to know we're + # lying, since it could check to see if the shortid matches what we're + # sending, and eg disconnect us for misbehavior. If that behavior + # change was made, we could just modify this test by having a + # different peer provide the block further down, so that we're still + # verifying that the block isn't marked bad permanently. This is good + # enough for now. + msg = msg_witness_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) + peer.send_and_ping(msg) - # Check that the tip didn't advance - assert int(node.getbestblockhash(), 16) is not block.sha256 - test_node.sync_with_ping() + # Tip should not have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) - # Helper for enabling cb announcements - # Send the sendcmpct request and sync headers - def request_cb_announcements(self, peer, node, version): - tip = node.getbestblockhash() - peer.get_headers(locator=[int(tip, 16)], hashstop=0) + # We should receive a getdata request + wait_until(lambda: "getdata" in peer.last_message, timeout=10, lock=mininode_lock) + assert_equal(len(peer.last_message["getdata"].inv), 1) + assert_equal(peer.last_message["getdata"].inv[0].type, 2 | MSG_WITNESS_FLAG) + assert_equal(peer.last_message["getdata"].inv[0].hash, block.sha256) - msg = msg_sendcmpct() - msg.version = version - msg.announce = True - peer.send_and_ping(msg) + # Deliver the block + peer.send_and_ping(msg_witness_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) - def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): - assert len(self.utxos) + def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): + """Test that a compact block can be reconstructed from multiple peers.""" def announce_cmpct_block(node, peer): utxo = self.utxos.pop(0) @@ -757,166 +715,101 @@ def announce_cmpct_block(node, peer): assert "getblocktxn" in peer.last_message return block, cmpct_block - block, cmpct_block = announce_cmpct_block(node, stalling_peer) + block, cmpct_block = announce_cmpct_block(self.nodes[0], stalling_peer) + # stalling peer doesn't send the block txs. Instead, delivery peer sends + # them as txs. for tx in block.vtx[1:]: - delivery_peer.send_message(msg_tx(tx)) + delivery_peer.send_message(msg_witness_tx(tx)) delivery_peer.sync_with_ping() - mempool = node.getrawmempool() + mempool = self.nodes[0].getrawmempool() for tx in block.vtx[1:]: assert tx.hash in mempool + # node can construct the block after delivery peer sends a compact block. delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) # Now test that delivering an invalid compact block won't break relay - - block, cmpct_block = announce_cmpct_block(node, stalling_peer) + block, cmpct_block = announce_cmpct_block(self.nodes[0], stalling_peer) for tx in block.vtx[1:]: - delivery_peer.send_message(msg_tx(tx)) + delivery_peer.send_message(msg_witness_tx(tx)) delivery_peer.sync_with_ping() + # Malleate the coinbase witness before sending it to the node in a compact + # block from delivery node. cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [CTxInWitness()] - cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] - - cmpct_block.use_witness = True + cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(1)] delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert int(node.getbestblockhash(), 16) != block.sha256 + assert int(self.nodes[0].getbestblockhash(), 16) != block.sha256 + # Send the coinbase to the node in a blocktxn message from the stalling peer. + # The node should still be able to reconstruct the block. msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 msg.block_transactions.transactions = block.vtx[1:] stalling_peer.send_and_ping(msg) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - def run_test(self): - # Setup the p2p connections - self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) - self.segwit_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) - self.old_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) - - # We will need UTXOs to construct transactions in later tests. - self.make_utxos() - - self.log.info("Running tests, pre-segwit activation:") + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) - self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) + def test_end_to_end_block_relay(self, peers): + """Test that if we submitblock to the node, we'll get a compact block announcement to all peers.""" + for peer in peers: + self.request_cb_announcements(peer) + utxo = self.utxos.pop(0) - self.log.info("Testing compactblock requests... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) + block = self.build_block_with_transactions(self.nodes[0], utxo, 10) - self.log.info("Testing getblocktxn requests...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) + [l.clear_block_announcement() for l in peers] - self.log.info("Testing getblocktxn handler...") - self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - sync_blocks(self.nodes) + self.nodes[0].submitblock(block.serialize(with_witness=True).hex()) - self.log.info("Testing compactblock requests/announcements not at chain tip...") - self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) - sync_blocks(self.nodes) - self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) - self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) - sync_blocks(self.nodes) + for l in peers: + wait_until(lambda: l.received_block_announcement(), timeout=30, lock=mininode_lock) + with mininode_lock: + for l in peers: + assert "cmpctblock" in l.last_message + l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() + assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) - self.log.info("Testing handling of incorrect blocktxn responses...") - self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) + def test_invalid_tx_in_compactblock(self, peer): + """Test that we don't get disconnected if we relay a compact block with valid header and invalid transactions.""" + utxo = self.utxos[0] - # End-to-end block relay tests - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + block = self.build_block_with_transactions(self.nodes[0], utxo, 5) - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) + # Delete an intermediate transaction + del block.vtx[3] + block.hashMerkleRoot = block.calc_merkle_root() - self.log.info("Testing reconstructing compact blocks from all peers...") - self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) - sync_blocks(self.nodes) + # Also drop the coinbase witness but include the witness commitment. + add_witness_commitment(block) + block.vtx[0].wit.vtxinwit = [] + block.solve() - # Advance to segwit activation - self.log.info("Advancing to segwit activation") - self.activate_segwit(self.nodes[1]) - self.log.info("Running tests, post-segwit activation...") + # Now send the compact block with all transactions prefilled, and + # verify that we don't get disconnected. + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4]) + msg = msg_cmpctblock(comp_block.to_p2p()) + peer.send_and_ping(msg) - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests (unupgraded node)... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) - - self.log.info("Testing getblocktxn requests (unupgraded node)...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - - # Need to manually sync node0 and node1, because post-segwit activation, - # node1 will not download blocks from node0. - self.log.info("Syncing nodes...") - assert self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash() - while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): - block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount() + 1) - self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) - assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) - - self.log.info("Testing compactblock requests (segwit node)... ") - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) - - self.log.info("Testing getblocktxn requests (segwit node)...") - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - - # Test that if we submitblock to node1, we'll get a compact block - # announcement to all peers. - # (Post-segwit activation, blocks won't propagate from node0 to node1 - # automatically, so don't bother testing a block announced to node0.) - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + # Check that the tip didn't advance + assert int(self.nodes[0].getbestblockhash(), 16) != block.sha256 + peer.sync_with_ping() - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) + def test_invalid_cmpctblock_message(self, peer): + """Test that the node disconnects a peer that sends an invalid compact block.""" + block = self.build_block_on_tip(self.nodes[0]) - self.log.info("Testing invalid index in cmpctblock message...") - self.test_invalid_cmpctblock_message() + cmpct_block = P2PHeaderAndShortIDs() + cmpct_block.header = CBlockHeader(block) + cmpct_block.prefilled_txn_length = 1 + # This index is too high + prefilled_txn = PrefilledTransaction(1, block.vtx[0]) + cmpct_block.prefilled_txn = [prefilled_txn] + peer.send_await_disconnect(msg_cmpctblock(cmpct_block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) if __name__ == '__main__': CompactBlocksTest().main() diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 7cf51d9223ac..c8a9edf2b271 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -759,14 +759,13 @@ def calculate_shortid(k0, k1, tx_hash): # This version gets rid of the array lengths, and reinterprets the differential # encoding into indices that can be used for lookup. class HeaderAndShortIDs: - __slots__ = ("header", "nonce", "prefilled_txn", "shortids", "use_witness") + __slots__ = ("header", "nonce", "prefilled_txn", "shortids") def __init__(self, p2pheaders_and_shortids = None): self.header = CBlockHeader() self.nonce = 0 self.shortids = [] self.prefilled_txn = [] - self.use_witness = False if p2pheaders_and_shortids is not None: self.header = p2pheaders_and_shortids.header @@ -778,10 +777,7 @@ def __init__(self, p2pheaders_and_shortids = None): last_index = self.prefilled_txn[-1].index def to_p2p(self): - if self.use_witness: - ret = P2PHeaderAndShortWitnessIDs() - else: - ret = P2PHeaderAndShortIDs() + ret = P2PHeaderAndShortWitnessIDs() ret.header = self.header ret.nonce = self.nonce ret.shortids_length = len(self.shortids) @@ -803,18 +799,15 @@ def get_siphash_keys(self): return [ key0, key1 ] # Version 2 compact blocks use wtxid in shortids (rather than txid) - def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False): + def initialize_from_block(self, block, nonce=0, prefill_list = [0]): self.header = CBlockHeader(block) self.nonce = nonce self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] self.shortids = [] - self.use_witness = use_witness [k0, k1] = self.get_siphash_keys() for i in range(len(block.vtx)): if i not in prefill_list: - tx_hash = block.vtx[i].sha256 - if use_witness: - tx_hash = block.vtx[i].calc_sha256(with_witness=True) + tx_hash = block.vtx[i].calc_sha256(with_witness=True) self.shortids.append(calculate_shortid(k0, k1, tx_hash)) def __repr__(self):