From 4c57ad1c050ca27c0ff356afbb7c26a9a3d52141 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Fri, 13 Sep 2024 12:58:27 +0700 Subject: [PATCH 1/7] chore: increase batch size from 10 to 50 for faster block generation in functional tests It significantly improve speed of forks activation because reduces overhead for block generations Bigger batch size can cause time-outs for RPC for tsan job (time-out is 30 seconds) --- test/functional/feature_asset_locks.py | 16 ++++++++-------- test/functional/test_framework/test_framework.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/functional/feature_asset_locks.py b/test/functional/feature_asset_locks.py index 23b5d03e1fa1..de207dbb9402 100755 --- a/test/functional/feature_asset_locks.py +++ b/test/functional/feature_asset_locks.py @@ -219,11 +219,11 @@ def send_tx(self, tx, expected_error = None, reason = None): except JSONRPCException as e: assert expected_error in e.error['message'] - def slowly_generate_batch(self, count): - self.log.info(f"Slowly generate {count} blocks") + def generate_batch(self, count): + self.log.info(f"Generate {count} blocks") while count > 0: self.log.info(f"Generating batch of blocks {count} left") - batch = min(10, count) + batch = min(50, count) count -= batch self.bump_mocktime(batch) self.nodes[1].generate(batch) @@ -426,7 +426,7 @@ def test_asset_unlocks(self, node_wallet, node, pubkey): self.validate_credit_pool_balance(locked - 2 * COIN) self.log.info("Generating many blocks to make quorum far behind (even still active)...") - self.slowly_generate_batch(too_late_height - node.getblockcount() - 1) + self.generate_batch(too_late_height - node.getblockcount() - 1) self.check_mempool_result(tx=asset_unlock_tx_too_late, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}}) node.generate(1) self.sync_all() @@ -444,7 +444,7 @@ def test_asset_unlocks(self, node_wallet, node, pubkey): for inode in self.nodes: inode.invalidateblock(block_asset_unlock) self.validate_credit_pool_balance(locked) - self.slowly_generate_batch(50) + self.generate_batch(50) self.validate_credit_pool_balance(locked) for inode in self.nodes: inode.reconsiderblock(block_to_reconsider) @@ -510,7 +510,7 @@ def test_withdrawal_limits(self, node_wallet, node, pubkey): assert spend_txid_in_block in block['tx'] self.log.info("Fast forward to the next day to reset all current unlock limits...") - self.slowly_generate_batch(blocks_in_one_day) + self.generate_batch(blocks_in_one_day) self.mine_quorum_2_nodes(llmq_type_name='llmq_test_platform', llmq_type=106) total = self.get_credit_pool_balance() @@ -585,7 +585,7 @@ def test_withdrawal_limits(self, node_wallet, node, pubkey): assert pending_txid in node.getrawmempool() self.log.info("Fast forward to next day again...") - self.slowly_generate_batch(blocks_in_one_day - 1) + self.generate_batch(blocks_in_one_day - 1) self.log.info("Checking mempool is empty now...") self.mempool_size = 0 self.check_mempool_size() @@ -618,7 +618,7 @@ def test_withdrawal_limits(self, node_wallet, node, pubkey): self.log.info("generate many blocks to be sure that mempool is empty after expiring txes...") - self.slowly_generate_batch(60) + self.generate_batch(60) self.log.info("Checking that credit pool is not changed...") assert_equal(new_total, self.get_credit_pool_balance()) self.check_mempool_size() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index ccbc4596672e..758cac21eb9f 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1136,7 +1136,7 @@ def activate_by_name(self, name, expected_activation_height=None): self.wait_for_sporks_same() # mine blocks in batches - batch_size = 10 + batch_size = 50 if expected_activation_height is not None: height = self.nodes[0].getblockcount() assert height < expected_activation_height From 132d95e651e5737831461e066c4e4f46e8a56c26 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 12 Sep 2024 16:46:30 +0700 Subject: [PATCH 2/7] perf: remove sleep(1) from each step of quorum creation in functional tests --- test/functional/test_framework/test_framework.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 758cac21eb9f..b24acff69826 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1794,7 +1794,6 @@ def wait_func(): wait_until_helper(wait_func, timeout=timeout, sleep=sleep) def move_blocks(self, nodes, num_blocks): - time.sleep(1) self.bump_mocktime(1, nodes=nodes) self.nodes[0].generate(num_blocks) self.sync_blocks(nodes) From 3f17a01a83bc7928d746209741f806c8a66f9574 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Fri, 13 Sep 2024 20:15:01 +0700 Subject: [PATCH 3/7] fix: bump mocktime in simplepose when generating blocks to improve robustness --- test/functional/feature_llmq_simplepose.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/functional/feature_llmq_simplepose.py b/test/functional/feature_llmq_simplepose.py index e556c538f413..c93f503b0f30 100755 --- a/test/functional/feature_llmq_simplepose.py +++ b/test/functional/feature_llmq_simplepose.py @@ -101,7 +101,7 @@ def mine_quorum_no_check(self, expected_good_nodes, mninfos_online): # move forward to next DKG skip_count = 24 - (self.nodes[0].getblockcount() % 24) if skip_count != 0: - self.bump_mocktime(1, nodes=nodes) + self.bump_mocktime(skip_count, nodes=nodes) self.nodes[0].generate(skip_count) self.sync_blocks(nodes) @@ -147,6 +147,7 @@ def mine_quorum_no_check(self, expected_good_nodes, mninfos_online): quorum_info = self.nodes[0].quorum("info", 100, new_quorum) # Mine 8 (SIGN_HEIGHT_OFFSET) more blocks to make sure that the new quorum gets eligible for signing sessions + self.bump_mocktime(8) self.nodes[0].generate(8) self.sync_blocks(nodes) self.log.info("New quorum: height=%d, quorumHash=%s, quorumIndex=%d, minedBlock=%s" % (quorum_info["height"], new_quorum, quorum_info["quorumIndex"], quorum_info["minedBlock"])) From cd1958c82a20ba6219fd8748c16dfc586ba785c7 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 12 Sep 2024 17:43:21 +0700 Subject: [PATCH 4/7] perf: removed sleep(6) from mine_cycle_quorum in functional tests --- test/functional/test_framework/test_framework.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index b24acff69826..6811c7948a48 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1995,14 +1995,12 @@ def mine_cycle_quorum(self, llmq_type_name="llmq_test_dip0024", llmq_type=103, self.log.info("quorumIndex 1: Waiting for phase 6 (finalization)") self.wait_for_quorum_phase(q_1, 6, expected_members, None, 0, mninfos_online, llmq_type_name) - time.sleep(6) self.log.info("Mining final commitments") self.bump_mocktime(1, nodes=nodes) self.nodes[0].getblocktemplate() # this calls CreateNewBlock self.nodes[0].generate(1) self.sync_blocks(nodes) - time.sleep(6) self.log.info("Waiting for quorum(s) to appear in the list") self.wait_for_quorums_list(q_0, q_1, nodes, llmq_type_name) From fe49f3f17881bb9e6e2f223de52688843f621e87 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 12 Sep 2024 17:42:36 +0700 Subject: [PATCH 5/7] refactor: removed dead and commented code from test_framework.py --- test/functional/test_framework/test_framework.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 6811c7948a48..2d30b6fd741a 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1170,9 +1170,6 @@ def activate_by_name(self, name, expected_activation_height=None): self.nodes[0].sporkupdate("SPORK_17_QUORUM_DKG_ENABLED", spork17_value) self.wait_for_sporks_same() - def activate_dip0024(self, expected_activation_height=None): - self.activate_by_name('dip0024', expected_activation_height) - def activate_v19(self, expected_activation_height=None): self.activate_by_name('v19', expected_activation_height) @@ -1912,19 +1909,11 @@ def mine_cycle_quorum(self, llmq_type_name="llmq_test_dip0024", llmq_type=103, # move forward to next DKG skip_count = 24 - (self.nodes[0].getblockcount() % 24) - # if skip_count != 0: - # self.bump_mocktime(1, nodes=nodes) - # self.nodes[0].generate(skip_count) - # time.sleep(4) - # self.sync_blocks(nodes) - self.move_blocks(nodes, skip_count) q_0 = self.nodes[0].getbestblockhash() self.log.info("Expected quorum_0 at:" + str(self.nodes[0].getblockcount())) - # time.sleep(4) self.log.info("Expected quorum_0 hash:" + str(q_0)) - # time.sleep(4) self.log.info("quorumIndex 0: Waiting for phase 1 (init)") self.wait_for_quorum_phase(q_0, 1, expected_members, None, 0, mninfos_online, llmq_type_name) self.log.info("quorumIndex 0: Waiting for quorum connections (init)") @@ -1936,9 +1925,7 @@ def mine_cycle_quorum(self, llmq_type_name="llmq_test_dip0024", llmq_type=103, q_1 = self.nodes[0].getbestblockhash() self.log.info("Expected quorum_1 at:" + str(self.nodes[0].getblockcount())) - # time.sleep(2) self.log.info("Expected quorum_1 hash:" + str(q_1)) - # time.sleep(2) self.log.info("quorumIndex 1: Waiting for phase 1 (init)") self.wait_for_quorum_phase(q_1, 1, expected_members, None, 0, mninfos_online, llmq_type_name) self.log.info("quorumIndex 1: Waiting for quorum connections (init)") @@ -2035,7 +2022,6 @@ def move_to_next_cycle(self): self.bump_mocktime(1, nodes=nodes) self.nodes[0].generate(skip_count) self.sync_blocks(nodes) - time.sleep(1) self.log.info('Moved from block %d to %d' % (cur_block, self.nodes[0].getblockcount())) def wait_for_recovered_sig(self, rec_sig_id, rec_sig_msg_hash, llmq_type=100, timeout=10): From 4f636f47b419877c5e09a3e6485eea451510733b Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 8 Aug 2024 11:42:21 +0700 Subject: [PATCH 6/7] fix: re-order functional tests: move governance to 60+seconds category --- test/functional/test_runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6cd236849606..1d85da781f6b 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -133,6 +133,10 @@ 'feature_dip4_coinbasemerkleroots.py', # NOTE: needs dash_hash to pass 'feature_asset_locks.py', # NOTE: needs dash_hash to pass 'feature_mnehf.py', # NOTE: needs dash_hash to pass + 'feature_governance.py --legacy-wallet', + 'feature_governance.py --descriptors', + 'feature_governance_cl.py --legacy-wallet', + 'feature_governance_cl.py --descriptors', # vv Tests less than 60s vv 'p2p_sendheaders.py', # NOTE: needs dash_hash to pass 'p2p_sendheaders_compressed.py', # NOTE: needs dash_hash to pass @@ -293,10 +297,6 @@ 'feature_cltv.py', 'feature_new_quorum_type_activation.py', 'feature_governance_objects.py', - 'feature_governance.py --legacy-wallet', - 'feature_governance.py --descriptors', - 'feature_governance_cl.py --legacy-wallet', - 'feature_governance_cl.py --descriptors', 'p2p_governance_invs.py', 'rpc_uptime.py', 'feature_discover.py', From 874ef8cda231eebde408263874a28e465de50bb6 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Tue, 17 Sep 2024 17:46:12 +0300 Subject: [PATCH 7/7] fix: mine_quorum_no_checks -> mine_quorum_less_checks: do some checks to make sure quorums are mined correctly --- test/functional/feature_llmq_simplepose.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/functional/feature_llmq_simplepose.py b/test/functional/feature_llmq_simplepose.py index c93f503b0f30..6ca819968adf 100755 --- a/test/functional/feature_llmq_simplepose.py +++ b/test/functional/feature_llmq_simplepose.py @@ -92,10 +92,10 @@ def test_no_banning(self, expected_connections=None): for mn in self.mninfo: assert not self.check_punished(mn) and not self.check_banned(mn) - def mine_quorum_no_check(self, expected_good_nodes, mninfos_online): + def mine_quorum_less_checks(self, expected_good_nodes, mninfos_online): # Unlike in mine_quorum we skip most of the checks and only care about - # nodes moving forward from phase to phase and the fact that the quorum is actually mined. - self.log.info("Mining a quorum with no checks") + # nodes moving forward from phase to phase correctly and the fact that the quorum is actually mined. + self.log.info("Mining a quorum with less checks") nodes = [self.nodes[0]] + [mn.node for mn in mninfos_online] # move forward to next DKG @@ -112,7 +112,7 @@ def mine_quorum_no_check(self, expected_good_nodes, mninfos_online): self.move_blocks(nodes, 2) self.log.info("Waiting for phase 2 (contribute)") - self.wait_for_quorum_phase(q, 2, expected_good_nodes, None, 0, mninfos_online) + self.wait_for_quorum_phase(q, 2, expected_good_nodes, "receivedContributions", expected_good_nodes, mninfos_online) self.move_blocks(nodes, 2) self.log.info("Waiting for phase 3 (complain)") @@ -124,7 +124,7 @@ def mine_quorum_no_check(self, expected_good_nodes, mninfos_online): self.move_blocks(nodes, 2) self.log.info("Waiting for phase 5 (commit)") - self.wait_for_quorum_phase(q, 5, expected_good_nodes, None, 0, mninfos_online) + self.wait_for_quorum_phase(q, 5, expected_good_nodes, "receivedPrematureCommitments", expected_good_nodes, mninfos_online) self.move_blocks(nodes, 2) self.log.info("Waiting for phase 6 (mining)") @@ -175,7 +175,7 @@ def test_banning(self, invalidate_proc, expected_connections): # 6th time is when it should be banned for sure. for _ in range(6): self.reset_probe_timeouts() - self.mine_quorum_no_check(expected_contributors - 1, mninfos_online) + self.mine_quorum_less_checks(expected_contributors - 1, mninfos_online) assert self.check_banned(mn)