From 267abb8b8460c0cf0b4bd53e1538544edd005b87 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Thu, 12 Feb 2026 02:30:53 +0100 Subject: [PATCH] feat: improve generation script, add wallet export, add tests - Refactor `generate.py` into a base `Generator` class with one strategy: `WalletSyncGenerator` (SPV wallet sync edge cases). - Extract shared wallet stats collection into `generator/wallet_export.py`. - Add `export_wallets.py` for exporting stats from existing blockchain data. - Improve dashd_manager extra_args support and rpc_client error handling. - Add test suite with unit and integration tests to make sure the script is working. --- .gitignore | 5 + contrib/setup-dashd.py | 104 ++++ export_wallets.py | 175 ++++++ generate.py | 1182 ++++++++++++++++++------------------ generator/dashd_manager.py | 166 +++-- generator/errors.py | 12 +- generator/rpc_client.py | 48 +- generator/wallet_export.py | 83 +++ tests/__init__.py | 0 tests/test_generate.py | 169 ++++++ tests/test_integration.py | 192 ++++++ 11 files changed, 1435 insertions(+), 701 deletions(-) create mode 100644 contrib/setup-dashd.py create mode 100644 export_wallets.py create mode 100644 generator/wallet_export.py create mode 100644 tests/__init__.py create mode 100644 tests/test_generate.py create mode 100644 tests/test_integration.py diff --git a/.gitignore b/.gitignore index 69dea49..a80d296 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,11 @@ # IDE .idea/ +# Python +__pycache__/ +.venv/ +.pytest_cache/ + # Data (distributed via releases) data/ diff --git a/contrib/setup-dashd.py b/contrib/setup-dashd.py new file mode 100644 index 0000000..5966be6 --- /dev/null +++ b/contrib/setup-dashd.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Cross-platform setup script for downloading dashd binaries. + +Downloads the Dash Core binary for integration tests. +Outputs DASHD_PATH line suitable for appending to GITHUB_ENV +or evaluating in a shell. + +Environment variables: + DASHVERSION - Dash Core version (default: 23.0.2) + CACHE_DIR - Cache directory (default: ~/.regtest-blockchain-test) +""" + +import os +import platform +import sys +import tarfile +import urllib.request +import zipfile + +DASHVERSION = os.environ.get("DASHVERSION", "23.0.2") + + +def get_cache_dir(): + if "CACHE_DIR" in os.environ: + return os.environ["CACHE_DIR"] + home = os.environ.get("HOME") or os.environ.get("USERPROFILE") + if not home: + sys.exit("Cannot determine home directory: neither HOME nor USERPROFILE is set") + return os.path.join(home, ".regtest-blockchain-test") + + +def get_asset_filename(): + """Return the asset filename for the current platform.""" + system = platform.system() + machine = platform.machine() + + if system == "Linux": + arch = "aarch64" if machine in ("aarch64", "arm64") else "x86_64" + return f"dashcore-{DASHVERSION}-{arch}-linux-gnu.tar.gz" + elif system == "Darwin": + arch = "arm64" if machine == "arm64" else "x86_64" + return f"dashcore-{DASHVERSION}-{arch}-apple-darwin.tar.gz" + elif system == "Windows": + return f"dashcore-{DASHVERSION}-win64.zip" + else: + sys.exit(f"Unsupported platform: {system}") + + +def log(msg): + print(msg, file=sys.stderr) + + +def download(url, dest): + log(f"Downloading {url} ...") + urllib.request.urlretrieve(url, dest) + + +def extract(archive_path, dest_dir): + if archive_path.endswith(".zip"): + with zipfile.ZipFile(archive_path, "r") as zf: + zf.extractall(dest_dir) + else: + with tarfile.open(archive_path, "r:gz") as tf: + tf.extractall(dest_dir) + + +def setup_dashd(cache_dir): + """Download and extract dashd binary. Returns the path to the dashd binary.""" + asset = get_asset_filename() + dashd_dir = os.path.join(cache_dir, f"dashcore-{DASHVERSION}") + + ext = ".exe" if platform.system() == "Windows" else "" + dashd_bin = os.path.join(dashd_dir, "bin", f"dashd{ext}") + + if os.path.isfile(dashd_bin): + log(f"dashd {DASHVERSION} already available at {dashd_bin}") + return dashd_bin + + log(f"Downloading dashd {DASHVERSION}...") + archive_path = os.path.join(cache_dir, asset) + url = f"https://github.com/dashpay/dash/releases/download/v{DASHVERSION}/{asset}" + download(url, archive_path) + extract(archive_path, cache_dir) + os.remove(archive_path) + log(f"Extracted dashd to {dashd_dir}") + + if not os.path.isfile(dashd_bin): + sys.exit(f"Expected binary not found after extraction: {dashd_bin}") + + return dashd_bin + + +def main(): + cache_dir = get_cache_dir() + os.makedirs(cache_dir, exist_ok=True) + + dashd_path = setup_dashd(cache_dir) + + # Output for GITHUB_ENV or shell eval + print(f"DASHD_PATH={dashd_path}") + + +if __name__ == "__main__": + main() diff --git a/export_wallets.py b/export_wallets.py new file mode 100644 index 0000000..501d008 --- /dev/null +++ b/export_wallets.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Export wallet statistics from existing blockchain data.""" + +import argparse +import signal +import subprocess +import sys +import time +from pathlib import Path + +# Add generator module to path +sys.path.insert(0, str(Path(__file__).parent)) + +from generator.dashd_manager import dashd_preexec_fn +from generator.errors import RPCError +from generator.rpc_client import DashRPCClient +from generator.wallet_export import collect_wallet_stats, save_wallet_file + + +def find_free_port(start=19998): + import socket + + for port in range(start, start + 20): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", port)) + return port + except OSError: + continue + raise RuntimeError("No free port found") + + +def main(): + parser = argparse.ArgumentParser(description="Re-export wallet statistics from existing blockchain data") + parser.add_argument("datadir", type=str, help="Path to dashd data directory (contains network subdirectory)") + parser.add_argument("--dashd-path", type=str, help="Path to dashd executable (default: dashd in PATH)") + parser.add_argument( + "--network", + type=str, + default="regtest", + choices=["regtest", "testnet", "mainnet"], + help="Dash network (default: regtest)", + ) + args = parser.parse_args() + + datadir = Path(args.datadir) + if not datadir.exists(): + print(f"Directory not found: {datadir}") + sys.exit(1) + + # dashd stores chain data in a network-named subdirectory + network_subdirs = {"regtest": "regtest", "testnet": "testnet3", "mainnet": ""} + network_subdir_name = network_subdirs[args.network] + network_subdir = datadir / network_subdir_name if network_subdir_name else datadir + if network_subdir_name and not network_subdir.exists(): + print(f"No {network_subdir_name}/ subdirectory found in {datadir}") + sys.exit(1) + + wallets_dir = datadir / "wallets" + wallets_dir.mkdir(exist_ok=True) + + # Determine dashd and dash-cli paths + if args.dashd_path: + dashd_executable = args.dashd_path + dashcli_path = str(Path(args.dashd_path).parent / "dash-cli") + else: + dashd_executable = "dashd" + dashcli_path = "dash-cli" + + # Find free ports + rpc_port = find_free_port(19998) + p2p_port = find_free_port(rpc_port + 1) + + print(f"Starting dashd ({args.network}) on RPC port {rpc_port}...") + + cmd = [ + dashd_executable, + f"-{args.network}" if args.network != "mainnet" else "", + f"-datadir={datadir}", + f"-port={p2p_port}", + f"-rpcport={rpc_port}", + "-server=1", + "-daemon=0", + "-rpcbind=127.0.0.1", + "-rpcallowip=127.0.0.1", + "-listen=0", + ] + # Remove empty strings from cmd (mainnet needs no network flag) + cmd = [c for c in cmd if c] + + try: + proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, preexec_fn=dashd_preexec_fn) + except FileNotFoundError: + print(f"dashd executable not found: {dashd_executable}") + sys.exit(1) + except OSError as e: + print(f"Failed to start dashd ({dashd_executable}): {e}") + sys.exit(1) + + def cleanup(exit_code=0): + print("\nStopping dashd...") + try: + proc.terminate() + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + sys.exit(exit_code) + + signal.signal(signal.SIGINT, lambda sig, frame: cleanup(1)) + + rpc = DashRPCClient(dashcli_path=dashcli_path, datadir=str(datadir), network=args.network, rpc_port=rpc_port) + + print("Waiting for dashd to start...") + for _ in range(30): + try: + height = rpc.call("getblockcount") + print(f"Connected! Block height: {height}") + break + except Exception: + time.sleep(1) + else: + print("Failed to connect to dashd") + cleanup(1) + + # Discover and load all wallets from the datadir + wallet_names = [] + try: + wallet_dir_info = rpc.call("listwalletdir") + wallet_names = [w["name"] for w in wallet_dir_info.get("wallets", [])] + except RPCError: + # Fallback: scan filesystem for wallet directories + wallets_path = network_subdir / "wallets" + if wallets_path.exists(): + wallet_names = [d.name for d in wallets_path.iterdir() if d.is_dir()] + + if not wallet_names: + print("No wallets found in datadir") + cleanup(1) + + print(f"Found {len(wallet_names)} wallet(s): {', '.join(wallet_names)}") + + for name in wallet_names: + try: + rpc.call("loadwallet", name) + print(f" Loaded wallet: {name}") + except RPCError as e: + if "already loaded" in str(e).lower(): + print(f" Wallet already loaded: {name}") + else: + print(f" Warning: Could not load {name}: {e}") + + # Collect and export stats for each wallet + print("\nCollecting wallet statistics...") + + for wallet_name in wallet_names: + print(f" Processing {wallet_name}...") + stats = collect_wallet_stats(rpc, wallet_name) + + unique_txs = len({tx["txid"] for tx in stats["transactions"]}) + print( + f" {len(stats['transactions'])} entries, {unique_txs} unique txs, " + f"{len(stats['utxos'])} UTXOs, balance: {stats['balance']:.8f} DASH" + ) + + wallet_file = wallets_dir / f"{wallet_name}.json" + save_wallet_file(stats, wallet_file) + print(f" Saved to {wallet_file}") + + print("\nDone! Stopping dashd...") + cleanup() + + +if __name__ == "__main__": + main() diff --git a/generate.py b/generate.py index ca67b30..523a1de 100755 --- a/generate.py +++ b/generate.py @@ -2,87 +2,84 @@ """ Dash Regtest Test Data Generator -Efficient generation of comprehensive blockchain test data with: +Generates blockchain test data optimized for SPV wallet sync testing with: - Automatic dashd startup in temporary directory -- Proactive UTXO management (prevents "Insufficient funds") -- Diverse transaction types (edge case coverage) +- Targeted transactions exercising SPV sync edge cases - Robust error handling - Portable operation from any directory Usage: - python3 generate.py --blocks 100 - python3 generate.py --blocks 1000 --dashd-path /path/to/dashd - python3 generate.py --blocks 500 --keep-temp + python3 generate.py --blocks 40000 + python3 generate.py --blocks 200 --dashd-path /path/to/dashd + python3 generate.py --blocks 1000 --output-dir /tmp/output """ -import sys -import os -import json -import subprocess +import datetime import random -import struct import shutil -from pathlib import Path -from typing import List, Dict, Any, Optional -from dataclasses import dataclass +import sys import time -import datetime - +from dataclasses import dataclass, field +from pathlib import Path # Add generator module to path sys.path.insert(0, str(Path(__file__).parent)) -from generator.errors import * -from generator.rpc_client import DashRPCClient from generator.dashd_manager import DashdManager +from generator.errors import ( + ConfigError, + DashdConnectionError, + GeneratorError, + InsufficientFundsError, + RPCError, +) +from generator.rpc_client import DashRPCClient +from generator.wallet_export import collect_wallet_stats, save_wallet_file @dataclass class Config: """Configuration for test data generation""" + target_blocks: int - batch_size: int - min_utxo_threshold: int - target_utxo_count: int dashcli_path: str dashd_executable: str auto_start_dashd: bool - dashd_datadir: Optional[str] + dashd_datadir: str | None dashd_wallet: str - rpc_port: Optional[int] + rpc_port: int | None output_base: str - # Transaction generation parameters - tx_probability_none: float = 0.30 # Probability of 0 transactions per block - tx_probability_low: float = 0.70 # Cumulative probability for 1-3 transactions - tx_probability_medium: float = 0.90 # Cumulative probability for 4-10 transactions - # else: 11-25 transactions (high) + # Extra dashd args (block filter index for SPV testing) + extra_dashd_args: list[str] = field(default_factory=list) class Generator: - """Main test data generator with integrated UTXO management""" + """Base generator with shared infrastructure. + + Subclasses must implement: + - _load_addresses(): wallet setup + - _initialize_utxo_pool(): funding + - _generate_blocks(): block generation logic + """ def __init__(self, config: Config, keep_temp: bool = False): self.config = config self.keep_temp = keep_temp - self.dashd_manager: Optional[DashdManager] = None - self.rpc: Optional[DashRPCClient] = None - self.wallets = [] # List of wallet dictionaries with name, mnemonic, addresses, tier - self.all_addresses = [] # Flat list of all addresses for easy access + self.dashd_manager: DashdManager | None = None + self.rpc: DashRPCClient | None = None + self.wallets = [] # List of wallet dictionaries with name, mnemonic, addresses self.utxo_count = 0 - self.output_dir: Optional[Path] = None - self.stats = { - 'blocks_generated': 0, - 'transactions_created': 0, - 'utxo_replenishments': 0 - } + self.output_dir: Path | None = None + self.mining_address: str | None = None # Faucet address for mining rewards + self.stats = {"blocks_generated": 0, "transactions_created": 0, "coinbase_rewards": 0, "utxo_replenishments": 0} def generate(self): """Main generation workflow""" print("=" * 60) print("Dash Regtest Test Data Generator") print("=" * 60) + print(f"Strategy: {self.strategy_name()}") print(f"Target blocks: {self.config.target_blocks}") - print(f"UTXO threshold: {self.config.min_utxo_threshold}") print() generation_start_time = time.time() @@ -100,16 +97,17 @@ def generate(self): duration_str = str(datetime.timedelta(seconds=int(generation_duration))) print("\n" + "=" * 60) - print("✓ Generation complete!") + print("Generation complete!") print("=" * 60) print(f"Blocks: {self.stats['blocks_generated']}") print(f"Transactions: {self.stats['transactions_created']}") + print(f"Coinbase rewards: {self.stats['coinbase_rewards']}") print(f"UTXO replenishments: {self.stats['utxo_replenishments']}") print(f"Total duration: {duration_str}") except KeyboardInterrupt: print("\n\nGeneration interrupted by user") - raise GeneratorError("User interrupted") + raise GeneratorError("User interrupted") from None finally: # Cleanup if not already done if self.dashd_manager: @@ -119,9 +117,13 @@ def generate(self): # Process stopped but temp dir not cleaned up yet try: shutil.rmtree(self.dashd_manager.temp_dir, ignore_errors=True) - except: + except OSError: pass + def strategy_name(self) -> str: + """Return the name of this strategy""" + return "base" + def _ensure_dashd_running(self): """Start dashd if auto_start is enabled""" if not self.config.auto_start_dashd: @@ -133,7 +135,8 @@ def _ensure_dashd_running(self): self.dashd_manager = DashdManager( dashd_executable=self.config.dashd_executable, - rpc_port=self.config.rpc_port + rpc_port=self.config.rpc_port, + extra_args=self.config.extra_dashd_args, ) rpc_port, temp_dir = self.dashd_manager.start(keep_temp=self.keep_temp) @@ -148,482 +151,94 @@ def _ensure_dashd_running(self): def _initialize_rpc_client(self): """Initialize RPC client with appropriate settings""" self.rpc = DashRPCClient( - dashcli_path=self.config.dashcli_path, - datadir=self.config.dashd_datadir, - rpc_port=self.config.rpc_port + dashcli_path=self.config.dashcli_path, datadir=self.config.dashd_datadir, rpc_port=self.config.rpc_port ) def _verify_dashd(self): """Verify dashd is running and responsive, create wallet if needed""" print("Verifying dashd connection...") try: - block_count = self.rpc.call('getblockcount') - print(f"✓ Connected to dashd (current height: {block_count})") + block_count = self.rpc.call("getblockcount") + print(f" Connected to dashd (current height: {block_count})") except DashdConnectionError as e: - print(f"✗ Cannot connect to dashd: {e}") + print(f" Cannot connect to dashd: {e}") print("\nPlease ensure dashd is running in regtest mode:") - print(f" dashd -regtest -daemon") + print(" dashd -regtest -daemon") raise # Ensure wallet exists and is loaded try: # Try to load the wallet if it exists but isn't loaded - self.rpc.call('loadwallet', self.config.dashd_wallet) - print(f"✓ Loaded wallet: {self.config.dashd_wallet}") + self.rpc.call("loadwallet", self.config.dashd_wallet) + print(f" Loaded wallet: {self.config.dashd_wallet}") except RPCError as e: error_msg = str(e).lower() if "already loaded" in error_msg: - print(f"✓ Wallet already loaded: {self.config.dashd_wallet}") + print(f" Wallet already loaded: {self.config.dashd_wallet}") elif "not found" in error_msg or "does not exist" in error_msg: # Wallet doesn't exist, create it - print(f"Creating new wallet: {self.config.dashd_wallet}") - self.rpc.call('createwallet', self.config.dashd_wallet) - print(f"✓ Created wallet: {self.config.dashd_wallet}") + print(f" Creating new wallet: {self.config.dashd_wallet}") + self.rpc.call("createwallet", self.config.dashd_wallet) + print(f" Created wallet: {self.config.dashd_wallet}") else: - print(f"✗ Unexpected wallet error: {e}") + print(f" Unexpected wallet error: {e}") raise def _load_addresses(self): - """Generate addresses using separate dashd wallets for each tier""" - print("\nGenerating separate HD wallets via dashd...") - - # Define wallet configurations: (name, tier, num_addresses) - wallet_configs = [ - ('light', 'light', 20), # Light usage: few addresses - ('normal', 'normal', 60), # Normal usage: moderate addresses - ('heavy', 'heavy', 120), # Heavy usage: many addresses - ] + """Wallet setup - must be implemented by subclass""" + raise NotImplementedError - for wallet_name, tier, num_addresses in wallet_configs: - print(f" Creating {wallet_name} wallet ({tier} tier, {num_addresses} addresses)...") + def _initialize_utxo_pool(self): + """UTXO funding - must be implemented by subclass""" + raise NotImplementedError - # Create a separate dashd wallet for this tier - try: - self.rpc.call('createwallet', wallet_name) - print(f" ✓ Created dashd wallet: {wallet_name}") - except RPCError as e: - error_msg = str(e).lower() - if "already exists" in error_msg or "already loaded" in error_msg: - print(f" ✓ Wallet already exists: {wallet_name}") - else: - raise - - # Get HD wallet info from this specific wallet - hd_info = self.rpc.call('dumphdinfo', wallet=wallet_name) - mnemonic = hd_info.get('mnemonic', '') - - addresses = [] - for i in range(num_addresses): - label = f"{wallet_name}_{i:03d}" - - # Generate new address in THIS wallet - address = self.rpc.call('getnewaddress', label, wallet=wallet_name) - - # Get the private key from THIS wallet - private_key = self.rpc.call('dumpprivkey', address, wallet=wallet_name) - - # Get address info to determine HD path - addr_info = self.rpc.call('getaddressinfo', address, wallet=wallet_name) - hd_path = addr_info.get('hdkeypath', f"m/44'/1'/0'/0/{i}") - - addr_dict = { - 'address': address, - 'label': label, - 'private_key': private_key, - 'hd_path': hd_path, - 'tier': tier, - 'wallet_name': wallet_name # Track which dashd wallet this address belongs to - } - - addresses.append(addr_dict) - self.all_addresses.append(addr_dict) - - # Progress update every 10 addresses - if (i + 1) % 10 == 0: - print(f" Generated {i + 1}/{num_addresses} addresses...") - - # Store wallet info with empty transaction and UTXO lists (will be filled during generation) - self.wallets.append({ - 'wallet_name': wallet_name, - 'mnemonic': mnemonic, - 'addresses': addresses, - 'tier': tier, - 'transactions': [], # Will be populated during block generation - 'utxos': [], # Will be populated at the end - 'balance': 0 # Will be calculated at the end (in satoshis) - }) - - print(f" ✓ {wallet_name} complete: {len(addresses)} addresses") - - print(f"\n✓ Generated {len(self.wallets)} wallets with {len(self.all_addresses)} total addresses") + def _generate_blocks(self): + """Block generation - must be implemented by subclass""" + raise NotImplementedError def _collect_wallet_statistics(self): - """Collect transaction history, UTXOs, and balance for each wallet (including miner)""" + """Collect transaction history, UTXOs, and balance for each wallet (including faucet)""" print("\n Collecting wallet statistics...") - # Collect stats for faucet (default) wallet - it mines blocks and funds others - faucet_wallet = self._collect_single_wallet_stats(self.config.dashd_wallet, 'faucet') - - # Add faucet wallet to beginning of wallets list so it's exported too - self.wallets.insert(0, faucet_wallet) - - # Collect stats for tier wallets (light, normal, heavy) - # Skip index 0 since that's the miner wallet we just added - for wallet in self.wallets[1:]: - wallet_name = wallet['wallet_name'] - stats = self._collect_single_wallet_stats(wallet_name, wallet.get('tier', 'unknown')) - - # Update the wallet dict with collected stats - wallet['transactions'] = stats['transactions'] - wallet['utxos'] = stats['utxos'] - wallet['balance'] = stats['balance'] - - def _collect_single_wallet_stats(self, wallet_name: str, tier: str) -> dict: - """Collect statistics for a single wallet""" - print(f" Processing {wallet_name}...") - - # Get all transactions from THIS wallet's dashd wallet - transactions = [] - try: - addr_txs = self.rpc.call('listtransactions', '*', 10000, 0, True, wallet=wallet_name) - for tx in addr_txs: - transactions.append({ - 'txid': tx['txid'], - 'address': tx.get('address', ''), - 'amount': tx['amount'], - 'confirmations': tx.get('confirmations', 0), - 'blockhash': tx.get('blockhash', ''), - 'time': tx.get('time', 0) - }) - except RPCError as e: - print(f" Warning: Error getting transactions: {e}") - - # Get UTXOs from THIS wallet - utxos_list = [] - balance = 0.0 - try: - wallet_utxos = self.rpc.call('listunspent', 1, 9999999, [], wallet=wallet_name) - - utxos_list = [ - { - 'txid': utxo['txid'], - 'vout': utxo['vout'], - 'address': utxo['address'], - 'amount': utxo['amount'], - 'confirmations': utxo['confirmations'] - } - for utxo in wallet_utxos - ] - balance = sum(utxo['amount'] for utxo in wallet_utxos) - except RPCError as e: - print(f" Warning: Error getting UTXOs: {e}") + for wallet in self.wallets: + wallet_name = wallet["wallet_name"] + print(f" Processing {wallet_name}...") + stats = collect_wallet_stats(self.rpc, wallet_name) - print(f" {len(transactions)} txs, {len(utxos_list)} UTXOs, balance: {balance:.8f} DASH") + wallet["transactions"] = stats["transactions"] + wallet["utxos"] = stats["utxos"] + wallet["balance"] = stats["balance"] + if stats.get("mnemonic"): + wallet["mnemonic"] = stats["mnemonic"] - # For miner wallet, get addresses if not already defined - addresses = [] - if wallet_name == self.config.dashd_wallet: - # Get HD info for miner wallet - try: - hd_info = self.rpc.call('dumphdinfo', wallet=wallet_name) - mnemonic = hd_info.get('mnemonic', '') - except RPCError: - mnemonic = '' - else: - mnemonic = '' - - return { - 'wallet_name': wallet_name, - 'mnemonic': mnemonic, - 'addresses': addresses, # Empty for miner, already set for tier wallets - 'tier': tier, - 'transactions': transactions, - 'utxos': utxos_list, - 'balance': balance - } + print( + f" {len(stats['transactions'])} txs, " + f"{len(stats['utxos'])} UTXOs, balance: {stats['balance']:.8f} DASH" + ) def _save_wallet_files(self): """Save each wallet to a separate JSON file in wallets/ directory""" print("\n Saving wallet files...") - # Create wallets directory wallets_dir = self.output_dir / "wallets" wallets_dir.mkdir(parents=True, exist_ok=True) for wallet in self.wallets: wallet_file = wallets_dir / f"{wallet['wallet_name']}.json" + save_wallet_file(wallet, wallet_file) - # Create wallet data structure - # Note: Addresses are not included in export since they're derived from mnemonic - wallet_data = { - 'wallet_name': wallet['wallet_name'], - 'mnemonic': wallet['mnemonic'], - 'balance': wallet['balance'], - 'transaction_count': len(wallet['transactions']), - 'utxo_count': len(wallet['utxos']), - 'transactions': wallet['transactions'], - 'utxos': wallet['utxos'] - } - - # Write to file - with open(wallet_file, 'w') as f: - json.dump(wallet_data, f, indent=2) - - print(f" ✓ {wallet['wallet_name']}.json: " - f"{len(wallet['addresses'])} addrs, {len(wallet['transactions'])} txs, " - f"{len(wallet['utxos'])} UTXOs, balance: {wallet['balance']:.8f} DASH") - - def _initialize_utxo_pool(self): - """ - Create initial UTXO pool and fund each wallet separately. - - This is critical for preventing "Insufficient funds" errors. - """ - print("\nInitializing UTXO pool and funding wallets...") - - # Generate initial blocks to default wallet for mining rewards - print(f" Generating initial blocks to default wallet...") - default_addr = self.rpc.call('getnewaddress', wallet=self.config.dashd_wallet) - self.rpc.call('generatetoaddress', 200, default_addr) - - print(f" Waiting for maturity (100 blocks)...") - self.rpc.call('generatetoaddress', 100, default_addr) - - # Generate more blocks to have plenty of funds for both splitting and wallet funding - print(f" Generating additional blocks for funding...") - self.rpc.call('generatetoaddress', 100, default_addr) - - print(f" Splitting default wallet into {self.config.target_utxo_count} UTXOs...") - self._split_utxos(self.config.target_utxo_count, self.config.dashd_wallet) - - self.utxo_count = len(self.rpc.call('listunspent', 1, wallet=self.config.dashd_wallet)) - print(f" ✓ Default wallet UTXO pool: {self.utxo_count} UTXOs") - - # Now fund each tier wallet from the default wallet - print(f"\n Funding tier wallets from default wallet...") - for wallet in self.wallets: - wallet_name = wallet['wallet_name'] - tier = wallet['tier'] - - if tier == 'light': - num_funding_txs = 20 - elif tier == 'normal': - num_funding_txs = 40 - else: - num_funding_txs = 60 - - print(f" Funding {wallet_name} with {num_funding_txs} transactions...") - - for i in range(num_funding_txs): - addr = wallet['addresses'][i % len(wallet['addresses'])]['address'] - amount = round(random.uniform(0.1, 5.0), 8) - try: - self.rpc.call('sendtoaddress', addr, amount, wallet=self.config.dashd_wallet) - except RPCError as e: - print(f" Warning: Failed to fund {addr}: {e}") - - self.rpc.call('generatetoaddress', 1, default_addr) - - balance = self.rpc.call('getbalance', wallet=wallet_name) - print(f" ✓ {wallet_name} funded: {balance:.2f} DASH") - - print(f"\n✓ All wallets funded and initialized") - - def _split_utxos(self, target_count: int, wallet_name: str): - """Split UTXOs to create enough for transactions""" - print(f" Creating UTXOs (target: {target_count})...") - - try: - default_addr = self.rpc.call('getnewaddress', wallet=self.config.dashd_wallet) - - iteration = 0 - while iteration < 10: - utxos = self.rpc.call('listunspent', 1, wallet=wallet_name) - current_count = len(utxos) - - if iteration % 2 == 0: - print(f" Current UTXOs: {current_count}/{target_count}") - - if current_count >= target_count: - break - - balance = self.rpc.call('getbalance', wallet=wallet_name) - if balance < 10: - break - - recipients = {} - for i in range(min(50, target_count - current_count)): - wallet_addr = self.rpc.call('getnewaddress', wallet=wallet_name) - recipients[wallet_addr] = 1.0 - - try: - self.rpc.call('sendmany', '', json.dumps(recipients), wallet=wallet_name) - except (RPCError, InsufficientFundsError): - break - - self.rpc.call('generatetoaddress', 1, default_addr) - iteration += 1 - - final_count = len(self.rpc.call('listunspent', 1, wallet=wallet_name)) - print(f" ✓ UTXO creation complete: {final_count} UTXOs") - except InsufficientFundsError: - print(f" ⚠ UTXO creation stopped early") - print(f" Will use existing UTXOs") - - def _generate_blocks(self): - """Generate blocks with transactions to reach target height""" - # Calculate how many blocks we need to generate to reach target - current_height = self.rpc.call('getblockcount') - blocks_to_generate = self.config.target_blocks - current_height - - if blocks_to_generate <= 0: - print(f"\nAlready at or past target height ({current_height} >= {self.config.target_blocks})") - return - - print(f"\nGenerating {blocks_to_generate} blocks to reach height {self.config.target_blocks}...") - print(f" Current height: {current_height}") - - start_time = time.time() - tx_attempts = 0 - tx_successes = 0 - - for block_num in range(blocks_to_generate): - tx_count = self._determine_tx_count() - - # Create transactions (they will accumulate in mempool) - if tx_count > 0: - for _ in range(tx_count): - tx_attempts += 1 - before_count = self.stats['transactions_created'] - self._create_transaction() - if self.stats['transactions_created'] > before_count: - tx_successes += 1 - - # Mine a block (this confirms all mempool transactions) - addr = random.choice(self.all_addresses)['address'] - self.rpc.call('generatetoaddress', 1, addr) - self.stats['blocks_generated'] += 1 - - if (block_num + 1) % 100 == 0: - elapsed = time.time() - start_time - rate = (block_num + 1) / elapsed - eta_seconds = (blocks_to_generate - block_num - 1) / rate if rate > 0 else 0 - eta = datetime.timedelta(seconds=int(eta_seconds)) - - # Calculate actual current height (start height + blocks generated so far) - actual_height = self.rpc.call('getblockcount') - print(f" Height {actual_height}/{self.config.target_blocks} " - f"({rate:.1f} blocks/sec, ETA: {eta}) " - f"[Tx: {tx_successes}/{tx_attempts} successful]") - - print(f"✓ Generated {self.stats['blocks_generated']} blocks") - print(f"✓ Transaction success rate: {tx_successes}/{tx_attempts} ({100*tx_successes/tx_attempts if tx_attempts > 0 else 0:.1f}%)") - - def _determine_tx_count(self) -> int: - """Determine how many transactions for this block based on configured probabilities""" - rand = random.random() - - if rand < self.config.tx_probability_none: - return 0 - elif rand < self.config.tx_probability_low: - return random.randint(1, 3) - elif rand < self.config.tx_probability_medium: - return random.randint(4, 10) - else: - return random.randint(11, 25) - - def _refund_wallet(self, wallet: dict): - """Refund a wallet from the faucet when it runs low on funds""" - wallet_name = wallet['wallet_name'] - tier = wallet['tier'] - - if tier == 'light': - refund_amount = 20.0 - elif tier == 'normal': - refund_amount = 40.0 - else: - refund_amount = 60.0 - - # Send funds from faucet to a random address in this wallet - addr = random.choice(wallet['addresses'])['address'] - - try: - self.rpc.call('sendtoaddress', addr, refund_amount, wallet=self.config.dashd_wallet) - # Refund will be confirmed in next block generated by main loop - self.stats['utxo_replenishments'] += 1 - except RPCError as e: - print(f" Warning: Failed to refund {wallet_name}: {e}") - - def _create_transaction(self): - """Create a single transaction using manual UTXO selection""" - source_wallet = random.choice(self.wallets) - source_wallet_name = source_wallet['wallet_name'] - - try: - # Get balance to check if wallet needs refunding - balance = self.rpc.call('getbalance', wallet=source_wallet_name) - - # Check if wallet needs refunding - if balance < 10.0: - self._refund_wallet(source_wallet) - balance = self.rpc.call('getbalance', wallet=source_wallet_name) - if balance < 1.0: # Still too low after refund - return - - # Decide transaction type - # More multi-output transactions to create varied UTXO sets - if random.random() < 0.4: - tx_type = 'multi_output' - num_outputs = random.randint(2, 5) - else: - tx_type = 'simple' - num_outputs = 1 - - # Use sendtoaddress or sendmany which automatically calculates proper fees - if tx_type == 'simple': - dest_addr = random.choice(self.all_addresses)['address'] - # Send a reasonable amount (1-5 DASH or 10-50% of balance, whichever is smaller) - max_amount = min(balance * random.uniform(0.1, 0.5), random.uniform(1, 5)) - amount = round(max_amount, 8) - - if amount < 0.01: - return - - # Use sendtoaddress which handles fees automatically - self.rpc.call('sendtoaddress', dest_addr, amount, wallet=source_wallet_name) - else: - # Multi-output: use sendmany - outputs = {} - # Split 20-50% of balance across multiple outputs - total_to_send = min(balance * random.uniform(0.2, 0.5), random.uniform(5, 15)) - amount_per_output = round(total_to_send / num_outputs, 8) - - if amount_per_output < 0.01: - return - - for _ in range(num_outputs): - dest_addr = random.choice(self.all_addresses)['address'] - outputs[dest_addr] = amount_per_output - - # Use sendmany which handles fees automatically - self.rpc.call('sendmany', "", json.dumps(outputs), wallet=source_wallet_name) - - self.stats['transactions_created'] += 1 - - except (TransactionCreationError, RPCError, InsufficientFundsError): - # Transaction failed (insufficient funds, fee issues, etc.) - this is expected - pass - except Exception as e: - # Catch any other unexpected errors - print(f" Unexpected error in transaction creation: {type(e).__name__}: {e}") - pass + print( + f" {wallet['wallet_name']}.json: " + f"{len(wallet.get('addresses', []))} addrs, {len(wallet['transactions'])} txs, " + f"{len(wallet['utxos'])} UTXOs, balance: {wallet['balance']:.8f} DASH" + ) def _export_data(self): """Export blockchain data""" print("\nExporting blockchain data...") # Verify we reached target height - final_height = self.rpc.call('getblockcount') + final_height = self.rpc.call("getblockcount") if final_height != self.config.target_blocks: print(f" Warning: Final height ({final_height}) differs from target ({self.config.target_blocks})") @@ -668,40 +283,27 @@ def _export_data(self): print(f" Warning: Could not remove temp directory: {e}") self.dashd_manager.temp_dir = None - print(f"\n✓ Exported to {self.output_dir}") + print(f"\n Exported to {self.output_dir}") def _copy_dashd_datadir(self, output_dir: Path): - """Copy dashd datadir for direct use in tests - - Structure will be: - regtest-N/ - regtest/ # Blockchain data (what dashd expects when using -datadir=regtest-N) - blocks/ - chainstate/ - default/ # Wallet directories - light/ - normal/ - heavy/ - wallets/ # JSON wallet files for SPV client - default.json - light.json - normal.json - heavy.json + """Copy dashd datadir for direct use in tests. + + Wallet directory names are derived from self.wallets. """ if not self.config.dashd_datadir: - print(" ⚠ No dashd datadir to copy (not using auto-start)") + print(" No dashd datadir to copy (not using auto-start)") return source_dir = Path(self.config.dashd_datadir) if not source_dir.exists(): - print(f" ⚠ Source datadir does not exist: {source_dir}") + print(f" Source datadir does not exist: {source_dir}") return print(f" Copying dashd datadir from {source_dir}...") regtest_source = source_dir / "regtest" if regtest_source.exists(): - print(f" Copying regtest directory...") + print(" Copying regtest directory...") # Copy regtest/ directory to output_dir/regtest/ (preserve directory structure) regtest_dest = output_dir / "regtest" @@ -710,12 +312,13 @@ def _copy_dashd_datadir(self, output_dir: Path): shutil.copytree(regtest_source, regtest_dest, symlinks=False) - total_size = sum(f.stat().st_size for f in regtest_dest.rglob('*') if f.is_file()) + total_size = sum(f.stat().st_size for f in regtest_dest.rglob("*") if f.is_file()) size_mb = total_size / 1024 / 1024 - print(f" ✓ Copied regtest data ({size_mb:.1f} MB)") + print(f" Copied regtest data ({size_mb:.1f} MB)") - expected_wallets = ['default', 'light', 'normal', 'heavy'] + # Derive expected wallet names from self.wallets + expected_wallets = [w["wallet_name"] for w in self.wallets] found_wallets = [] for wallet_name in expected_wallets: wallet_dir = regtest_dest / wallet_name @@ -723,121 +326,546 @@ def _copy_dashd_datadir(self, output_dir: Path): found_wallets.append(wallet_name) if found_wallets: - print(f" ✓ Wallet directories copied ({len(found_wallets)} wallets: {', '.join(found_wallets)})") + print(f" Wallet directories copied ({len(found_wallets)} wallets: {', '.join(found_wallets)})") else: - print(f" ⚠ No wallet directories found in regtest") + print(" No wallet directories found in regtest") + else: + print(f" No regtest directory found in {source_dir}") + + +class WalletSyncGenerator(Generator): + """Generates a blockchain optimized for SPV wallet sync testing. + + Creates ~40K blocks with very few targeted transactions (~50-80) + that exercise every critical SPV sync edge case: + - Address discovery at various indices + - Gap limit boundary and extension + - Change address activity (spending from wallet) + - Immature and mature coinbase rewards + - Dust and large value transactions + - Batched payments (sendmany) and consolidation (raw tx) + - Empty block stretches + - Address reuse + - Transactions at filter batch boundaries + """ + + WALLET_NAME = "wallet" + NUM_ADDRESSES = 50 + + def __init__(self, config: Config, keep_temp: bool = False): + super().__init__(config, keep_temp) + # address index -> address string + self.wallet_addresses: dict[int, str] = {} + + def strategy_name(self) -> str: + return "wallet-sync" + + def _load_addresses(self): + """Create a single test wallet with pre-generated addresses.""" + print(f"\nCreating test wallet '{self.WALLET_NAME}' with {self.NUM_ADDRESSES} addresses...") + + # Initialize faucet wallet placeholder + self.wallets.append( + { + "wallet_name": self.config.dashd_wallet, + "mnemonic": "", + "addresses": [], + "tier": "faucet", + "transactions": [], + "utxos": [], + "balance": 0, + } + ) + + # Create the test wallet in dashd + try: + self.rpc.call("createwallet", self.WALLET_NAME) + print(f" Created dashd wallet: {self.WALLET_NAME}") + except RPCError as e: + error_msg = str(e).lower() + if "already exists" in error_msg or "already loaded" in error_msg: + print(f" Wallet already exists: {self.WALLET_NAME}") + else: + raise + + # Get HD wallet mnemonic + hd_info = self.rpc.call("dumphdinfo", wallet=self.WALLET_NAME) + mnemonic = hd_info.get("mnemonic", "") + + # Pre-generate addresses at specific indices + # dashd generates addresses sequentially, so generating N addresses + # gives us indices 0 through N-1 + addresses = [] + for i in range(self.NUM_ADDRESSES): + address = self.rpc.call("getnewaddress", f"addr_{i}", wallet=self.WALLET_NAME) + self.wallet_addresses[i] = address + addresses.append({"address": address, "index": i}) + + self.wallets.append( + { + "wallet_name": self.WALLET_NAME, + "mnemonic": mnemonic, + "addresses": addresses, + "tier": "test", + "transactions": [], + "utxos": [], + "balance": 0, + } + ) + + print(f" Generated {len(self.wallet_addresses)} addresses") + print(f" Mnemonic: {mnemonic}") + + def _initialize_utxo_pool(self): + """Mine initial blocks for coinbase maturity and split faucet UTXOs.""" + print("\nBootstrap: mining initial blocks and creating faucet UTXOs...") + + self.mining_address = self.rpc.call("getnewaddress", wallet=self.config.dashd_wallet) + self.rpc.call("generatetoaddress", 110, self.mining_address) + + current_height = self.rpc.call("getblockcount") + print(f" Mined 110 blocks (height: {current_height})") + + # Split into ~50 UTXOs for funding operations + print(" Splitting faucet into ~50 UTXOs...") + recipients = {} + for _ in range(50): + addr = self.rpc.call("getnewaddress", wallet=self.config.dashd_wallet) + recipients[addr] = 10.0 + self.rpc.call("sendmany", "", recipients, wallet=self.config.dashd_wallet) + self.rpc.call("generatetoaddress", 1, self.mining_address) + + utxo_count = len(self.rpc.call("listunspent", 1, wallet=self.config.dashd_wallet)) + print(f" Faucet UTXO pool: {utxo_count} UTXOs") + + def _generate_blocks(self): + """Execute phased block generation.""" + current_height = self.rpc.call("getblockcount") + target = self.config.target_blocks + + print(f"\nGenerating blocks to reach height {target}...") + print(f" Current height: {current_height}") + + start_time = time.time() + + # Phase 2: Normal activity (heights ~116-200) + self._phase_normal_activity() + + # Phase 3: Gap limit boundary (heights ~201-230) + self._phase_gap_limit_boundary() + + # Phase 4: Beyond gap limit (heights ~231-260) + self._phase_beyond_gap_limit() + + # Phase 5: Transaction variety (heights ~261-320) + self._phase_transaction_variety() + + # Phase 6: Bulk generation with boundary transactions + self._phase_bulk_generation() + + elapsed = time.time() - start_time + final_height = self.rpc.call("getblockcount") + self.stats["blocks_generated"] = final_height - current_height + + print("\n Completed all phases") + print(f" Final height: {final_height}") + print(f" Transactions to test wallet: {self.stats['transactions_created']}") + print(f" Coinbase rewards to test wallet: {self.stats['coinbase_rewards']}") + print(f" Duration: {datetime.timedelta(seconds=int(elapsed))}") + + def _send_to_wallet(self, index: int, amount: float, description: str = ""): + """Send funds from faucet to the test wallet at a specific address index.""" + address = self.wallet_addresses[index] + self.rpc.call("sendtoaddress", address, amount, wallet=self.config.dashd_wallet) + self.stats["transactions_created"] += 1 + if description: + print(f" Sent {amount} DASH to index {index} ({description})") else: - print(f" ⚠ No regtest directory found in {source_dir}") + print(f" Sent {amount} DASH to index {index}") + + def _mine_blocks(self, count: int, address: str | None = None): + """Mine blocks to the given address (or faucet if not specified).""" + if address is None: + address = self.mining_address + self.rpc.call("generatetoaddress", count, address) + + def _mine_and_log(self, count: int, description: str = ""): + """Mine blocks to faucet and log progress.""" + self._mine_blocks(count) + height = self.rpc.call("getblockcount") + if description: + print(f" Mined {count} blocks -> height {height} ({description})") + + def _phase_normal_activity(self): + """Phase 2: Normal transaction activity to various address indices.""" + print("\n Phase 2: Normal activity") + + # Basic address discovery at various indices + # Dust transaction + self._send_to_wallet(0, 0.00001, "dust") + self._mine_blocks(2) + + # Small amounts + self._send_to_wallet(2, 0.05, "small") + self._send_to_wallet(5, 0.5, "medium") + self._mine_blocks(2) + + # Medium amounts + self._send_to_wallet(8, 1.0, "medium") + self._send_to_wallet(12, 2.5, "medium") + self._mine_blocks(2) + + # Large value + self._send_to_wallet(15, 100.0, "large") + self._send_to_wallet(20, 0.1, "small") + self._mine_blocks(2) + + # Address reuse: send again to index 5 + self._send_to_wallet(5, 0.25, "address reuse") + self._mine_blocks(1) + + # Batched payment (sendmany) hitting multiple indices + recipients = { + self.wallet_addresses[3]: 0.1, + self.wallet_addresses[7]: 0.2, + self.wallet_addresses[14]: 0.3, + } + self.rpc.call("sendmany", "", recipients, wallet=self.config.dashd_wallet) + self.stats["transactions_created"] += 1 + print(" Sendmany to indices 3, 7, 14") + self._mine_blocks(1) + + self._mine_and_log(10, "padding after normal activity") + + height = self.rpc.call("getblockcount") + print(f" Phase 2 complete at height {height}") + + def _phase_gap_limit_boundary(self): + """Phase 3: Transactions at the gap limit boundary (indices 27, 28, 29).""" + print("\n Phase 3: Gap limit boundary") + + # Gap limit in most HD wallets is 20-30 (typically 30 for external addresses) + # Indices 27, 28, 29 are the last addresses within the initial gap of 30 + self._send_to_wallet(27, 0.3, "gap limit -3") + self._mine_blocks(3) + + self._send_to_wallet(28, 0.4, "gap limit -2") + self._mine_blocks(3) + + self._send_to_wallet(29, 0.5, "gap limit -1 (last in initial gap)") + self._mine_blocks(3) + + self._mine_and_log(10, "padding after gap limit") + + height = self.rpc.call("getblockcount") + print(f" Phase 3 complete at height {height}") + + def _phase_beyond_gap_limit(self): + """Phase 4: Transactions beyond initial gap limit. + + These are only discoverable after index 29 triggers gap extension to index 59. + """ + print("\n Phase 4: Beyond gap limit") + + self._send_to_wallet(32, 0.6, "beyond gap (discoverable after rescan)") + self._mine_blocks(5) + + self._send_to_wallet(35, 0.7, "beyond gap (discoverable after rescan)") + self._mine_blocks(5) + + self._mine_and_log(10, "padding after beyond-gap") + + height = self.rpc.call("getblockcount") + print(f" Phase 4 complete at height {height}") + + def _phase_transaction_variety(self): + """Phase 5: Various transaction types - spend from wallet, consolidation.""" + print("\n Phase 5: Transaction variety") + + # First fund the wallet enough to be able to spend from it + self._send_to_wallet(0, 5.0, "funding for spend-from-wallet") + self._mine_blocks(1) + + # Spend FROM the test wallet (generates change to internal address) + # Send from test wallet to faucet + faucet_addr = self.rpc.call("getnewaddress", wallet=self.config.dashd_wallet) + try: + self.rpc.call("sendtoaddress", faucet_addr, 1.0, wallet=self.WALLET_NAME) + self.stats["transactions_created"] += 1 + print(" Spent 1.0 DASH from test wallet (generates change output)") + except RPCError as e: + print(f" Warning: Failed to spend from wallet: {e}") - def _export_blocks_dat(self, output_dir: Path): - """Export blocks in binary format (legacy format, kept for backward compatibility)""" - print(" Exporting blocks.dat...") + self._mine_blocks(3) - block_count = self.rpc.call('getblockcount') - blocks_file = output_dir / "blocks.dat" + # Consolidation: raw transaction merging wallet UTXOs + try: + wallet_utxos = self.rpc.call("listunspent", 1, 9999999, [], wallet=self.WALLET_NAME) + if len(wallet_utxos) >= 2: + # Pick 2 small UTXOs to consolidate + selected = sorted(wallet_utxos, key=lambda u: u["amount"])[:2] + total = sum(u["amount"] for u in selected) + fee = 0.0001 + + if total > fee: + inputs = [{"txid": u["txid"], "vout": u["vout"]} for u in selected] + dest = self.rpc.call("getnewaddress", wallet=self.WALLET_NAME) + outputs = {dest: round(total - fee, 8)} + + raw_tx = self.rpc.call("createrawtransaction", inputs, outputs) + signed = self.rpc.call("signrawtransactionwithwallet", raw_tx, wallet=self.WALLET_NAME) + + if signed.get("complete", False): + self.rpc.call("sendrawtransaction", signed["hex"]) + self.stats["transactions_created"] += 1 + print(f" Consolidation tx: merged {len(selected)} UTXOs") + except RPCError as e: + print(f" Warning: Consolidation failed: {e}") + + self._mine_blocks(3) + + self._mine_and_log(10, "padding after transaction variety") + + height = self.rpc.call("getblockcount") + print(f" Phase 5 complete at height {height}") + + def _phase_bulk_generation(self): + """Phase 6: Generate remaining blocks in large batches. - with open(blocks_file, 'wb') as f: - f.write(struct.pack(' 0 and height % 50 == 0: - progress_pct = (height * 100) // block_count - print(f" Exporting blocks: {height}/{block_count} ({progress_pct}%)") + print(f"\n Phase 6: Bulk generation ({blocks_remaining} blocks remaining)") - block_hash = self.rpc.call('getblockhash', height) - raw_hex = self.rpc.call('getblock', block_hash, 0) - raw_bytes = bytes.fromhex(raw_hex) + # Calculate batch boundary heights (every 5000 blocks) + batch_boundaries = self._calculate_batch_boundaries(current_height, target) + if batch_boundaries: + print(f" Batch boundaries to hit: {batch_boundaries}") - f.write(struct.pack(' current_height: + next_important_height = min(next_important_height, boundary) + + # Check if we need to handle mature coinbase range + if current_height < mature_coinbase_start and mature_coinbase_start < next_important_height: + next_important_height = mature_coinbase_start + if current_height < mature_coinbase_end and mature_coinbase_end < next_important_height: + next_important_height = mature_coinbase_end + if current_height < immature_coinbase_start and immature_coinbase_start < next_important_height: + next_important_height = immature_coinbase_start + + blocks_to_mine = min(next_important_height - current_height, batch_size) + blocks_to_mine = max(blocks_to_mine, 1) + + # Determine if this batch includes special blocks + batch_end = current_height + blocks_to_mine + + # Check if we're in the mature coinbase range + if mature_coinbase_start <= current_height < mature_coinbase_end: + # Mine some blocks to test wallet for mature coinbase + wallet_blocks = min(5, batch_end - current_height) + self._mine_blocks(wallet_blocks, coinbase_wallet_addr) + self.stats["coinbase_rewards"] += wallet_blocks + current_height += wallet_blocks + print(f" Mined {wallet_blocks} blocks to wallet (mature coinbase) at height {current_height}") + + # Mine remaining to faucet + remaining = batch_end - current_height + if remaining > 0: + self._mine_blocks(remaining) + current_height += remaining + continue + + # Check if we're in the immature coinbase range + if immature_coinbase_start <= current_height: + # Mine some blocks to test wallet for immature coinbase + wallet_blocks = min(5, target - current_height) + self._mine_blocks(wallet_blocks, coinbase_wallet_addr) + self.stats["coinbase_rewards"] += wallet_blocks + current_height += wallet_blocks + print(f" Mined {wallet_blocks} blocks to wallet (immature coinbase) at height {current_height}") + + # Mine remaining to faucet + remaining = target - current_height + if remaining > 0: + self._mine_blocks(remaining) + current_height += remaining + continue + + # Normal bulk mining + self._mine_blocks(blocks_to_mine) + current_height += blocks_to_mine + + # Place batch boundary transaction if we just passed one + for boundary in list(batch_boundaries): + if boundary <= current_height: + if boundary_addr_index < self.NUM_ADDRESSES: + self._send_to_wallet(boundary_addr_index, 0.01, f"batch boundary near height {boundary}") + self._mine_blocks(1) + current_height += 1 + boundary_addr_index += 1 + batch_boundaries.remove(boundary) + + # Periodic send to test wallet (~every 1000 blocks) + if current_height >= next_periodic_height: + idx = periodic_addresses[periodic_counter % len(periodic_addresses)] + amt = periodic_amounts[periodic_counter % len(periodic_amounts)] + try: + self._send_to_wallet(idx, amt, f"periodic at height {current_height}") + self._mine_blocks(1) + current_height += 1 + except RPCError: + pass + periodic_counter += 1 + next_periodic_height = current_height + periodic_interval + + # Occasional faucet self-send for filter variety + if random.random() < 0.01: + try: + faucet_addr = self.rpc.call("getnewaddress", wallet=self.config.dashd_wallet) + self.rpc.call("sendtoaddress", faucet_addr, 1.0, wallet=self.config.dashd_wallet) + self._mine_blocks(1) + current_height += 1 + except RPCError: + pass + + # Progress logging + elapsed = time.time() - start_time + rate = (current_height - (target - blocks_remaining)) / elapsed if elapsed > 0 else 0 + remaining_blocks = target - current_height + eta_seconds = remaining_blocks / rate if rate > 0 else 0 + eta = datetime.timedelta(seconds=int(eta_seconds)) + + if current_height % 5000 < batch_size or current_height >= target: + print(f" Height {current_height}/{target} ({rate:.0f} blocks/sec, ETA: {eta})") + + # Verify final height + actual = self.rpc.call("getblockcount") + if actual > target: + print(f" Warning: overshot target by {actual - target} blocks (height: {actual})") + elif actual < target: + # Mine remaining blocks + self._mine_blocks(target - actual) + actual = self.rpc.call("getblockcount") + + print(f" Phase 6 complete at height {actual}") + + @staticmethod + def _calculate_batch_boundaries(current_height: int, target: int) -> list[int]: + """Calculate filter batch boundary heights between current and target. + + Boundaries are at every 5000 blocks. We place a transaction just before + each boundary (at boundary - 1). + """ + boundaries = [] + # Start from next 5000 boundary above current height + first_boundary = ((current_height // 5000) + 1) * 5000 + for boundary in range(first_boundary, target + 1, 5000): + # Place transaction just before the boundary + height = boundary - 1 + if height > current_height and height < target: + boundaries.append(height) + return boundaries def main(): import argparse - parser = argparse.ArgumentParser(description="Generate Dash regtest test data") - parser.add_argument('--blocks', type=int, default=100, - help="Number of blocks to generate (default: 100)") - parser.add_argument('--dashd-path', type=str, - help="Path to dashd executable (default: dashd in PATH)") - parser.add_argument('--no-auto-start', action='store_true', - help="Disable automatic dashd startup") - parser.add_argument('--rpc-port', type=int, - help="RPC port to use (default: auto-detect)") - parser.add_argument('--keep-temp', action='store_true', - help="Keep temporary directory after completion") - parser.add_argument('--tx-density', type=str, choices=['minimal', 'light', 'normal', 'heavy'], default='normal', - help="Transaction density per block (default: normal)") + parser = argparse.ArgumentParser(description="Generate Dash regtest test data for SPV wallet sync testing") + parser.add_argument( + "--strategy", + type=str, + choices=["wallet-sync"], + default="wallet-sync", + help="Generation strategy (default: wallet-sync)", + ) + parser.add_argument("--blocks", type=int, default=200, help="Target blockchain height (minimum: 120, default: 200)") + parser.add_argument("--dashd-path", type=str, help="Path to dashd executable (default: dashd in PATH)") + parser.add_argument("--no-auto-start", action="store_true", help="Disable automatic dashd startup") + parser.add_argument("--rpc-port", type=int, help="RPC port to use (default: auto-detect)") + parser.add_argument("--keep-temp", action="store_true", help="Keep temporary directory after completion") + parser.add_argument("--output-dir", type=str, help="Output base directory (default: data/ next to generate.py)") args = parser.parse_args() - # Get script directory for resolving relative paths - script_dir = Path(__file__).parent.resolve() + # Validate minimum block count (need 100+ for coinbase maturity plus setup) + min_blocks = 120 + if args.blocks < min_blocks: + print(f"ERROR: --blocks must be at least {min_blocks} (coinbase maturity requirement)") + sys.exit(1) # Determine dashd and dash-cli paths if args.dashd_path: dashd_executable = args.dashd_path dashd_dir = Path(args.dashd_path).parent - dashcli_path = str(dashd_dir / 'dash-cli') + dashcli_path = str(dashd_dir / "dash-cli") else: - dashd_executable = 'dashd' - dashcli_path = 'dash-cli' - - # Resolve paths relative to script directory - # Output test data to test-data directory - output_base = str(script_dir.resolve()) - - # Configure transaction density based on --tx-density argument - tx_density_configs = { - 'minimal': { - 'tx_probability_none': 0.90, # 90% empty blocks - 'tx_probability_low': 0.98, # 8% with 1-3 transactions - 'tx_probability_medium': 1.0 # 2% with 4-10 transactions - }, - 'light': { - 'tx_probability_none': 0.60, # 60% empty blocks - 'tx_probability_low': 0.90, # 30% with 1-3 transactions - 'tx_probability_medium': 0.98 # 8% with 4-10 transactions, 2% with 11-25 - }, - 'normal': { - 'tx_probability_none': 0.30, # 30% empty blocks - 'tx_probability_low': 0.70, # 40% with 1-3 transactions - 'tx_probability_medium': 0.90 # 20% with 4-10 transactions, 10% with 11-25 - }, - 'heavy': { - 'tx_probability_none': 0.10, # 10% empty blocks - 'tx_probability_low': 0.40, # 30% with 1-3 transactions - 'tx_probability_medium': 0.70 # 30% with 4-10 transactions, 30% with 11-25 - } - } + dashd_executable = "dashd" + dashcli_path = "dash-cli" - density_config = tx_density_configs[args.tx_density] + # Output directory + if args.output_dir: + output_base = args.output_dir + else: + script_dir = Path(__file__).parent.resolve() + output_base = str(script_dir / "data") - # Create config with defaults config = Config( target_blocks=args.blocks, - batch_size=50, # Not used currently - min_utxo_threshold=150, - target_utxo_count=200, dashcli_path=dashcli_path, dashd_executable=dashd_executable, auto_start_dashd=not args.no_auto_start, dashd_datadir=None, - dashd_wallet='default', + dashd_wallet="default", rpc_port=args.rpc_port, output_base=output_base, - tx_probability_none=density_config['tx_probability_none'], - tx_probability_low=density_config['tx_probability_low'], - tx_probability_medium=density_config['tx_probability_medium'] + extra_dashd_args=[ + "-blockfilterindex=1", + "-peerblockfilters=1", + ], ) + strategies = { + "wallet-sync": WalletSyncGenerator, + } + generator = strategies[args.strategy](config, keep_temp=args.keep_temp) + try: - generator = Generator(config, keep_temp=args.keep_temp) generator.generate() except ConfigError as e: @@ -847,9 +875,7 @@ def main(): print(f"ERROR: Cannot connect to dashd: {e}") sys.exit(2) except InsufficientFundsError as e: - # This can happen during UTXO splitting or transaction creation and is handled gracefully print(f"ERROR: Insufficient funds: {e}") - print("Note: This typically happens during aggressive UTXO splitting and is expected") sys.exit(3) except GeneratorError as e: print(f"ERROR: {e}") @@ -859,5 +885,5 @@ def main(): sys.exit(130) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/generator/dashd_manager.py b/generator/dashd_manager.py index a1da7ed..f7fddb5 100644 --- a/generator/dashd_manager.py +++ b/generator/dashd_manager.py @@ -4,27 +4,51 @@ Handles automatic startup and lifecycle management of dashd for test data generation. """ -import subprocess +import atexit +import os +import shutil import socket -import time +import subprocess import tempfile -import shutil -import atexit +import time from pathlib import Path -from typing import Optional, Tuple + from .errors import DashdConnectionError +def dashd_preexec_fn(): + """Set file descriptor limit before starting dashd. + + dashd requires a numeric limit and fails silently with "unlimited", + so we cap at a reasonable target without exceeding the system hard limit. + """ + try: + import resource + + _soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + target = 10000 + if hard == resource.RLIM_INFINITY: + new_soft = target + new_hard = target + else: + new_soft = min(target, hard) + new_hard = min(hard, target) + resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, new_hard)) + except (ImportError, OSError): + pass + + class DashdManager: """Manages dashd process lifecycle with automatic port detection and cleanup""" - def __init__(self, dashd_executable: str = "dashd", rpc_port: Optional[int] = None): + def __init__(self, dashd_executable: str = "dashd", rpc_port: int | None = None, extra_args: list | None = None): self.dashd_executable = dashd_executable self.requested_port = rpc_port - self.actual_port: Optional[int] = None - self.p2p_port: Optional[int] = None - self.temp_dir: Optional[Path] = None - self.process: Optional[subprocess.Popen] = None + self.extra_args = extra_args or [] + self.actual_port: int | None = None + self.p2p_port: int | None = None + self.temp_dir: Path | None = None + self.process: subprocess.Popen | None = None self.should_cleanup = True def is_port_available(self, port: int) -> bool: @@ -32,7 +56,7 @@ def is_port_available(self, port: int) -> bool: try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(('127.0.0.1', port)) + sock.bind(("127.0.0.1", port)) return True except OSError: return False @@ -42,24 +66,17 @@ def find_free_port(self, start_port: int = 19998, max_attempts: int = 20) -> int for port in range(start_port, start_port + max_attempts): if self.is_port_available(port): return port - raise DashdConnectionError( - f"No free RPC port found in range {start_port}-{start_port + max_attempts - 1}" - ) + raise DashdConnectionError(f"No free RPC port found in range {start_port}-{start_port + max_attempts - 1}") def verify_dashd_executable(self) -> bool: """Check if dashd executable exists and is runnable""" - try: - result = subprocess.run( - [self.dashd_executable, '--version'], - capture_output=True, - text=True, - timeout=5 - ) - return result.returncode == 0 - except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError): - return False + path = Path(self.dashd_executable) + if path.is_absolute(): + return path.is_file() and os.access(path, os.X_OK) + # For non-absolute paths, check if it's findable via shutil.which + return shutil.which(self.dashd_executable) is not None - def start(self, keep_temp: bool = False) -> Tuple[int, Path]: + def start(self, keep_temp: bool = False) -> tuple[int, Path]: """ Start dashd in a temporary directory with auto-detected port. @@ -81,9 +98,7 @@ def start(self, keep_temp: bool = False) -> Tuple[int, Path]: # Determine RPC port if self.requested_port: if not self.is_port_available(self.requested_port): - raise DashdConnectionError( - f"Requested RPC port {self.requested_port} is not available" - ) + raise DashdConnectionError(f"Requested RPC port {self.requested_port} is not available") self.actual_port = self.requested_port else: self.actual_port = self.find_free_port(19998) @@ -92,11 +107,11 @@ def start(self, keep_temp: bool = False) -> Tuple[int, Path]: self.p2p_port = self.find_free_port(self.actual_port + 1) # Create temporary directory - self.temp_dir = Path(tempfile.mkdtemp(prefix='dash-testdata-')) + self.temp_dir = Path(tempfile.mkdtemp(prefix="dash-testdata-")) self.should_cleanup = not keep_temp # Create regtest subdirectory (dashd requires it to exist) - regtest_dir = self.temp_dir / 'regtest' + regtest_dir = self.temp_dir / "regtest" regtest_dir.mkdir(exist_ok=True) print(f" Using temporary directory: {self.temp_dir}") @@ -106,46 +121,39 @@ def start(self, keep_temp: bool = False) -> Tuple[int, Path]: # Build dashd command cmd = [ self.dashd_executable, - '-regtest', - f'-datadir={self.temp_dir}', - f'-port={self.p2p_port}', - f'-rpcport={self.actual_port}', - '-server=1', - '-daemon=0', # Run in foreground (we manage the process) - '-fallbackfee=0.00001', - '-rpcbind=127.0.0.1', - '-rpcallowip=127.0.0.1', - '-listen=1', - '-txindex=0', - '-addressindex=0', - '-spentindex=0', - '-timestampindex=0', + "-regtest", + f"-datadir={self.temp_dir}", + f"-port={self.p2p_port}", + f"-rpcport={self.actual_port}", + "-server=1", + "-daemon=0", # Run in foreground (we manage the process) + "-fallbackfee=0.00001", + "-rpcbind=127.0.0.1", + "-rpcallowip=127.0.0.1", + "-listen=1", + "-txindex=0", + "-addressindex=0", + "-spentindex=0", + "-timestampindex=0", ] - # Start dashd process with increased file descriptor limit - try: - import resource - - def preexec_fn(): - """Set file descriptor limit before starting dashd""" - try: - # Increase file descriptor limit to 10000 - resource.setrlimit(resource.RLIMIT_NOFILE, (10000, 10000)) - except: - pass # Ignore if we can't set it + # Append strategy-specific args + cmd.extend(self.extra_args) + # Start dashd process with a finite file descriptor limit. + # dashd requires a numeric limit and fails silently with "unlimited". + try: self.process = subprocess.Popen( cmd, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stderr=subprocess.PIPE, cwd=str(self.temp_dir), - preexec_fn=preexec_fn + preexec_fn=dashd_preexec_fn, ) - except FileNotFoundError: + except FileNotFoundError as e: raise DashdConnectionError( - f"Failed to execute dashd: {self.dashd_executable}\n" - f"Please check the path or install Dash Core" - ) + f"Failed to execute dashd: {self.dashd_executable}\nPlease check the path or install Dash Core" + ) from e # Register cleanup handler atexit.register(self.stop) @@ -154,11 +162,9 @@ def preexec_fn(): print(f" Waiting for dashd to be ready (PID: {self.process.pid})...") if not self._wait_for_ready(timeout=30): self.stop() - raise DashdConnectionError( - "dashd failed to start within 30 seconds. Check dashd installation." - ) + raise DashdConnectionError("dashd failed to start within 30 seconds. Check dashd installation.") - print(f"✓ dashd started successfully") + print("✓ dashd started successfully") return self.actual_port, self.temp_dir def _wait_for_ready(self, timeout: int = 30) -> bool: @@ -168,16 +174,12 @@ def _wait_for_ready(self, timeout: int = 30) -> bool: # Determine dash-cli path from dashd path dashd_path = Path(self.dashd_executable) if dashd_path.is_absolute(): - dashcli_path = str(dashd_path.parent / 'dash-cli') + dashcli_path = str(dashd_path.parent / "dash-cli") else: - dashcli_path = 'dash-cli' + dashcli_path = "dash-cli" # Create RPC client for this instance - rpc = DashRPCClient( - dashcli_path=dashcli_path, - datadir=str(self.temp_dir), - rpc_port=self.actual_port - ) + rpc = DashRPCClient(dashcli_path=dashcli_path, datadir=str(self.temp_dir), rpc_port=self.actual_port) start_time = time.time() last_error = None @@ -185,11 +187,17 @@ def _wait_for_ready(self, timeout: int = 30) -> bool: while time.time() - start_time < timeout: # Check if process died if self.process and self.process.poll() is not None: + # Read stderr for error details + if self.process.stderr: + stderr = self.process.stderr.read() + if stderr: + err_text = stderr.decode("utf-8", errors="replace").strip() + print(f" dashd exited with error: {err_text}") return False try: # Try to get block count - rpc.call('getblockcount') + rpc.call("getblockcount") return True except Exception as e: last_error = str(e) @@ -221,15 +229,3 @@ def stop(self): except Exception as e: print(f" Warning: Could not remove temp directory: {e}") self.temp_dir = None - - def get_port(self) -> int: - """Get the RPC port being used""" - if not self.actual_port: - raise DashdConnectionError("dashd not started yet") - return self.actual_port - - def get_temp_dir(self) -> Path: - """Get the temporary directory path""" - if not self.temp_dir: - raise DashdConnectionError("dashd not started yet") - return self.temp_dir diff --git a/generator/errors.py b/generator/errors.py index cb21b3a..cbfedec 100644 --- a/generator/errors.py +++ b/generator/errors.py @@ -7,6 +7,7 @@ class GeneratorError(Exception): """Base exception for all generator errors""" + pass @@ -24,26 +25,19 @@ class InsufficientFundsError(GeneratorError): This should never happen with proper UTXO management. """ + pass class ConfigError(GeneratorError): """Invalid configuration""" - pass - -class ValidationError(GeneratorError): - """Data validation failures""" - pass - - -class ExportError(GeneratorError): - """Export operation failures""" pass class DashdConnectionError(RPCError): """Cannot connect to dashd""" + pass diff --git a/generator/rpc_client.py b/generator/rpc_client.py index e4f7d96..061a663 100644 --- a/generator/rpc_client.py +++ b/generator/rpc_client.py @@ -2,11 +2,12 @@ Efficient RPC client with retry logic and error handling. """ -import subprocess import json +import subprocess import time -from typing import Any, List, Optional -from .errors import RPCError, DashdConnectionError, InsufficientFundsError +from typing import Any + +from .errors import DashdConnectionError, InsufficientFundsError, RPCError class DashRPCClient: @@ -19,11 +20,11 @@ class DashRPCClient: def __init__( self, dashcli_path: str = "dash-cli", - datadir: Optional[str] = None, + datadir: str | None = None, network: str = "regtest", - rpc_timeout: int = 30, + rpc_timeout: int = 120, max_retries: int = 3, - rpc_port: Optional[int] = None + rpc_port: int | None = None, ): self.dashcli = dashcli_path self.datadir = datadir @@ -32,7 +33,7 @@ def __init__( self.max_retries = max_retries self.rpc_port = rpc_port - def call(self, method: str, *params, wallet: Optional[str] = None) -> Any: + def call(self, method: str, *params, wallet: str | None = None) -> Any: """ Call RPC method with retry logic and comprehensive error handling. """ @@ -41,21 +42,14 @@ def call(self, method: str, *params, wallet: Optional[str] = None) -> Any: return self._execute(method, params, wallet) except subprocess.TimeoutExpired: if attempt == self.max_retries - 1: - raise RPCError( - f"RPC timeout after {self.rpc_timeout}s: {method}", - code=-1 - ) - time.sleep(2 ** attempt) - except ConnectionRefusedError as e: + raise RPCError(f"RPC timeout after {self.rpc_timeout}s: {method}", code=-1) from None + time.sleep(2**attempt) + except DashdConnectionError: if attempt == self.max_retries - 1: - raise DashdConnectionError( - f"Cannot connect to dashd: {e}" - ) - time.sleep(2 ** attempt) - - raise RPCError(f"Failed after {self.max_retries} retries: {method}") + raise + time.sleep(2**attempt) - def _execute(self, method: str, params: tuple, wallet: Optional[str]) -> Any: + def _execute(self, method: str, params: tuple, wallet: str | None) -> Any: """Execute single RPC call""" cmd = [self.dashcli, f"-{self.network}"] @@ -71,16 +65,13 @@ def _execute(self, method: str, params: tuple, wallet: Optional[str]) -> Any: cmd.append(method) for p in params: if isinstance(p, bool): - cmd.append('true' if p else 'false') + cmd.append("true" if p else "false") + elif isinstance(p, (dict, list)): + cmd.append(json.dumps(p)) else: cmd.append(str(p)) - result = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=self.rpc_timeout - ) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=self.rpc_timeout) if result.returncode != 0: self._handle_error(method, result.stderr) @@ -99,8 +90,7 @@ def _handle_error(self, method: str, stderr: str): if "error code: -6" in stderr_lower or "insufficient funds" in stderr_lower: raise InsufficientFundsError( - f"UTXO pool depleted during {method}. " - "This indicates a bug in UTXO management." + f"UTXO pool depleted during {method}. This indicates a bug in UTXO management." ) if "error code: -28" in stderr_lower: diff --git a/generator/wallet_export.py b/generator/wallet_export.py new file mode 100644 index 0000000..2df081f --- /dev/null +++ b/generator/wallet_export.py @@ -0,0 +1,83 @@ +"""Shared wallet statistics collection and export logic.""" + +import json +from pathlib import Path + +from .errors import RPCError +from .rpc_client import DashRPCClient + + +def collect_wallet_stats(rpc: DashRPCClient, wallet_name: str) -> dict: + """Collect transaction history, UTXOs, balance, and mnemonic for a wallet. + + Returns a dict with keys: wallet_name, mnemonic, transactions, utxos, balance. + """ + transactions = [] + try: + txs = rpc.call("listtransactions", "*", 999999999, 0, True, wallet=wallet_name) + for tx in txs: + transactions.append( + { + "txid": tx["txid"], + "address": tx.get("address", ""), + "amount": tx["amount"], + "confirmations": tx.get("confirmations", 0), + "blockhash": tx.get("blockhash", ""), + "time": tx.get("time", 0), + } + ) + except RPCError as e: + print(f" Warning: Error getting transactions for {wallet_name}: {e}") + + utxos = [] + balance = 0.0 + try: + wallet_utxos = rpc.call("listunspent", 1, 9999999, [], wallet=wallet_name) + utxos = [ + { + "txid": u["txid"], + "vout": u["vout"], + "address": u.get("address"), + "amount": u["amount"], + "confirmations": u.get("confirmations", 0), + } + for u in wallet_utxos + ] + balance = sum(u["amount"] for u in wallet_utxos) + except RPCError as e: + print(f" Warning: Error getting UTXOs for {wallet_name}: {e}") + + mnemonic = "" + try: + hd_info = rpc.call("dumphdinfo", wallet=wallet_name) + mnemonic = hd_info.get("mnemonic", "") + except RPCError: + pass + + return { + "wallet_name": wallet_name, + "mnemonic": mnemonic, + "transactions": transactions, + "utxos": utxos, + "balance": balance, + } + + +def save_wallet_file(wallet_data: dict, output_path: Path) -> None: + """Save wallet statistics to a JSON file. + + wallet_data should contain: wallet_name, mnemonic, balance, transactions, utxos. + """ + export_data = { + "wallet_name": wallet_data["wallet_name"], + "mnemonic": wallet_data.get("mnemonic", ""), + "balance": wallet_data["balance"], + "transaction_count": len(wallet_data["transactions"]), + "unique_transaction_count": len({tx["txid"] for tx in wallet_data["transactions"]}), + "utxo_count": len(wallet_data["utxos"]), + "transactions": wallet_data["transactions"], + "utxos": wallet_data["utxos"], + } + + with open(output_path, "w") as f: + json.dump(export_data, f, indent=2) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..ee0dd86 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,169 @@ +"""Tests for the WalletSyncGenerator and Config.""" + +import sys +from pathlib import Path + +import pytest + +# Add parent directory to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from generate import Config, Generator, WalletSyncGenerator + + +def create_test_config(**overrides): + """Create a Config with defaults for testing.""" + defaults = dict( + target_blocks=200, + dashcli_path="dash-cli", + dashd_executable="dashd", + auto_start_dashd=False, + dashd_datadir=None, + dashd_wallet="default", + rpc_port=None, + output_base="/tmp", + ) + defaults.update(overrides) + return Config(**defaults) + + +def create_wallet_sync_generator(**config_overrides): + """Create a WalletSyncGenerator instance for testing.""" + config = create_test_config(**config_overrides) + return WalletSyncGenerator(config) + + +# --- WalletSyncGenerator tests --- + + +class TestWalletSyncConfig: + """Test WalletSyncGenerator configuration.""" + + def test_wallet_name(self): + """Verify the test wallet is named 'wallet'.""" + assert WalletSyncGenerator.WALLET_NAME == "wallet" + + def test_address_count(self): + """Verify 50 addresses are pre-generated.""" + assert WalletSyncGenerator.NUM_ADDRESSES == 50 + + def test_strategy_name(self): + """Verify strategy name is 'wallet-sync'.""" + gen = create_wallet_sync_generator() + assert gen.strategy_name() == "wallet-sync" + + +class TestBatchBoundaryCalculation: + """Test batch boundary height calculation.""" + + def test_40k_blocks(self): + """Verify boundaries for 40000-block target.""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(200, 40000) + assert len(boundaries) == 8, f"Expected 8 boundaries, got {len(boundaries)}: {boundaries}" + assert boundaries[0] == 4999 + assert boundaries[-1] == 39999 + + def test_10k_blocks(self): + """Verify boundaries for 10000-block target.""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(200, 10000) + assert len(boundaries) == 2, f"Expected 2 boundaries, got {len(boundaries)}: {boundaries}" + assert boundaries[0] == 4999 + assert boundaries[1] == 9999 + + def test_1k_blocks(self): + """Verify no boundaries for 1000-block target (too short).""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(200, 1000) + assert len(boundaries) == 0, f"Expected 0 boundaries for 1000 blocks, got {len(boundaries)}" + + def test_boundaries_above_current_height(self): + """Verify all boundaries are above current height.""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(5500, 40000) + for b in boundaries: + assert b > 5500, f"Boundary {b} should be above current height 5500" + + def test_boundaries_below_target(self): + """Verify all boundaries are below target height.""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(200, 40000) + for b in boundaries: + assert b < 40000, f"Boundary {b} should be below target 40000" + + def test_boundary_spacing(self): + """Verify boundaries are spaced ~5000 apart.""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(200, 40000) + for i in range(1, len(boundaries)): + spacing = boundaries[i] - boundaries[i - 1] + assert spacing == 5000, f"Spacing between boundaries should be 5000, got {spacing}" + + def test_exact_boundary_start(self): + """Verify behavior when current height is exactly at a boundary.""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(5000, 15000) + assert len(boundaries) == 2, f"Expected 2 boundaries, got {len(boundaries)}: {boundaries}" + assert boundaries[0] == 9999 + assert boundaries[1] == 14999 + + def test_small_target_with_one_boundary(self): + """Verify a target that spans exactly one boundary.""" + boundaries = WalletSyncGenerator._calculate_batch_boundaries(200, 5500) + assert len(boundaries) == 1 + assert boundaries[0] == 4999 + + +class TestGeneratorBase: + """Test base Generator class.""" + + def test_wallet_sync_is_subclass(self): + """Verify WalletSyncGenerator is a Generator subclass.""" + gen = WalletSyncGenerator(create_test_config()) + assert gen.strategy_name() == "wallet-sync" + assert isinstance(gen, Generator) + + def test_base_generator_raises(self): + """Verify base Generator raises NotImplementedError for abstract methods.""" + gen = Generator(create_test_config()) + + with pytest.raises(NotImplementedError): + gen._load_addresses() + + with pytest.raises(NotImplementedError): + gen._initialize_utxo_pool() + + with pytest.raises(NotImplementedError): + gen._generate_blocks() + + +class TestConfigExtraArgs: + """Test extra_dashd_args in Config.""" + + def test_default_empty(self): + """Verify extra_dashd_args defaults to empty list.""" + config = create_test_config() + assert config.extra_dashd_args == [] + + def test_custom_args(self): + """Verify extra_dashd_args can be set.""" + config = create_test_config(extra_dashd_args=["-blockfilterindex=1"]) + assert config.extra_dashd_args == ["-blockfilterindex=1"] + + +class TestWalletSyncAddressIndices: + """Test that WalletSyncGenerator targets the right address indices.""" + + def test_wallet_addresses_dict(self): + """Verify wallet_addresses dict is initialized empty.""" + gen = create_wallet_sync_generator() + assert gen.wallet_addresses == {} + + def test_phase_coverage(self): + """Verify all phases exist as methods.""" + gen = create_wallet_sync_generator() + assert hasattr(gen, "_phase_normal_activity") + assert hasattr(gen, "_phase_gap_limit_boundary") + assert hasattr(gen, "_phase_beyond_gap_limit") + assert hasattr(gen, "_phase_transaction_variety") + assert hasattr(gen, "_phase_bulk_generation") + + +if __name__ == "__main__": + import pytest + + pytest.main([__file__, "-v"]) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..28659dc --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,192 @@ +"""Integration tests that exercise generate.py via the CLI. + +Requires the DASHD_PATH environment variable pointing to the dashd binary. +These tests invoke generate.py as a subprocess to verify the full CLI interface. +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.integration + +GENERATE_PY = str(Path(__file__).parent.parent / "generate.py") +PYTHON = sys.executable + + +def get_dashd_path(): + path = os.environ.get("DASHD_PATH", "") + assert path, "DASHD_PATH environment variable is not set (run contrib/setup-dashd.py first)" + dashd = Path(path) + assert dashd.is_file(), f"dashd binary not found at {path}" + assert os.access(path, os.X_OK), f"dashd binary not executable at {path}" + return path + + +def run_generate(*args, timeout=120): + """Run generate.py with the given arguments and return the CompletedProcess.""" + cmd = [PYTHON, GENERATE_PY, "--dashd-path", get_dashd_path(), *args] + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + + +# --- CLI argument validation --- + + +class TestCLIArguments: + """Test generate.py CLI argument parsing and validation.""" + + def test_help(self): + """Verify --help works and shows expected options.""" + result = subprocess.run([PYTHON, GENERATE_PY, "--help"], capture_output=True, text=True) + assert result.returncode == 0 + assert "--strategy" in result.stdout + assert "--blocks" in result.stdout + assert "--dashd-path" in result.stdout + assert "--output-dir" in result.stdout + assert "--no-auto-start" in result.stdout + assert "--rpc-port" in result.stdout + assert "--keep-temp" in result.stdout + + def test_blocks_below_minimum_rejected(self): + """Verify --blocks below 120 is rejected.""" + result = subprocess.run( + [PYTHON, GENERATE_PY, "--blocks", "50"], + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert "120" in result.stdout or "120" in result.stderr + + def test_blocks_at_minimum_accepted(self, tmp_path): + """Verify --blocks 120 is accepted and runs successfully.""" + result = run_generate("--blocks", "120", "--output-dir", str(tmp_path)) + assert result.returncode == 0, f"generate.py failed:\n{result.stderr}\n{result.stdout}" + + def test_invalid_dashd_path_rejected(self, tmp_path): + """Verify a nonexistent --dashd-path causes failure.""" + result = subprocess.run( + [ + PYTHON, + GENERATE_PY, + "--dashd-path", + "/nonexistent/dashd", + "--blocks", + "120", + "--output-dir", + str(tmp_path), + ], + capture_output=True, + text=True, + ) + assert result.returncode != 0 + + def test_output_dir_argument(self, tmp_path): + """Verify --output-dir controls where data is written.""" + result = run_generate("--blocks", "120", "--output-dir", str(tmp_path)) + assert result.returncode == 0, f"generate.py failed:\n{result.stderr}\n{result.stdout}" + + data_dir = tmp_path / "regtest-120" + assert data_dir.is_dir(), f"Output not written to specified --output-dir: {tmp_path}" + + +# --- End-to-end generation --- + + +@pytest.fixture(scope="module") +def generated_data(tmp_path_factory): + """Run generate.py once for all end-to-end tests in this module.""" + output_dir = tmp_path_factory.mktemp("integration") + result = run_generate("--blocks", "200", "--output-dir", str(output_dir)) + assert result.returncode == 0, f"generate.py failed:\n{result.stderr}\n{result.stdout}" + + data_dir = output_dir / "regtest-200" + return {"data_dir": data_dir, "output": result.stdout} + + +class TestWalletSyncGeneration: + """End-to-end test of the wallet-sync generation via generate.py CLI.""" + + def test_output_directory_created(self, generated_data): + """Verify the generator creates the expected output directory.""" + assert generated_data["data_dir"].is_dir() + + def test_completion_message(self, generated_data): + """Verify generate.py prints completion output.""" + output = generated_data["output"] + assert "Generation complete!" in output + assert "Blocks:" in output + + def test_regtest_blockchain_data(self, generated_data): + """Verify blockchain data files exist.""" + regtest_dir = generated_data["data_dir"] / "regtest" + assert regtest_dir.is_dir(), "regtest directory missing" + + blocks_dir = regtest_dir / "blocks" + assert blocks_dir.is_dir(), "blocks directory missing" + + chainstate_dir = regtest_dir / "chainstate" + assert chainstate_dir.is_dir(), "chainstate directory missing" + + block_files = list(blocks_dir.glob("blk*.dat")) + assert len(block_files) > 0, "No block data files found" + for bf in block_files: + assert bf.stat().st_size > 0, f"Block file is empty: {bf.name}" + + def test_wallet_json_files(self, generated_data): + """Verify wallet JSON files are created with valid structure.""" + wallets_dir = generated_data["data_dir"] / "wallets" + assert wallets_dir.is_dir(), "wallets directory missing" + + # wallet-sync strategy creates: default (faucet) and wallet (test) + for wallet_name in ["default", "wallet"]: + wallet_file = wallets_dir / f"{wallet_name}.json" + assert wallet_file.is_file(), f"Wallet file missing: {wallet_name}.json" + + with open(wallet_file) as f: + data = json.load(f) + + assert data["wallet_name"] == wallet_name + assert "balance" in data + assert isinstance(data["transactions"], list) + assert isinstance(data["utxos"], list) + assert "transaction_count" in data + assert "utxo_count" in data + + def test_wallet_mnemonic(self, generated_data): + """Verify the test wallet has an HD mnemonic.""" + wallet_file = generated_data["data_dir"] / "wallets" / "wallet.json" + with open(wallet_file) as f: + data = json.load(f) + + assert data.get("mnemonic"), "Test wallet should have a mnemonic" + words = data["mnemonic"].strip().split() + assert len(words) >= 12, f"Mnemonic too short: {len(words)} words" + + def test_wallet_has_transactions(self, generated_data): + """Verify the test wallet received transactions from the generator.""" + wallet_file = generated_data["data_dir"] / "wallets" / "wallet.json" + with open(wallet_file) as f: + data = json.load(f) + + assert data["transaction_count"] > 0, "Test wallet should have received transactions" + assert data["utxo_count"] > 0, "Test wallet should have UTXOs" + assert data["balance"] > 0, "Test wallet should have a positive balance" + + def test_faucet_has_balance(self, generated_data): + """Verify the faucet wallet has a positive balance from mining rewards.""" + wallet_file = generated_data["data_dir"] / "wallets" / "default.json" + with open(wallet_file) as f: + data = json.load(f) + + assert data["balance"] > 0, "Faucet wallet should have balance from mining" + assert data["utxo_count"] > 0, "Faucet wallet should have UTXOs" + + def test_wallet_directory_in_regtest(self, generated_data): + """Verify wallet directories are copied into the regtest data.""" + regtest_dir = generated_data["data_dir"] / "regtest" + wallet_dir = regtest_dir / "wallet" + assert wallet_dir.is_dir(), "Wallet directory 'wallet' not found in regtest data"