From 62946b285097d24c2a4a43264c7b1cd64862cb7d Mon Sep 17 00:00:00 2001 From: thunderbiscuit Date: Fri, 14 Apr 2023 07:03:06 -0400 Subject: [PATCH 1/5] Add compact block filters blockchain feature using the Nakamoto client --- Cargo.lock | 131 ++++++++++++++++++++++++++++++++++++++ bdk-ffi/src/bdk.udl | 7 ++ bdk-ffi/src/blockchain.rs | 29 +++++++++ 3 files changed, 167 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index d0eec568..9501d4e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,6 +131,7 @@ dependencies = [ "bip39", "bitcoin", "bitcoincore-rpc", + "cc", "electrum-client", "esplora-client", "getrandom", @@ -138,10 +139,12 @@ dependencies = [ "log", "miniscript", "rand", + "rocksdb", "rusqlite", "serde", "serde_json", "sled", + "socks", "tokio", ] @@ -180,6 +183,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.59.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bip39" version = "2.0.0" @@ -298,6 +320,18 @@ name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] [[package]] name = "cfg-if" @@ -305,6 +339,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "3.2.23" @@ -557,6 +602,15 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.61" @@ -578,12 +632,46 @@ dependencies = [ "serde_json", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "librocksdb-sys" +version = "6.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d" +dependencies = [ + "bindgen", + "cc", + "glob", + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.25.2" @@ -723,6 +811,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.2.0" @@ -834,6 +928,21 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "ring" version = "0.16.20" @@ -849,6 +958,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "rocksdb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61aa17a99a2413cd71c1106691bf59dad7de0cd5099127f90e9d99c429c40d4a" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rusqlite" version = "0.28.0" @@ -863,6 +982,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustls" version = "0.20.8" @@ -978,6 +1103,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "siphasher" version = "0.3.10" diff --git a/bdk-ffi/src/bdk.udl b/bdk-ffi/src/bdk.udl index 420dec32..1016b829 100644 --- a/bdk-ffi/src/bdk.udl +++ b/bdk-ffi/src/bdk.udl @@ -46,6 +46,7 @@ enum BdkError { "Sled", "Rusqlite", "Rpc", + "Cbf" }; dictionary AddressInfo { @@ -133,6 +134,11 @@ dictionary EsploraConfig { u64? timeout; }; +dictionary CBFConfig { + Network network; + string datadir; +}; + [Enum] interface Auth { None(); @@ -160,6 +166,7 @@ interface BlockchainConfig { Electrum(ElectrumConfig config); Esplora(EsploraConfig config); Rpc(RpcConfig config); + CBF(CBFConfig config); }; interface Blockchain { diff --git a/bdk-ffi/src/blockchain.rs b/bdk-ffi/src/blockchain.rs index 9d206c4a..8bd9b033 100644 --- a/bdk-ffi/src/blockchain.rs +++ b/bdk-ffi/src/blockchain.rs @@ -2,6 +2,7 @@ use crate::{BdkError, Transaction}; use bdk::bitcoin::Network; use bdk::blockchain::any::{AnyBlockchain, AnyBlockchainConfig}; +use bdk::blockchain::compact_filters::nakamoto::CBFBlockchainConfig; use bdk::blockchain::rpc::Auth as BdkAuth; use bdk::blockchain::rpc::RpcSyncParams as BdkRpcSyncParams; use bdk::blockchain::Blockchain as BdkBlockchain; @@ -49,6 +50,27 @@ impl Blockchain { wallet_name: config.wallet_name, sync_params: config.sync_params.map(|p| p.into()), }), + // TODO: not sure this is the right way to deal with the PathBuf + // Attempt 1, if you make datadir an Option in the CBFConfig struct + // BlockchainConfig::CBF { config } => { + // let path0: String = config + // .datadir + // .map(|d| d) + // .unwrap_or(String::from("./nakamoto/")); + // let path: PathBuf = PathBuf::from(path0); + // AnyBlockchainConfig::CompactFilters(CBFBlockchainConfig { + // network: config.network, + // datadir: Some(path), + // }) + // } + + // Attempt 2, if you make the datadir field mandatory + BlockchainConfig::CBF { config } => { + AnyBlockchainConfig::CompactFilters(CBFBlockchainConfig { + network: config.network, + datadir: Some(PathBuf::from(config.datadir)), + }) + } }; let blockchain = AnyBlockchain::from_config(&any_blockchain_config)?; Ok(Self { @@ -120,6 +142,11 @@ pub struct EsploraConfig { pub timeout: Option, } +pub struct CBFConfig { + network: Network, + datadir: String, +} + pub enum Auth { /// No authentication None, @@ -198,4 +225,6 @@ pub enum BlockchainConfig { Esplora { config: EsploraConfig }, /// Bitcoin Core RPC client Rpc { config: RpcConfig }, + /// Nakamoto client + CBF { config: CBFConfig }, } From 19d71c88e9158a6b490e4fa07f084490b5327411 Mon Sep 17 00:00:00 2001 From: andreasgriffin Date: Thu, 13 Apr 2023 16:29:13 +0200 Subject: [PATCH 2/5] generating bindings OK with compact filters --- bdk-ffi/src/bdk.udl | 18 +++++++++--------- bdk-ffi/src/blockchain.rs | 30 ++++++++++++++++++++++++++++-- bdk-ffi/src/lib.rs | 2 +- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/bdk-ffi/src/bdk.udl b/bdk-ffi/src/bdk.udl index 1016b829..f1dd9ecc 100644 --- a/bdk-ffi/src/bdk.udl +++ b/bdk-ffi/src/bdk.udl @@ -46,7 +46,7 @@ enum BdkError { "Sled", "Rusqlite", "Rpc", - "Cbf" + "CompactFilters", }; dictionary AddressInfo { @@ -134,11 +134,6 @@ dictionary EsploraConfig { u64? timeout; }; -dictionary CBFConfig { - Network network; - string datadir; -}; - [Enum] interface Auth { None(); @@ -161,12 +156,19 @@ dictionary RpcConfig { RpcSyncParams? sync_params; }; + +dictionary CompactFiltersConfig { + Network network; + string storage_dir; +}; + + [Enum] interface BlockchainConfig { Electrum(ElectrumConfig config); Esplora(EsploraConfig config); Rpc(RpcConfig config); - CBF(CBFConfig config); + Cbf(CompactFiltersConfig config); }; interface Blockchain { @@ -228,8 +230,6 @@ interface Wallet { [Throws=BdkError] constructor(Descriptor descriptor, Descriptor? change_descriptor, Network network, DatabaseConfig database_config); - Network network(); - [Throws=BdkError] AddressInfo get_address(AddressIndex address_index); diff --git a/bdk-ffi/src/blockchain.rs b/bdk-ffi/src/blockchain.rs index 8bd9b033..1934dc3b 100644 --- a/bdk-ffi/src/blockchain.rs +++ b/bdk-ffi/src/blockchain.rs @@ -11,6 +11,7 @@ use bdk::blockchain::GetHeight; use bdk::blockchain::{ electrum::ElectrumBlockchainConfig, esplora::EsploraBlockchainConfig, rpc::RpcConfig as BdkRpcConfig, ConfigurableBlockchain, + compact_filters::CompactFiltersBlockchainConfig, compact_filters::BitcoinPeerConfig, }; use bdk::FeeRate; use std::convert::{From, TryFrom}; @@ -43,6 +44,24 @@ impl Blockchain { timeout: config.timeout, }) } + BlockchainConfig::Cbf { config } => { + + let mut peers = Vec::new(); + let peer = BitcoinPeerConfig{ + address:"btcd-mainnet.lightning.computer:8333".to_string(), + socks5:None, + socks5_credentials:None + }; + peers.push(peer); + + + AnyBlockchainConfig::CompactFilters(CompactFiltersBlockchainConfig { + peers: peers, + network: config.network, + storage_dir: config.storage_dir, + skip_blocks : None, + }) + } BlockchainConfig::Rpc { config } => AnyBlockchainConfig::Rpc(BdkRpcConfig { url: config.url, auth: config.auth.into(), @@ -71,6 +90,7 @@ impl Blockchain { datadir: Some(PathBuf::from(config.datadir)), }) } + }) }; let blockchain = AnyBlockchain::from_config(&any_blockchain_config)?; Ok(Self { @@ -217,6 +237,12 @@ pub struct RpcConfig { pub sync_params: Option, } + +pub struct CompactFiltersConfig { + pub network: Network, + pub storage_dir: String, +} + /// Type that can contain any of the blockchain configurations defined by the library. pub enum BlockchainConfig { /// Electrum client @@ -225,6 +251,6 @@ pub enum BlockchainConfig { Esplora { config: EsploraConfig }, /// Bitcoin Core RPC client Rpc { config: RpcConfig }, - /// Nakamoto client - CBF { config: CBFConfig }, + /// CompactFilters + Cbf { config: CompactFiltersConfig }, } diff --git a/bdk-ffi/src/lib.rs b/bdk-ffi/src/lib.rs index 7854eb30..f680f237 100644 --- a/bdk-ffi/src/lib.rs +++ b/bdk-ffi/src/lib.rs @@ -6,7 +6,7 @@ mod psbt; mod wallet; use crate::blockchain::{ - Auth, Blockchain, BlockchainConfig, ElectrumConfig, EsploraConfig, RpcConfig, RpcSyncParams, + Auth, Blockchain, BlockchainConfig, ElectrumConfig, EsploraConfig, RpcConfig, CompactFiltersConfig, RpcSyncParams, }; use crate::database::DatabaseConfig; use crate::descriptor::Descriptor; From 6b6d154ce296136c9ab313b0d807ab474a2cf1f0 Mon Sep 17 00:00:00 2001 From: andreasgriffin Date: Thu, 13 Apr 2023 16:58:30 +0200 Subject: [PATCH 3/5] addres 1 address --- bdk-ffi/src/bdk.udl | 1 + bdk-ffi/src/blockchain.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bdk-ffi/src/bdk.udl b/bdk-ffi/src/bdk.udl index f1dd9ecc..92c5a2c6 100644 --- a/bdk-ffi/src/bdk.udl +++ b/bdk-ffi/src/bdk.udl @@ -158,6 +158,7 @@ dictionary RpcConfig { dictionary CompactFiltersConfig { + string address; Network network; string storage_dir; }; diff --git a/bdk-ffi/src/blockchain.rs b/bdk-ffi/src/blockchain.rs index 1934dc3b..b941d58d 100644 --- a/bdk-ffi/src/blockchain.rs +++ b/bdk-ffi/src/blockchain.rs @@ -48,7 +48,7 @@ impl Blockchain { let mut peers = Vec::new(); let peer = BitcoinPeerConfig{ - address:"btcd-mainnet.lightning.computer:8333".to_string(), + address:config.address, socks5:None, socks5_credentials:None }; @@ -239,6 +239,7 @@ pub struct RpcConfig { pub struct CompactFiltersConfig { + pub address: String, pub network: Network, pub storage_dir: String, } From 2c35b4abbcb5a0b4942c73d0c78d99da0d44a516 Mon Sep 17 00:00:00 2001 From: andreasgriffin Date: Thu, 13 Apr 2023 21:35:16 +0200 Subject: [PATCH 4/5] Add multiple peer addresses --- bdk-ffi/src/bdk.udl | 4 +++- bdk-ffi/src/blockchain.rs | 7 +------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bdk-ffi/src/bdk.udl b/bdk-ffi/src/bdk.udl index 92c5a2c6..33705d7f 100644 --- a/bdk-ffi/src/bdk.udl +++ b/bdk-ffi/src/bdk.udl @@ -157,8 +157,10 @@ dictionary RpcConfig { }; + + dictionary CompactFiltersConfig { - string address; + sequence addresses; Network network; string storage_dir; }; diff --git a/bdk-ffi/src/blockchain.rs b/bdk-ffi/src/blockchain.rs index b941d58d..442f5836 100644 --- a/bdk-ffi/src/blockchain.rs +++ b/bdk-ffi/src/blockchain.rs @@ -162,11 +162,6 @@ pub struct EsploraConfig { pub timeout: Option, } -pub struct CBFConfig { - network: Network, - datadir: String, -} - pub enum Auth { /// No authentication None, @@ -239,7 +234,7 @@ pub struct RpcConfig { pub struct CompactFiltersConfig { - pub address: String, + pub addresses: Vec, pub network: Network, pub storage_dir: String, } From 3a332976e346e9d3bf9798d983cc779c92abc123 Mon Sep 17 00:00:00 2001 From: andreasgriffin Date: Fri, 14 Apr 2023 07:17:29 +0200 Subject: [PATCH 5/5] Added skip_blocks --- bdk-ffi/src/bdk.udl | 1 + bdk-ffi/src/blockchain.rs | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/bdk-ffi/src/bdk.udl b/bdk-ffi/src/bdk.udl index 33705d7f..04273f65 100644 --- a/bdk-ffi/src/bdk.udl +++ b/bdk-ffi/src/bdk.udl @@ -163,6 +163,7 @@ dictionary CompactFiltersConfig { sequence addresses; Network network; string storage_dir; + u32 skip_blocks; }; diff --git a/bdk-ffi/src/blockchain.rs b/bdk-ffi/src/blockchain.rs index 442f5836..48ceac25 100644 --- a/bdk-ffi/src/blockchain.rs +++ b/bdk-ffi/src/blockchain.rs @@ -55,11 +55,16 @@ impl Blockchain { peers.push(peer); + AnyBlockchainConfig::CompactFilters(CompactFiltersBlockchainConfig { peers: peers, network: config.network, storage_dir: config.storage_dir, - skip_blocks : None, + skip_blocks : match usize::try_from(config.skip_blocks) { + Ok(value) => Some(value), + Err(_) => None, + } + , }) } BlockchainConfig::Rpc { config } => AnyBlockchainConfig::Rpc(BdkRpcConfig { @@ -237,6 +242,7 @@ pub struct CompactFiltersConfig { pub addresses: Vec, pub network: Network, pub storage_dir: String, + pub skip_blocks: u32, } /// Type that can contain any of the blockchain configurations defined by the library.