From 8fae53ad3c254bf02374b5bd7796cab9c1110654 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Mon, 20 Jan 2025 22:04:52 -0600 Subject: [PATCH 01/74] minimum code refactor for bitcoin RBF --- testutil/sample/crypto.go | 13 +- x/crosschain/types/cctx_test.go | 2 +- x/crosschain/types/revert_options_test.go | 2 +- .../chains/bitcoin/client/client_test.go | 4 +- zetaclient/chains/bitcoin/client/helpers.go | 43 ++ zetaclient/chains/bitcoin/client/mockgen.go | 1 + zetaclient/chains/bitcoin/common/fee.go | 65 ++- zetaclient/chains/bitcoin/common/fee_test.go | 71 ++- zetaclient/chains/bitcoin/observer/db.go | 69 +++ zetaclient/chains/bitcoin/observer/db_test.go | 120 +++++ .../chains/bitcoin/observer/event_test.go | 12 +- .../chains/bitcoin/observer/gas_price.go | 70 +++ .../chains/bitcoin/observer/inbound_test.go | 4 +- .../chains/bitcoin/observer/observer.go | 233 ++-------- .../chains/bitcoin/observer/observer_test.go | 98 ++-- .../chains/bitcoin/observer/outbound.go | 371 ++++++---------- .../chains/bitcoin/observer/outbound_test.go | 116 ++--- zetaclient/chains/bitcoin/observer/utxos.go | 199 +++++++++ .../chains/bitcoin/signer/outbound_data.go | 137 ++++++ .../bitcoin/signer/outbound_data_test.go | 241 ++++++++++ zetaclient/chains/bitcoin/signer/sign.go | 246 +++++++++++ zetaclient/chains/bitcoin/signer/sign_test.go | 326 ++++++++++++++ zetaclient/chains/bitcoin/signer/signer.go | 386 ++++------------ .../chains/bitcoin/signer/signer_test.go | 417 +++++++++--------- .../chains/evm/observer/observer_test.go | 43 -- zetaclient/chains/evm/rpc/rpc_live_test.go | 13 + zetaclient/chains/evm/signer/signer.go | 5 +- zetaclient/logs/fields.go | 1 + zetaclient/orchestrator/orchestrator.go | 36 +- zetaclient/orchestrator/v2_bootstrap.go | 1 - ...20c6d8465e4528fc0f3410b599dc0b26753a0.json | 95 ++++ ...20c6d8465e4528fc0f3410b599dc0b26753a0.json | 58 +++ zetaclient/testutils/mocks/bitcoin_client.go | 28 ++ .../testutils/mocks/zetacore_client_opts.go | 11 + zetaclient/testutils/testdata.go | 8 + zetaclient/testutils/testdata_naming.go | 5 + zetaclient/zetacore/broadcast.go | 10 +- zetaclient/zetacore/broadcast_test.go | 2 +- 38 files changed, 2337 insertions(+), 1225 deletions(-) create mode 100644 zetaclient/chains/bitcoin/observer/db.go create mode 100644 zetaclient/chains/bitcoin/observer/db_test.go create mode 100644 zetaclient/chains/bitcoin/observer/gas_price.go create mode 100644 zetaclient/chains/bitcoin/observer/utxos.go create mode 100644 zetaclient/chains/bitcoin/signer/outbound_data.go create mode 100644 zetaclient/chains/bitcoin/signer/outbound_data_test.go create mode 100644 zetaclient/chains/bitcoin/signer/sign.go create mode 100644 zetaclient/chains/bitcoin/signer/sign_test.go create mode 100644 zetaclient/testdata/btc/chain_8332_msgtx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json create mode 100644 zetaclient/testdata/btc/chain_8332_tx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json diff --git a/testutil/sample/crypto.go b/testutil/sample/crypto.go index 783ffa4a8d..6ebf295010 100644 --- a/testutil/sample/crypto.go +++ b/testutil/sample/crypto.go @@ -12,6 +12,7 @@ import ( "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" "github.com/cometbft/cometbft/crypto/secp256k1" "github.com/cosmos/cosmos-sdk/crypto/keys/ed25519" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" @@ -91,7 +92,7 @@ func EthAddressFromRand(r *rand.Rand) ethcommon.Address { } // BtcAddressP2WPKH returns a sample btc P2WPKH address -func BtcAddressP2WPKH(t *testing.T, net *chaincfg.Params) string { +func BtcAddressP2WPKH(t *testing.T, net *chaincfg.Params) *btcutil.AddressWitnessPubKeyHash { privateKey, err := btcec.NewPrivateKey() require.NoError(t, err) @@ -99,7 +100,15 @@ func BtcAddressP2WPKH(t *testing.T, net *chaincfg.Params) string { addr, err := btcutil.NewAddressWitnessPubKeyHash(pubKeyHash, net) require.NoError(t, err) - return addr.String() + return addr +} + +// BtcAddressP2WPKH returns a pkscript for a sample btc P2WPKH address +func BtcAddressP2WPKHScript(t *testing.T, net *chaincfg.Params) []byte { + addr := BtcAddressP2WPKH(t, net) + script, err := txscript.PayToAddrScript(addr) + require.NoError(t, err) + return script } // SolanaPrivateKey returns a sample solana private key diff --git a/x/crosschain/types/cctx_test.go b/x/crosschain/types/cctx_test.go index 1369e2dee0..1e2d6c830d 100644 --- a/x/crosschain/types/cctx_test.go +++ b/x/crosschain/types/cctx_test.go @@ -140,7 +140,7 @@ func Test_SetRevertOutboundValues(t *testing.T) { cctx := sample.CrossChainTx(t, "test") cctx.InboundParams.SenderChainId = chains.BitcoinTestnet.ChainId cctx.OutboundParams = cctx.OutboundParams[:1] - cctx.RevertOptions.RevertAddress = sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params) + cctx.RevertOptions.RevertAddress = sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params).String() err := cctx.AddRevertOutbound(100) require.NoError(t, err) diff --git a/x/crosschain/types/revert_options_test.go b/x/crosschain/types/revert_options_test.go index c91927dd86..3fa6a6e8ba 100644 --- a/x/crosschain/types/revert_options_test.go +++ b/x/crosschain/types/revert_options_test.go @@ -49,7 +49,7 @@ func TestRevertOptions_GetEVMRevertAddress(t *testing.T) { func TestRevertOptions_GetBTCRevertAddress(t *testing.T) { t.Run("valid Bitcoin revert address", func(t *testing.T) { - addr := sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params) + addr := sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params).String() actualAddr, valid := types.RevertOptions{ RevertAddress: addr, }.GetBTCRevertAddress(chains.BitcoinTestnet.ChainId) diff --git a/zetaclient/chains/bitcoin/client/client_test.go b/zetaclient/chains/bitcoin/client/client_test.go index d4b4ffd140..55091ebbef 100644 --- a/zetaclient/chains/bitcoin/client/client_test.go +++ b/zetaclient/chains/bitcoin/client/client_test.go @@ -137,7 +137,7 @@ func TestClientLive(t *testing.T) { require.NoError(t, err) require.Len(t, inbounds, 1) - require.Equal(t, inbounds[0].Value, 0.0001) + require.Equal(t, inbounds[0].Value+inbounds[0].DepositorFee, 0.0001) require.Equal(t, inbounds[0].ToAddress, "tb1qsa222mn2rhdq9cruxkz8p2teutvxuextx3ees2") // the text memo is base64 std encoded string:DSRR1RmDCwWmxqY201/TMtsJdmA= @@ -332,7 +332,7 @@ func TestClientLive(t *testing.T) { require.NoError(t, err) // go back whatever blocks as needed - endBlock := startBlock - 100 + endBlock := startBlock - 1 // loop through mempool.space blocks backwards for bn := startBlock; bn >= endBlock; { diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index d33050337b..2c4590e7ff 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "math/big" "time" types "github.com/btcsuite/btcd/btcjson" @@ -11,6 +12,17 @@ import ( "github.com/pkg/errors" ) +const ( + // FeeRateRegnet is the hardcoded fee rate for regnet + FeeRateRegnet = 1 + + // maxBTCSupply is the maximum supply of Bitcoin + maxBTCSupply = 21000000.0 + + // bytesPerKB is the number of vB in a KB + bytesPerKB = 1000 +) + // GetBlockVerboseByStr alias for GetBlockVerbose func (c *Client) GetBlockVerboseByStr(ctx context.Context, blockHash string) (*types.GetBlockVerboseTxResult, error) { h, err := strToHash(blockHash) @@ -104,6 +116,37 @@ func (c *Client) GetRawTransactionResult(ctx context.Context, } } +// FeeRateToSatPerByte converts a fee rate from BTC/KB to sat/vB. +func FeeRateToSatPerByte(rate float64) *big.Int { + // #nosec G115 always in range + satPerKB := new(big.Int).SetInt64(int64(rate * btcutil.SatoshiPerBitcoin)) + return new(big.Int).Div(satPerKB, big.NewInt(bytesPerKB)) +} + +// GetEstimatedFeeRate gets estimated smart fee rate (BTC/Kb) targeting given block confirmation +func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) { + // RPC 'EstimateSmartFee' is not available in regnet + if regnet { + return FeeRateRegnet, nil + } + + feeResult, err := c.EstimateSmartFee(ctx, confTarget, &types.EstimateModeEconomical) + if err != nil { + return 0, errors.Wrap(err, "unable to estimate smart fee") + } + if feeResult.Errors != nil { + return 0, fmt.Errorf("fee result contains errors: %s", feeResult.Errors) + } + if feeResult.FeeRate == nil { + return 0, fmt.Errorf("nil fee rate") + } + if *feeResult.FeeRate <= 0 || *feeResult.FeeRate >= maxBTCSupply { + return 0, fmt.Errorf("invalid fee rate: %f", *feeResult.FeeRate) + } + + return FeeRateToSatPerByte(*feeResult.FeeRate).Int64(), nil +} + // GetTransactionFeeAndRate gets the transaction fee and rate for a given tx result func (c *Client) GetTransactionFeeAndRate(ctx context.Context, rawResult *types.TxRawResult) (int64, int64, error) { var ( diff --git a/zetaclient/chains/bitcoin/client/mockgen.go b/zetaclient/chains/bitcoin/client/mockgen.go index 8200cf20b3..69cfe1cd6b 100644 --- a/zetaclient/chains/bitcoin/client/mockgen.go +++ b/zetaclient/chains/bitcoin/client/mockgen.go @@ -45,6 +45,7 @@ type client interface { SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*hash.Hash, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) GetTransactionFeeAndRate(ctx context.Context, tx *types.TxRawResult) (int64, int64, error) EstimateSmartFee( ctx context.Context, diff --git a/zetaclient/chains/bitcoin/common/fee.go b/zetaclient/chains/bitcoin/common/fee.go index e77de1c6b9..f7ae0559fd 100644 --- a/zetaclient/chains/bitcoin/common/fee.go +++ b/zetaclient/chains/bitcoin/common/fee.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "fmt" "math" - "math/big" "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/btcjson" @@ -20,19 +19,17 @@ import ( const ( // constants related to transaction size calculations - bytesPerKB = 1000 - bytesPerInput = 41 // each input is 41 bytes - bytesPerOutputP2TR = 43 // each P2TR output is 43 bytes - bytesPerOutputP2WSH = 43 // each P2WSH output is 43 bytes - bytesPerOutputP2WPKH = 31 // each P2WPKH output is 31 bytes - bytesPerOutputP2SH = 32 // each P2SH output is 32 bytes - bytesPerOutputP2PKH = 34 // each P2PKH output is 34 bytes - bytesPerOutputAvg = 37 // average size of all above types of outputs (36.6 bytes) - bytes1stWitness = 110 // the 1st witness incurs about 110 bytes and it may vary - bytesPerWitness = 108 // each additional witness incurs about 108 bytes and it may vary - OutboundBytesMin = uint64(239) // 239vB == EstimateSegWitTxSize(2, 2, toP2WPKH) - OutboundBytesMax = uint64(1543) // 1543v == EstimateSegWitTxSize(21, 2, toP2TR) - OutboundBytesAvg = uint64(245) // 245vB is a suggested gas limit for zetacore + bytesPerInput = 41 // each input is 41 bytes + bytesPerOutputP2TR = 43 // each P2TR output is 43 bytes + bytesPerOutputP2WSH = 43 // each P2WSH output is 43 bytes + bytesPerOutputP2WPKH = 31 // each P2WPKH output is 31 bytes + bytesPerOutputP2SH = 32 // each P2SH output is 32 bytes + bytesPerOutputP2PKH = 34 // each P2PKH output is 34 bytes + bytesPerOutputAvg = 37 // average size of all above types of outputs (36.6 bytes) + bytes1stWitness = 110 // the 1st witness incurs about 110 bytes and it may vary + bytesPerWitness = 108 // each additional witness incurs about 108 bytes and it may vary + OutboundBytesMin = int64(239) // 239vB == EstimateOutboundSize(2, 2, toP2WPKH) + OutboundBytesMax = int64(1543) // 1543v == EstimateOutboundSize(21, 2, toP2TR) // defaultDepositorFeeRate is the default fee rate for depositor fee, 20 sat/vB defaultDepositorFeeRate = 20 @@ -49,6 +46,7 @@ var ( BtcOutboundBytesDepositor = OutboundSizeDepositor() // BtcOutboundBytesWithdrawer is the outbound size incurred by the withdrawer: 177vB + // This will be the suggested gas limit used for zetacore BtcOutboundBytesWithdrawer = OutboundSizeWithdrawer() // DefaultDepositorFee is the default depositor fee is 0.00001360 BTC (20 * 68vB / 100000000) @@ -67,34 +65,28 @@ type RPC interface { // DepositorFeeCalculator is a function type to calculate the Bitcoin depositor fee type DepositorFeeCalculator func(context.Context, RPC, *btcjson.TxRawResult, *chaincfg.Params) (float64, error) -// FeeRateToSatPerByte converts a fee rate in BTC/KB to sat/byte. -func FeeRateToSatPerByte(rate float64) *big.Int { - // #nosec G115 always in range - satPerKB := new(big.Int).SetInt64(int64(rate * btcutil.SatoshiPerBitcoin)) - return new(big.Int).Div(satPerKB, big.NewInt(bytesPerKB)) -} - // WiredTxSize calculates the wired tx size in bytes -func WiredTxSize(numInputs uint64, numOutputs uint64) uint64 { +func WiredTxSize(numInputs uint64, numOutputs uint64) int64 { // Version 4 bytes + LockTime 4 bytes + Serialized varint size for the // number of transaction inputs and outputs. // #nosec G115 always positive - return uint64(8 + wire.VarIntSerializeSize(numInputs) + wire.VarIntSerializeSize(numOutputs)) + return int64(8 + wire.VarIntSerializeSize(numInputs) + wire.VarIntSerializeSize(numOutputs)) } // EstimateOutboundSize estimates the size of an outbound in vBytes -func EstimateOutboundSize(numInputs uint64, payees []btcutil.Address) (uint64, error) { - if numInputs == 0 { +func EstimateOutboundSize(numInputs int64, payees []btcutil.Address) (int64, error) { + if numInputs <= 0 { return 0, nil } // #nosec G115 always positive numOutputs := 2 + uint64(len(payees)) - bytesWiredTx := WiredTxSize(numInputs, numOutputs) + // #nosec G115 checked positive + bytesWiredTx := WiredTxSize(uint64(numInputs), numOutputs) bytesInput := numInputs * bytesPerInput - bytesOutput := uint64(2) * bytesPerOutputP2WPKH // new nonce mark, change + bytesOutput := int64(2) * bytesPerOutputP2WPKH // new nonce mark, change // calculate the size of the outputs to payees - bytesToPayees := uint64(0) + bytesToPayees := int64(0) for _, to := range payees { sizeOutput, err := GetOutputSizeByAddress(to) if err != nil { @@ -112,7 +104,7 @@ func EstimateOutboundSize(numInputs uint64, payees []btcutil.Address) (uint64, e } // GetOutputSizeByAddress returns the size of a tx output in bytes by the given address -func GetOutputSizeByAddress(to btcutil.Address) (uint64, error) { +func GetOutputSizeByAddress(to btcutil.Address) (int64, error) { switch addr := to.(type) { case *btcutil.AddressTaproot: if addr == nil { @@ -145,16 +137,16 @@ func GetOutputSizeByAddress(to btcutil.Address) (uint64, error) { } // OutboundSizeDepositor returns outbound size (68vB) incurred by the depositor -func OutboundSizeDepositor() uint64 { +func OutboundSizeDepositor() int64 { return bytesPerInput + bytesPerWitness/blockchain.WitnessScaleFactor } // OutboundSizeWithdrawer returns outbound size (177vB) incurred by the withdrawer (1 input, 3 outputs) -func OutboundSizeWithdrawer() uint64 { +func OutboundSizeWithdrawer() int64 { bytesWiredTx := WiredTxSize(1, 3) - bytesInput := uint64(1) * bytesPerInput // nonce mark - bytesOutput := uint64(2) * bytesPerOutputP2WPKH // 2 P2WPKH outputs: new nonce mark, change - bytesOutput += bytesPerOutputAvg // 1 output to withdrawer's address + bytesInput := int64(1) * bytesPerInput // nonce mark + bytesOutput := int64(2) * bytesPerOutputP2WPKH // 2 P2WPKH outputs: new nonce mark, change + bytesOutput += bytesPerOutputAvg // 1 output to withdrawer's address return bytesWiredTx + bytesInput + bytesOutput + bytes1stWitness/blockchain.WitnessScaleFactor } @@ -255,7 +247,7 @@ func CalcDepositorFee( // GetRecentFeeRate gets the highest fee rate from recent blocks // Note: this method should be used for testnet ONLY -func GetRecentFeeRate(ctx context.Context, rpc RPC, netParams *chaincfg.Params) (uint64, error) { +func GetRecentFeeRate(ctx context.Context, rpc RPC, netParams *chaincfg.Params) (int64, error) { // should avoid using this method for mainnet if netParams.Name == chaincfg.MainNetParams.Name { return 0, errors.New("GetRecentFeeRate should not be used for mainnet") @@ -295,6 +287,5 @@ func GetRecentFeeRate(ctx context.Context, rpc RPC, netParams *chaincfg.Params) highestRate = defaultTestnetFeeRate } - // #nosec G115 always in range - return uint64(highestRate), nil + return highestRate, nil } diff --git a/zetaclient/chains/bitcoin/common/fee_test.go b/zetaclient/chains/bitcoin/common/fee_test.go index 8967c86cfc..e73e4150b9 100644 --- a/zetaclient/chains/bitcoin/common/fee_test.go +++ b/zetaclient/chains/bitcoin/common/fee_test.go @@ -183,6 +183,11 @@ func TestOutboundSize2In3Out(t *testing.T) { privateKey, _, payerScript := generateKeyPair(t, &chaincfg.TestNet3Params) _, payee, payeeScript := generateKeyPair(t, &chaincfg.TestNet3Params) + // return 0 vByte if no UTXO + vBytesEstimated, err := EstimateOutboundSize(0, []btcutil.Address{payee}) + require.NoError(t, err) + require.Zero(t, vBytesEstimated) + // 2 example UTXO txids to use in the test. utxosTxids := exampleTxids[:2] @@ -193,10 +198,9 @@ func TestOutboundSize2In3Out(t *testing.T) { addTxInputsOutputsAndSignTx(t, tx, privateKey, payerScript, utxosTxids, [][]byte{payeeScript}) // Estimate the tx size in vByte - // #nosec G115 always positive - vError := uint64(1) // 1 vByte error tolerance - vBytes := uint64(blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor) - vBytesEstimated, err := EstimateOutboundSize(uint64(len(utxosTxids)), []btcutil.Address{payee}) + vError := int64(1) // 1 vByte error tolerance + vBytes := blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor + vBytesEstimated, err = EstimateOutboundSize(int64(len(utxosTxids)), []btcutil.Address{payee}) require.NoError(t, err) if vBytes > vBytesEstimated { require.True(t, vBytes-vBytesEstimated <= vError) @@ -218,9 +222,9 @@ func TestOutboundSize21In3Out(t *testing.T) { // Estimate the tx size in vByte // #nosec G115 always positive - vError := uint64(21 / 4) // 5 vBytes error tolerance - vBytes := uint64(blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor) - vBytesEstimated, err := EstimateOutboundSize(uint64(len(exampleTxids)), []btcutil.Address{payee}) + vError := int64(21 / 4) // 5 vBytes error tolerance + vBytes := blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor + vBytesEstimated, err := EstimateOutboundSize(int64(len(exampleTxids)), []btcutil.Address{payee}) require.NoError(t, err) if vBytes > vBytesEstimated { require.True(t, vBytes-vBytesEstimated <= vError) @@ -242,11 +246,11 @@ func TestOutboundSizeXIn3Out(t *testing.T) { // Estimate the tx size // #nosec G115 always positive - vError := uint64( + vError := int64( 0.25 + float64(x)/4, ) // 1st witness incurs 0.25 more vByte error than others (which incurs 1/4 vByte per witness) - vBytes := uint64(blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor) - vBytesEstimated, err := EstimateOutboundSize(uint64(len(exampleTxids[:x])), []btcutil.Address{payee}) + vBytes := blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor + vBytesEstimated, err := EstimateOutboundSize(int64(len(exampleTxids[:x])), []btcutil.Address{payee}) require.NoError(t, err) if vBytes > vBytesEstimated { require.True(t, vBytes-vBytesEstimated <= vError) @@ -263,62 +267,62 @@ func TestGetOutputSizeByAddress(t *testing.T) { nilP2TR := (*btcutil.AddressTaproot)(nil) sizeNilP2TR, err := GetOutputSizeByAddress(nilP2TR) require.NoError(t, err) - require.Equal(t, uint64(0), sizeNilP2TR) + require.Zero(t, sizeNilP2TR) addrP2TR := getTestAddrScript(t, ScriptTypeP2TR) sizeP2TR, err := GetOutputSizeByAddress(addrP2TR) require.NoError(t, err) - require.Equal(t, uint64(bytesPerOutputP2TR), sizeP2TR) + require.Equal(t, int64(bytesPerOutputP2TR), sizeP2TR) // test nil P2WSH address and non-nil P2WSH address nilP2WSH := (*btcutil.AddressWitnessScriptHash)(nil) sizeNilP2WSH, err := GetOutputSizeByAddress(nilP2WSH) require.NoError(t, err) - require.Equal(t, uint64(0), sizeNilP2WSH) + require.Zero(t, sizeNilP2WSH) addrP2WSH := getTestAddrScript(t, ScriptTypeP2WSH) sizeP2WSH, err := GetOutputSizeByAddress(addrP2WSH) require.NoError(t, err) - require.Equal(t, uint64(bytesPerOutputP2WSH), sizeP2WSH) + require.Equal(t, int64(bytesPerOutputP2WSH), sizeP2WSH) // test nil P2WPKH address and non-nil P2WPKH address nilP2WPKH := (*btcutil.AddressWitnessPubKeyHash)(nil) sizeNilP2WPKH, err := GetOutputSizeByAddress(nilP2WPKH) require.NoError(t, err) - require.Equal(t, uint64(0), sizeNilP2WPKH) + require.Zero(t, sizeNilP2WPKH) addrP2WPKH := getTestAddrScript(t, ScriptTypeP2WPKH) sizeP2WPKH, err := GetOutputSizeByAddress(addrP2WPKH) require.NoError(t, err) - require.Equal(t, uint64(bytesPerOutputP2WPKH), sizeP2WPKH) + require.Equal(t, int64(bytesPerOutputP2WPKH), sizeP2WPKH) // test nil P2SH address and non-nil P2SH address nilP2SH := (*btcutil.AddressScriptHash)(nil) sizeNilP2SH, err := GetOutputSizeByAddress(nilP2SH) require.NoError(t, err) - require.Equal(t, uint64(0), sizeNilP2SH) + require.Zero(t, sizeNilP2SH) addrP2SH := getTestAddrScript(t, ScriptTypeP2SH) sizeP2SH, err := GetOutputSizeByAddress(addrP2SH) require.NoError(t, err) - require.Equal(t, uint64(bytesPerOutputP2SH), sizeP2SH) + require.Equal(t, int64(bytesPerOutputP2SH), sizeP2SH) // test nil P2PKH address and non-nil P2PKH address nilP2PKH := (*btcutil.AddressPubKeyHash)(nil) sizeNilP2PKH, err := GetOutputSizeByAddress(nilP2PKH) require.NoError(t, err) - require.Equal(t, uint64(0), sizeNilP2PKH) + require.Zero(t, sizeNilP2PKH) addrP2PKH := getTestAddrScript(t, ScriptTypeP2PKH) sizeP2PKH, err := GetOutputSizeByAddress(addrP2PKH) require.NoError(t, err) - require.Equal(t, uint64(bytesPerOutputP2PKH), sizeP2PKH) + require.Equal(t, int64(bytesPerOutputP2PKH), sizeP2PKH) // test unsupported address type nilP2PK := (*btcutil.AddressPubKey)(nil) sizeP2PK, err := GetOutputSizeByAddress(nilP2PK) require.ErrorContains(t, err, "cannot get output size for address type") - require.Equal(t, uint64(0), sizeP2PK) + require.Zero(t, sizeP2PK) } func TestOutputSizeP2TR(t *testing.T) { @@ -334,8 +338,7 @@ func TestOutputSizeP2TR(t *testing.T) { addTxInputsOutputsAndSignTx(t, tx, privateKey, payerScript, exampleTxids[:2], payeeScripts) // Estimate the tx size in vByte - // #nosec G115 always positive - vBytes := uint64(blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor) + vBytes := blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor vBytesEstimated, err := EstimateOutboundSize(2, payees) require.NoError(t, err) require.Equal(t, vBytes, vBytesEstimated) @@ -354,8 +357,7 @@ func TestOutputSizeP2WSH(t *testing.T) { addTxInputsOutputsAndSignTx(t, tx, privateKey, payerScript, exampleTxids[:2], payeeScripts) // Estimate the tx size in vByte - // #nosec G115 always positive - vBytes := uint64(blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor) + vBytes := blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor vBytesEstimated, err := EstimateOutboundSize(2, payees) require.NoError(t, err) require.Equal(t, vBytes, vBytesEstimated) @@ -374,8 +376,7 @@ func TestOutputSizeP2SH(t *testing.T) { addTxInputsOutputsAndSignTx(t, tx, privateKey, payerScript, exampleTxids[:2], payeeScripts) // Estimate the tx size in vByte - // #nosec G115 always positive - vBytes := uint64(blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor) + vBytes := blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor vBytesEstimated, err := EstimateOutboundSize(2, payees) require.NoError(t, err) require.Equal(t, vBytes, vBytesEstimated) @@ -394,8 +395,7 @@ func TestOutputSizeP2PKH(t *testing.T) { addTxInputsOutputsAndSignTx(t, tx, privateKey, payerScript, exampleTxids[:2], payeeScripts) // Estimate the tx size in vByte - // #nosec G115 always positive - vBytes := uint64(blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor) + vBytes := blockchain.GetTransactionWeight(btcutil.NewTx(tx)) / blockchain.WitnessScaleFactor vBytesEstimated, err := EstimateOutboundSize(2, payees) require.NoError(t, err) require.Equal(t, vBytes, vBytesEstimated) @@ -412,27 +412,26 @@ func TestOutboundSizeBreakdown(t *testing.T) { } // add all outbound sizes paying to each address - txSizeTotal := uint64(0) + txSizeTotal := int64(0) for _, payee := range payees { sizeOutput, err := EstimateOutboundSize(2, []btcutil.Address{payee}) require.NoError(t, err) txSizeTotal += sizeOutput } - // calculate the average outbound size + // calculate the average outbound size (245 vByte) // #nosec G115 always in range - txSizeAverage := uint64((float64(txSizeTotal))/float64(len(payees)) + 0.5) + txSizeAverage := int64((float64(txSizeTotal))/float64(len(payees)) + 0.5) // get deposit fee txSizeDepositor := OutboundSizeDepositor() - require.Equal(t, uint64(68), txSizeDepositor) + require.Equal(t, int64(68), txSizeDepositor) // get withdrawer fee txSizeWithdrawer := OutboundSizeWithdrawer() - require.Equal(t, uint64(177), txSizeWithdrawer) + require.Equal(t, int64(177), txSizeWithdrawer) // total outbound size == (deposit fee + withdrawer fee), 245 = 68 + 177 - require.Equal(t, OutboundBytesAvg, txSizeAverage) require.Equal(t, txSizeAverage, txSizeDepositor+txSizeWithdrawer) // check default depositor fee @@ -459,5 +458,5 @@ func TestOutboundSizeMinMaxError(t *testing.T) { nilP2PK := (*btcutil.AddressPubKey)(nil) size, err := EstimateOutboundSize(1, []btcutil.Address{nilP2PK}) require.Error(t, err) - require.Equal(t, uint64(0), size) + require.Zero(t, size) } diff --git a/zetaclient/chains/bitcoin/observer/db.go b/zetaclient/chains/bitcoin/observer/db.go new file mode 100644 index 0000000000..b36a58d4a4 --- /dev/null +++ b/zetaclient/chains/bitcoin/observer/db.go @@ -0,0 +1,69 @@ +package observer + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/zeta-chain/node/pkg/chains" + clienttypes "github.com/zeta-chain/node/zetaclient/types" +) + +// SaveBroadcastedTx saves successfully broadcasted transaction +func (ob *Observer) SaveBroadcastedTx(txHash string, nonce uint64) { + outboundID := ob.OutboundID(nonce) + ob.Mu().Lock() + ob.tssOutboundHashes[txHash] = true + ob.broadcastedTx[outboundID] = txHash + ob.Mu().Unlock() + + broadcastEntry := clienttypes.ToOutboundHashSQLType(txHash, outboundID) + if err := ob.DB().Client().Save(&broadcastEntry).Error; err != nil { + ob.logger.Outbound.Error(). + Err(err). + Msgf("SaveBroadcastedTx: error saving broadcasted txHash %s for outbound %s", txHash, outboundID) + } + ob.logger.Outbound.Info().Msgf("SaveBroadcastedTx: saved broadcasted txHash %s for outbound %s", txHash, outboundID) +} + +// LoadLastBlockScanned loads the last scanned block from the database +func (ob *Observer) LoadLastBlockScanned(ctx context.Context) error { + err := ob.Observer.LoadLastBlockScanned(ob.Logger().Chain) + if err != nil { + return errors.Wrapf(err, "error LoadLastBlockScanned for chain %d", ob.Chain().ChainId) + } + + // observer will scan from the last block when 'lastBlockScanned == 0', this happens when: + // 1. environment variable is set explicitly to "latest" + // 2. environment variable is empty and last scanned block is not found in DB + if ob.LastBlockScanned() == 0 { + blockNumber, err := ob.rpc.GetBlockCount(ctx) + if err != nil { + return errors.Wrapf(err, "error GetBlockCount for chain %d", ob.Chain().ChainId) + } + // #nosec G115 always positive + ob.WithLastBlockScanned(uint64(blockNumber)) + } + + // bitcoin regtest starts from hardcoded block 100 + if chains.IsBitcoinRegnet(ob.Chain().ChainId) { + ob.WithLastBlockScanned(RegnetStartBlock) + } + ob.Logger().Chain.Info().Msgf("chain %d starts scanning from block %d", ob.Chain().ChainId, ob.LastBlockScanned()) + + return nil +} + +// LoadBroadcastedTxMap loads broadcasted transactions from the database +func (ob *Observer) LoadBroadcastedTxMap() error { + var broadcastedTransactions []clienttypes.OutboundHashSQLType + if err := ob.DB().Client().Find(&broadcastedTransactions).Error; err != nil { + ob.logger.Chain.Error().Err(err).Msgf("error iterating over db for chain %d", ob.Chain().ChainId) + return err + } + for _, entry := range broadcastedTransactions { + ob.tssOutboundHashes[entry.Hash] = true + ob.broadcastedTx[entry.Key] = entry.Hash + } + return nil +} diff --git a/zetaclient/chains/bitcoin/observer/db_test.go b/zetaclient/chains/bitcoin/observer/db_test.go new file mode 100644 index 0000000000..1a126b7d68 --- /dev/null +++ b/zetaclient/chains/bitcoin/observer/db_test.go @@ -0,0 +1,120 @@ +package observer_test + +import ( + "context" + "os" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/testutil/sample" + "github.com/zeta-chain/node/zetaclient/chains/base" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" +) + +func Test_SaveBroadcastedTx(t *testing.T) { + t.Run("should be able to save broadcasted tx", func(t *testing.T) { + // test data + nonce := uint64(1) + txHash := sample.BtcHash().String() + + // create observer and open db + ob := newTestSuite(t, chains.BitcoinMainnet, "") + + // save a test tx + ob.SaveBroadcastedTx(txHash, nonce) + + // check if the txHash is a TSS outbound + require.True(t, ob.IsTSSTransaction(txHash)) + + // get the broadcasted tx + gotHash, found := ob.GetBroadcastedTx(nonce) + require.True(t, found) + require.Equal(t, txHash, gotHash) + }) +} + +func Test_LoadLastBlockScanned(t *testing.T) { + ctx := context.Background() + + // use Bitcoin mainnet chain for testing + chain := chains.BitcoinMainnet + + t.Run("should load last block scanned", func(t *testing.T) { + // create observer and write 199 as last block scanned + ob := newTestSuite(t, chain, "") + ob.WriteLastBlockScannedToDB(199) + + // load last block scanned + err := ob.LoadLastBlockScanned(ctx) + require.NoError(t, err) + require.EqualValues(t, 199, ob.LastBlockScanned()) + }) + t.Run("should fail on invalid env var", func(t *testing.T) { + // create observer + ob := newTestSuite(t, chain, "") + + // set invalid environment variable + envvar := base.EnvVarLatestBlockByChain(chain) + os.Setenv(envvar, "invalid") + defer os.Unsetenv(envvar) + + // load last block scanned + err := ob.LoadLastBlockScanned(ctx) + require.ErrorContains(t, err, "error LoadLastBlockScanned") + }) + t.Run("should fail on RPC error", func(t *testing.T) { + // create observer on separate path, as we need to reset last block scanned + obOther := newTestSuite(t, chain, "") + + // reset last block scanned to 0 so that it will be loaded from RPC + obOther.WithLastBlockScanned(0) + + // attach a mock btc client that returns rpc error + obOther.client.ExpectedCalls = nil + obOther.client.On("GetBlockCount", mock.Anything).Return(int64(0), errors.New("rpc error")) + + // load last block scanned + err := obOther.LoadLastBlockScanned(ctx) + require.ErrorContains(t, err, "rpc error") + }) + t.Run("should use hardcode block 100 for regtest", func(t *testing.T) { + // use regtest chain + obRegnet := newTestSuite(t, chains.BitcoinRegtest, "") + + // load last block scanned + err := obRegnet.LoadLastBlockScanned(ctx) + require.NoError(t, err) + require.EqualValues(t, observer.RegnetStartBlock, obRegnet.LastBlockScanned()) + }) +} + +func Test_LoadBroadcastedTxMap(t *testing.T) { + t.Run("should load broadcasted tx map", func(t *testing.T) { + // test data + nonce := uint64(1) + txHash := sample.BtcHash().String() + + // create observer and save a test tx + dbPath := sample.CreateTempDir(t) + obOld := newTestSuite(t, chains.BitcoinMainnet, dbPath) + obOld.SaveBroadcastedTx(txHash, nonce) + + // create new observer using same db path + obNew := newTestSuite(t, chains.BitcoinMainnet, dbPath) + + // load broadcasted tx map to new observer + err := obNew.LoadBroadcastedTxMap() + require.NoError(t, err) + + // check if the txHash is a TSS outbound + require.True(t, obNew.IsTSSTransaction(txHash)) + + // get the broadcasted tx + gotHash, found := obNew.GetBroadcastedTx(nonce) + require.True(t, found) + require.Equal(t, txHash, gotHash) + }) +} diff --git a/zetaclient/chains/bitcoin/observer/event_test.go b/zetaclient/chains/bitcoin/observer/event_test.go index ab78269527..ca8d79e155 100644 --- a/zetaclient/chains/bitcoin/observer/event_test.go +++ b/zetaclient/chains/bitcoin/observer/event_test.go @@ -32,7 +32,7 @@ func createTestBtcEvent( memoStd *memo.InboundMemo, ) observer.BTCInboundEvent { return observer.BTCInboundEvent{ - FromAddress: sample.BtcAddressP2WPKH(t, net), + FromAddress: sample.BtcAddressP2WPKH(t, net).String(), ToAddress: sample.EthAddress().Hex(), MemoBytes: memo, MemoStd: memoStd, @@ -249,7 +249,7 @@ func Test_ValidateStandardMemo(t *testing.T) { }, FieldsV0: memo.FieldsV0{ RevertOptions: crosschaintypes.RevertOptions{ - RevertAddress: sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params), + RevertAddress: sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params).String(), }, }, }, @@ -306,7 +306,7 @@ func Test_IsEventProcessable(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain) + ob := newTestSuite(t, chain, "") // setup compliance config cfg := config.Config{ @@ -354,7 +354,7 @@ func Test_NewInboundVoteFromLegacyMemo(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain) + ob := newTestSuite(t, chain, "") ob.zetacore.WithKeys(&keys.Keys{}).WithZetaChain() t.Run("should create new inbound vote msg V1", func(t *testing.T) { @@ -394,13 +394,13 @@ func Test_NewInboundVoteFromStdMemo(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain) + ob := newTestSuite(t, chain, "") ob.zetacore.WithKeys(&keys.Keys{}).WithZetaChain() t.Run("should create new inbound vote msg with standard memo", func(t *testing.T) { // create revert options revertOptions := crosschaintypes.NewEmptyRevertOptions() - revertOptions.RevertAddress = sample.BtcAddressP2WPKH(t, &chaincfg.MainNetParams) + revertOptions.RevertAddress = sample.BtcAddressP2WPKH(t, &chaincfg.MainNetParams).String() // create test event receiver := sample.EthAddress() diff --git a/zetaclient/chains/bitcoin/observer/gas_price.go b/zetaclient/chains/bitcoin/observer/gas_price.go new file mode 100644 index 0000000000..1263b3a2ef --- /dev/null +++ b/zetaclient/chains/bitcoin/observer/gas_price.go @@ -0,0 +1,70 @@ +package observer + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" +) + +// PostGasPrice posts gas price to zetacore +func (ob *Observer) PostGasPrice(ctx context.Context) error { + var ( + err error + feeRateEstimated int64 + ) + + // special handle regnet and testnet gas rate + // regnet: RPC 'EstimateSmartFee' is not available + // testnet: RPC 'EstimateSmartFee' returns unreasonable high gas rate + if ob.Chain().NetworkType != chains.NetworkType_mainnet { + feeRateEstimated, err = ob.specialHandleFeeRate(ctx) + if err != nil { + return errors.Wrap(err, "unable to execute specialHandleFeeRate") + } + } else { + isRegnet := chains.IsBitcoinRegnet(ob.Chain().ChainId) + feeRateEstimated, err = ob.rpc.GetEstimatedFeeRate(ctx, 1, isRegnet) + if err != nil { + return errors.Wrap(err, "unable to get estimated fee rate") + } + } + + // query the current block number + blockNumber, err := ob.rpc.GetBlockCount(ctx) + if err != nil { + return errors.Wrap(err, "GetBlockCount error") + } + + // Bitcoin has no concept of priority fee (like eth) + const priorityFee = 0 + + // #nosec G115 always positive + _, err = ob.ZetacoreClient(). + PostVoteGasPrice(ctx, ob.Chain(), uint64(feeRateEstimated), priorityFee, uint64(blockNumber)) + if err != nil { + return errors.Wrap(err, "PostVoteGasPrice error") + } + + return nil +} + +// specialHandleFeeRate handles the fee rate for regnet and testnet +func (ob *Observer) specialHandleFeeRate(ctx context.Context) (int64, error) { + switch ob.Chain().NetworkType { + case chains.NetworkType_privnet: + return client.FeeRateRegnet, nil + case chains.NetworkType_testnet: + feeRateEstimated, err := common.GetRecentFeeRate(ctx, ob.rpc, ob.netParams) + if err != nil { + return 0, errors.Wrapf(err, "error GetRecentFeeRate") + } + return feeRateEstimated, nil + default: + return 0, fmt.Errorf(" unsupported bitcoin network type %d", ob.Chain().NetworkType) + } +} diff --git a/zetaclient/chains/bitcoin/observer/inbound_test.go b/zetaclient/chains/bitcoin/observer/inbound_test.go index 60ac90cb18..d2775d9d28 100644 --- a/zetaclient/chains/bitcoin/observer/inbound_test.go +++ b/zetaclient/chains/bitcoin/observer/inbound_test.go @@ -155,7 +155,7 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain) + ob := newTestSuite(t, chain, "") ob.zetacore.WithKeys(&keys.Keys{}).WithZetaChain() // test cases @@ -167,7 +167,7 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { { name: "should return vote for standard memo", event: &observer.BTCInboundEvent{ - FromAddress: sample.BtcAddressP2WPKH(t, &chaincfg.MainNetParams), + FromAddress: sample.BtcAddressP2WPKH(t, &chaincfg.MainNetParams).String(), // a deposit and call MemoBytes: testutil.HexToBytes( t, diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 007f22f965..72e78a0c6b 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -3,10 +3,7 @@ package observer import ( "context" - "fmt" - "math" "math/big" - "sort" "sync/atomic" "time" @@ -21,11 +18,9 @@ import ( "github.com/zeta-chain/node/pkg/chains" observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/db" "github.com/zeta-chain/node/zetaclient/metrics" - clienttypes "github.com/zeta-chain/node/zetaclient/types" ) type RPC interface { @@ -44,6 +39,7 @@ type RPC interface { res *btcjson.GetTransactionResult, ) (btcjson.TxRawResult, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) GetTransactionFeeAndRate(ctx context.Context, tx *btcjson.TxRawResult) (int64, int64, error) EstimateSmartFee( @@ -108,8 +104,9 @@ type Observer struct { // utxos contains the UTXOs owned by the TSS address utxos []btcjson.ListUnspentResult - // includedTxHashes indexes included tx with tx hash - includedTxHashes map[string]bool + // tssOutboundHashes indexes included tx with tx hash + // tssOutboundHashes keeps track of outbound hashes sent from TSS address + tssOutboundHashes map[string]bool // includedTxResults indexes tx results with the outbound tx identifier includedTxResults map[string]*btcjson.GetTransactionResult @@ -162,9 +159,8 @@ func NewObserver( Observer: *baseObserver, netParams: netParams, rpc: rpc, - pendingNonce: 0, utxos: []btcjson.ListUnspentResult{}, - includedTxHashes: make(map[string]bool), + tssOutboundHashes: make(map[string]bool), includedTxResults: make(map[string]*btcjson.GetTransactionResult), broadcastedTx: make(map[string]string), logger: Logger{ @@ -191,10 +187,6 @@ func NewObserver( return ob, nil } -func (ob *Observer) isNodeEnabled() bool { - return ob.nodeEnabled.Load() -} - // GetPendingNonce returns the artificial pending nonce // Note: pending nonce is accessed concurrently func (ob *Observer) GetPendingNonce() uint64 { @@ -203,6 +195,13 @@ func (ob *Observer) GetPendingNonce() uint64 { return ob.pendingNonce } +// SetPendingNonce sets the artificial pending nonce +func (ob *Observer) SetPendingNonce(nonce uint64) { + ob.Mu().Lock() + defer ob.Mu().Unlock() + ob.pendingNonce = nonce +} + // ConfirmationsThreshold returns number of required Bitcoin confirmations depending on sent BTC amount. func (ob *Observer) ConfirmationsThreshold(amount *big.Int) int64 { if amount.Cmp(big.NewInt(BigValueSats)) >= 0 { @@ -216,142 +215,6 @@ func (ob *Observer) ConfirmationsThreshold(amount *big.Int) int64 { return int64(ob.ChainParams().ConfirmationCount) } -// PostGasPrice posts gas price to zetacore -// TODO(revamp): move to gas price file -func (ob *Observer) PostGasPrice(ctx context.Context) error { - var ( - err error - feeRateEstimated uint64 - ) - - // special handle regnet and testnet gas rate - // regnet: RPC 'EstimateSmartFee' is not available - // testnet: RPC 'EstimateSmartFee' returns unreasonable high gas rate - if ob.Chain().NetworkType != chains.NetworkType_mainnet { - feeRateEstimated, err = ob.specialHandleFeeRate(ctx) - if err != nil { - return errors.Wrap(err, "unable to execute specialHandleFeeRate") - } - } else { - // EstimateSmartFee returns the fees per kilobyte (BTC/kb) targeting given block confirmation - feeResult, err := ob.rpc.EstimateSmartFee(ctx, 1, &btcjson.EstimateModeEconomical) - if err != nil { - return errors.Wrap(err, "unable to estimate smart fee") - } - if feeResult.Errors != nil || feeResult.FeeRate == nil { - return fmt.Errorf("error getting gas price: %s", feeResult.Errors) - } - if *feeResult.FeeRate > math.MaxInt64 { - return fmt.Errorf("gas price is too large: %f", *feeResult.FeeRate) - } - feeRateEstimated = common.FeeRateToSatPerByte(*feeResult.FeeRate).Uint64() - } - - // query the current block number - blockNumber, err := ob.rpc.GetBlockCount(ctx) - if err != nil { - return errors.Wrap(err, "GetBlockCount error") - } - - // UTXO has no concept of priority fee (like eth) - const priorityFee = 0 - - // #nosec G115 always positive - _, err = ob.ZetacoreClient().PostVoteGasPrice(ctx, ob.Chain(), feeRateEstimated, priorityFee, uint64(blockNumber)) - if err != nil { - return errors.Wrap(err, "PostVoteGasPrice error") - } - - return nil -} - -// FetchUTXOs fetches TSS-owned UTXOs from the Bitcoin node -// TODO(revamp): move to UTXO file -func (ob *Observer) FetchUTXOs(ctx context.Context) error { - defer func() { - if err := recover(); err != nil { - ob.logger.UTXOs.Error().Msgf("BTC FetchUTXOs: caught panic error: %v", err) - } - }() - - // noop - if !ob.isNodeEnabled() { - return nil - } - - // This is useful when a zetaclient's pending nonce lagged behind for whatever reason. - ob.refreshPendingNonce(ctx) - - // get the current block height. - bh, err := ob.rpc.GetBlockCount(ctx) - if err != nil { - return errors.Wrap(err, "unable to get block height") - } - - maxConfirmations := int(bh) - - // List all unspent UTXOs (160ms) - tssAddr, err := ob.TSS().PubKey().AddressBTC(ob.Chain().ChainId) - if err != nil { - return errors.Wrap(err, "unable to get tss address") - } - - utxos, err := ob.rpc.ListUnspentMinMaxAddresses(ctx, 0, maxConfirmations, []btcutil.Address{tssAddr}) - if err != nil { - return errors.Wrap(err, "unable to list unspent utxo") - } - - // rigid sort to make utxo list deterministic - sort.SliceStable(utxos, func(i, j int) bool { - if utxos[i].Amount == utxos[j].Amount { - if utxos[i].TxID == utxos[j].TxID { - return utxos[i].Vout < utxos[j].Vout - } - return utxos[i].TxID < utxos[j].TxID - } - return utxos[i].Amount < utxos[j].Amount - }) - - // filter UTXOs good to spend for next TSS transaction - utxosFiltered := make([]btcjson.ListUnspentResult, 0) - for _, utxo := range utxos { - // UTXOs big enough to cover the cost of spending themselves - if utxo.Amount < common.DefaultDepositorFee { - continue - } - // we don't want to spend other people's unconfirmed UTXOs as they may not be safe to spend - if utxo.Confirmations == 0 { - if !ob.isTssTransaction(utxo.TxID) { - continue - } - } - utxosFiltered = append(utxosFiltered, utxo) - } - - ob.Mu().Lock() - ob.TelemetryServer().SetNumberOfUTXOs(len(utxosFiltered)) - ob.utxos = utxosFiltered - ob.Mu().Unlock() - return nil -} - -// SaveBroadcastedTx saves successfully broadcasted transaction -// TODO(revamp): move to db file -func (ob *Observer) SaveBroadcastedTx(txHash string, nonce uint64) { - outboundID := ob.OutboundID(nonce) - ob.Mu().Lock() - ob.broadcastedTx[outboundID] = txHash - ob.Mu().Unlock() - - broadcastEntry := clienttypes.ToOutboundHashSQLType(txHash, outboundID) - if err := ob.DB().Client().Save(&broadcastEntry).Error; err != nil { - ob.logger.Outbound.Error(). - Err(err). - Msgf("SaveBroadcastedTx: error saving broadcasted txHash %s for outbound %s", txHash, outboundID) - } - ob.logger.Outbound.Info().Msgf("SaveBroadcastedTx: saved broadcasted txHash %s for outbound %s", txHash, outboundID) -} - // GetBlockByNumberCached gets cached block (and header) by block number func (ob *Observer) GetBlockByNumberCached(ctx context.Context, blockNumber int64) (*BTCBlockNHeader, error) { if result, ok := ob.BlockCache().Get(blockNumber); ok { @@ -385,67 +248,23 @@ func (ob *Observer) GetBlockByNumberCached(ctx context.Context, blockNumber int6 return blockNheader, nil } -// LoadLastBlockScanned loads the last scanned block from the database -func (ob *Observer) LoadLastBlockScanned(ctx context.Context) error { - err := ob.Observer.LoadLastBlockScanned(ob.Logger().Chain) - if err != nil { - return errors.Wrapf(err, "error LoadLastBlockScanned for chain %d", ob.Chain().ChainId) - } - - // observer will scan from the last block when 'lastBlockScanned == 0', this happens when: - // 1. environment variable is set explicitly to "latest" - // 2. environment variable is empty and last scanned block is not found in DB - if ob.LastBlockScanned() == 0 { - blockNumber, err := ob.rpc.GetBlockCount(ctx) - if err != nil { - return errors.Wrapf(err, "error GetBlockCount for chain %d", ob.Chain().ChainId) - } - // #nosec G115 always positive - ob.WithLastBlockScanned(uint64(blockNumber)) - } - - // bitcoin regtest starts from hardcoded block 100 - if chains.IsBitcoinRegnet(ob.Chain().ChainId) { - ob.WithLastBlockScanned(RegnetStartBlock) - } - ob.Logger().Chain.Info().Msgf("chain %d starts scanning from block %d", ob.Chain().ChainId, ob.LastBlockScanned()) - - return nil +// IsTSSTransaction checks if a given transaction was sent by TSS itself. +// An unconfirmed transaction is safe to spend only if it was sent by TSS and verified by ourselves. +func (ob *Observer) IsTSSTransaction(txid string) bool { + _, found := ob.tssOutboundHashes[txid] + return found } -// LoadBroadcastedTxMap loads broadcasted transactions from the database -func (ob *Observer) LoadBroadcastedTxMap() error { - var broadcastedTransactions []clienttypes.OutboundHashSQLType - if err := ob.DB().Client().Find(&broadcastedTransactions).Error; err != nil { - ob.logger.Chain.Error().Err(err).Msgf("error iterating over db for chain %d", ob.Chain().ChainId) - return err - } - for _, entry := range broadcastedTransactions { - ob.broadcastedTx[entry.Key] = entry.Hash - } - return nil -} +// GetBroadcastedTx gets successfully broadcasted transaction by nonce +func (ob *Observer) GetBroadcastedTx(nonce uint64) (string, bool) { + ob.Mu().Lock() + defer ob.Mu().Unlock() -// specialHandleFeeRate handles the fee rate for regnet and testnet -func (ob *Observer) specialHandleFeeRate(ctx context.Context) (uint64, error) { - switch ob.Chain().NetworkType { - case chains.NetworkType_privnet: - // hardcode gas price for regnet - return 1, nil - case chains.NetworkType_testnet: - feeRateEstimated, err := common.GetRecentFeeRate(ctx, ob.rpc, ob.netParams) - if err != nil { - return 0, errors.Wrapf(err, "error GetRecentFeeRate") - } - return feeRateEstimated, nil - default: - return 0, fmt.Errorf(" unsupported bitcoin network type %d", ob.Chain().NetworkType) - } + outboundID := ob.OutboundID(nonce) + txHash, found := ob.broadcastedTx[outboundID] + return txHash, found } -// isTssTransaction checks if a given transaction was sent by TSS itself. -// An unconfirmed transaction is safe to spend only if it was sent by TSS and verified by ourselves. -func (ob *Observer) isTssTransaction(txid string) bool { - _, found := ob.includedTxHashes[txid] - return found +func (ob *Observer) isNodeEnabled() bool { + return ob.nodeEnabled.Load() } diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index 3b847297ef..2e664a3734 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -9,11 +9,11 @@ import ( "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/wire" - "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/zetaclient/db" + "github.com/zeta-chain/node/zetaclient/testutils" "gorm.io/gorm" "github.com/zeta-chain/node/pkg/chains" @@ -165,7 +165,7 @@ func Test_NewObserver(t *testing.T) { func Test_BlockCache(t *testing.T) { t.Run("should add and get block from cache", func(t *testing.T) { // create observer - ob := newTestSuite(t, chains.BitcoinMainnet) + ob := newTestSuite(t, chains.BitcoinMainnet, "") // feed block hash, header and block to btc client hash := sample.BtcHash() @@ -189,7 +189,7 @@ func Test_BlockCache(t *testing.T) { }) t.Run("should fail if stored type is not BlockNHeader", func(t *testing.T) { // create observer - ob := newTestSuite(t, chains.BitcoinMainnet) + ob := newTestSuite(t, chains.BitcoinMainnet, "") // add a string to cache blockNumber := int64(100) @@ -202,63 +202,22 @@ func Test_BlockCache(t *testing.T) { }) } -func Test_LoadLastBlockScanned(t *testing.T) { - // use Bitcoin mainnet chain for testing - chain := chains.BitcoinMainnet - ctx := context.Background() - - t.Run("should load last block scanned", func(t *testing.T) { - // create observer and write 199 as last block scanned - ob := newTestSuite(t, chain) - ob.WriteLastBlockScannedToDB(199) - - // load last block scanned - err := ob.LoadLastBlockScanned(ctx) - require.NoError(t, err) - require.EqualValues(t, 199, ob.LastBlockScanned()) - }) - t.Run("should fail on invalid env var", func(t *testing.T) { - // create observer - ob := newTestSuite(t, chain) - - // set invalid environment variable - envvar := base.EnvVarLatestBlockByChain(chain) - os.Setenv(envvar, "invalid") - defer os.Unsetenv(envvar) - - // load last block scanned - err := ob.LoadLastBlockScanned(ctx) - require.ErrorContains(t, err, "error LoadLastBlockScanned") - }) - t.Run("should fail on RPC error", func(t *testing.T) { - // create observer on separate path, as we need to reset last block scanned - obOther := newTestSuite(t, chain) - - // reset last block scanned to 0 so that it will be loaded from RPC - obOther.WithLastBlockScanned(0) +func Test_SetPendingNonce(t *testing.T) { + // create observer + ob := newTestSuite(t, chains.BitcoinMainnet, "") - // attach a mock btc client that returns rpc error - obOther.client.ExpectedCalls = nil - obOther.client.On("GetBlockCount", mock.Anything).Return(int64(0), errors.New("rpc error")) + // ensure pending nonce is 0 + require.Zero(t, ob.GetPendingNonce()) - // load last block scanned - err := obOther.LoadLastBlockScanned(ctx) - require.ErrorContains(t, err, "rpc error") - }) - t.Run("should use hardcode block 100 for regtest", func(t *testing.T) { - // use regtest chain - obRegnet := newTestSuite(t, chains.BitcoinRegtest) - - // load last block scanned - err := obRegnet.LoadLastBlockScanned(ctx) - require.NoError(t, err) - require.EqualValues(t, observer.RegnetStartBlock, obRegnet.LastBlockScanned()) - }) + // set and get pending nonce + nonce := uint64(100) + ob.SetPendingNonce(nonce) + require.Equal(t, nonce, ob.GetPendingNonce()) } func TestConfirmationThreshold(t *testing.T) { chain := chains.BitcoinMainnet - ob := newTestSuite(t, chain) + ob := newTestSuite(t, chain, "") t.Run("should return confirmations in chain param", func(t *testing.T) { ob.SetChainParams(observertypes.ChainParams{ConfirmationCount: 3}) @@ -307,7 +266,7 @@ type testSuite struct { db *db.DB } -func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { +func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { ctx := context.Background() require.True(t, chain.IsBitcoinChain()) @@ -315,24 +274,41 @@ func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { chainParams := mocks.MockChainParams(chain.ChainId, 10) client := mocks.NewBitcoinClient(t) - client.On("GetBlockCount", mock.Anything).Return(int64(100), nil).Maybe() + client.On("GetBlockCount", mock.Anything).Maybe().Return(int64(100), nil).Maybe() zetacore := mocks.NewZetacoreClient(t) - database, err := db.NewFromSqliteInMemory(true) + var tss interfaces.TSSSigner + if chains.IsBitcoinMainnet(chain.ChainId) { + tss = mocks.NewTSS(t).FakePubKey(testutils.TSSPubKeyMainnet) + } else { + tss = mocks.NewTSS(t).FakePubKey(testutils.TSSPubkeyAthens3) + } + + // create test database + var err error + var database *db.DB + if dbPath == "" { + database, err = db.NewFromSqliteInMemory(true) + } else { + database, err = db.NewFromSqlite(dbPath, "test.db", true) + } require.NoError(t, err) - log := zerolog.New(zerolog.NewTestWriter(t)) + // create logger + testLogger := zerolog.New(zerolog.NewTestWriter(t)) + logger := base.Logger{Std: testLogger, Compliance: testLogger} + // create observer ob, err := observer.NewObserver( chain, client, chainParams, zetacore, - nil, + tss, database, - base.Logger{Std: log, Compliance: log}, - nil, + logger, + &metrics.TelemetryServer{}, ) require.NoError(t, err) diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index ac4b75a172..7192b754b0 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -17,69 +17,88 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/compliance" + "github.com/zeta-chain/node/zetaclient/logs" "github.com/zeta-chain/node/zetaclient/zetacore" ) +const ( + // minTxConfirmations is the minimum confirmations for a Bitcoin tx to be considered valid by the observer + // Note: please change this value to 1 to be able to run the Bitcoin E2E RBF test + minTxConfirmations = 0 +) + func (ob *Observer) ObserveOutbound(ctx context.Context) error { chainID := ob.Chain().ChainId - trackers, err := ob.ZetacoreClient().GetAllOutboundTrackerByChain(ctx, chainID, interfaces.Ascending) if err != nil { return errors.Wrap(err, "unable to get all outbound trackers") } + // logger fields + lf := map[string]any{ + logs.FieldMethod: "ProcessOutboundTrackers", + } + for _, tracker := range trackers { - // get original cctx parameters - outboundID := ob.OutboundID(tracker.Nonce) + // set logger fields + lf[logs.FieldNonce] = tracker.Nonce + + // get the CCTX cctx, err := ob.ZetacoreClient().GetCctxByNonce(ctx, chainID, tracker.Nonce) if err != nil { - return errors.Wrapf(err, "unable to get cctx by nonce %d", tracker.Nonce) - } - - nonce := cctx.GetCurrentOutboundParam().TssNonce - if tracker.Nonce != nonce { // Tanmay: it doesn't hurt to check - return fmt.Errorf("tracker nonce %d not match cctx nonce %d", tracker.Nonce, nonce) + ob.logger.Outbound.Err(err).Fields(lf).Msg("cannot find cctx") + break } - if len(tracker.HashList) > 1 { - ob.logger.Outbound.Warn(). - Msgf("WatchOutbound: oops, outboundID %s got multiple (%d) outbound hashes", outboundID, len(tracker.HashList)) + ob.logger.Outbound.Warn().Msgf("oops, got multiple (%d) outbound hashes", len(tracker.HashList)) } - // iterate over all txHashes to find the truly included one. - // we do it this (inefficient) way because we don't rely on the first one as it may be a false positive (for unknown reason). - txCount := 0 - var txResult *btcjson.GetTransactionResult + // Iterate over all txHashes to find the truly included outbound. + // At any time, there is guarantee that only one single txHash will be considered valid and included for each nonce. + // The reasons are: + // 1. CCTX with nonce 'N = 0' is the past and well-controlled. + // 2. Given any CCTX with nonce 'N > 0', its outbound MUST spend the previous nonce-mark UTXO (nonce N-1) to be considered valid. + // 3. Bitcoin prevents double spending of the same UTXO except for RBF. + // 4. When RBF happens, the original tx will be removed from Bitcoin core, and only the new tx will be valid. for _, txHash := range tracker.HashList { - result, inMempool := ob.checkIncludedTx(ctx, cctx, txHash.TxHash) - if result != nil && !inMempool { // included - txCount++ - txResult = result - ob.logger.Outbound.Info(). - Msgf("WatchOutbound: included outbound %s for chain %d nonce %d", txHash.TxHash, chainID, tracker.Nonce) - if txCount > 1 { - ob.logger.Outbound.Error().Msgf( - "WatchOutbound: checkIncludedTx passed, txCount %d chain %d nonce %d result %v", txCount, chainID, tracker.Nonce, result) - } + _, included := ob.TryIncludeOutbound(ctx, cctx, txHash.TxHash) + if included { + break } } - - if txCount == 1 { // should be only one txHash included for each nonce - ob.setIncludedTx(tracker.Nonce, txResult) - } else if txCount > 1 { - ob.removeIncludedTx(tracker.Nonce) // we can't tell which txHash is true, so we remove all (if any) to be safe - ob.logger.Outbound.Error().Msgf("WatchOutbound: included multiple (%d) outbound for chain %d nonce %d", txCount, chainID, tracker.Nonce) - } } return nil } -// VoteOutboundIfConfirmed checks outbound status and returns (continueKeysign, error) -func (ob *Observer) VoteOutboundIfConfirmed( +// TryIncludeOutbound tries to include an outbound for the given cctx and txHash. +// +// Due to 10-min block time, zetaclient observes outbounds both in mempool and in blocks. +// An outbound is considered included if it satisfies one of the following two cases: +// 1. a valid tx pending in mempool with confirmation == 0 +// 2. a valid tx included in a block with confirmation > 0 +// +// Returns: (txResult, included) +// +// Note: A 'included' tx may still be considered stuck if it sits in the mempool for too long. +func (ob *Observer) TryIncludeOutbound( ctx context.Context, cctx *crosschaintypes.CrossChainTx, -) (bool, error) { + txHash string, +) (*btcjson.GetTransactionResult, bool) { + nonce := cctx.GetCurrentOutboundParam().TssNonce + + // check tx inclusion and save tx result + txResult, included := ob.checkTxInclusion(ctx, cctx, txHash) + if included { + ob.SetIncludedTx(nonce, txResult) + } + + return txResult, included +} + +// VoteOutboundIfConfirmed checks outbound status and returns (continueKeysign, error) +func (ob *Observer) VoteOutboundIfConfirmed(ctx context.Context, cctx *crosschaintypes.CrossChainTx) (bool, error) { const ( // not used with Bitcoin outboundGasUsed = 0 @@ -102,6 +121,9 @@ func (ob *Observer) VoteOutboundIfConfirmed( res, included := ob.includedTxResults[outboundID] ob.Mu().Unlock() + // Short-circuit in following two cases: + // 1. Outbound neither broadcasted nor included. It requires a keysign. + // 2. Outbound was broadcasted for nonce 0. It's an edge case (happened before) to avoid duplicate payments. if !included { if !broadcasted { return true, nil @@ -116,26 +138,15 @@ func (ob *Observer) VoteOutboundIfConfirmed( return false, nil } - // Try including this outbound broadcasted by myself - txResult, inMempool := ob.checkIncludedTx(ctx, cctx, txnHash) - if txResult == nil { // check failed, try again next time - return true, nil - } else if inMempool { // still in mempool (should avoid unnecessary Tss keysign) - ob.logger.Outbound.Info().Msgf("VoteOutboundIfConfirmed: outbound %s is still in mempool", outboundID) - return false, nil - } - // included - ob.setIncludedTx(nonce, txResult) - - // Get tx result again in case it is just included - res = ob.getIncludedTx(nonce) - if res == nil { + // Try including this outbound broadcasted by myself to supplement outbound trackers. + // Note: each Bitcoin outbound usually gets included right after broadcasting. + res, included = ob.TryIncludeOutbound(ctx, cctx, txnHash) + if !included { return true, nil } - ob.logger.Outbound.Info().Msgf("VoteOutboundIfConfirmed: setIncludedTx succeeded for outbound %s", outboundID) } - // It's safe to use cctx's amount to post confirmation because it has already been verified in observeOutbound() + // It's safe to use cctx's amount to post confirmation because it has already been verified in checkTxInclusion(). amountInSat := params.Amount.BigInt() if res.Confirmations < ob.ConfirmationsThreshold(amountInSat) { ob.logger.Outbound.Debug(). @@ -204,105 +215,6 @@ func (ob *Observer) VoteOutboundIfConfirmed( return false, nil } -// SelectUTXOs selects a sublist of utxos to be used as inputs. -// -// Parameters: -// - amount: The desired minimum total value of the selected UTXOs. -// - utxos2Spend: The maximum number of UTXOs to spend. -// - nonce: The nonce of the outbound transaction. -// - consolidateRank: The rank below which UTXOs will be consolidated. -// - test: true for unit test only. -// -// Returns: -// - a sublist (includes previous nonce-mark) of UTXOs or an error if the qualifying sublist cannot be found. -// - the total value of the selected UTXOs. -// - the number of consolidated UTXOs. -// - the total value of the consolidated UTXOs. -// -// TODO(revamp): move to utxo file -func (ob *Observer) SelectUTXOs( - ctx context.Context, - amount float64, - utxosToSpend uint16, - nonce uint64, - consolidateRank uint16, - test bool, -) ([]btcjson.ListUnspentResult, float64, uint16, float64, error) { - idx := -1 - if nonce == 0 { - // for nonce = 0; make exception; no need to include nonce-mark utxo - ob.Mu().Lock() - defer ob.Mu().Unlock() - } else { - // for nonce > 0; we proceed only when we see the nonce-mark utxo - preTxid, err := ob.getOutboundIDByNonce(ctx, nonce-1, test) - if err != nil { - return nil, 0, 0, 0, err - } - ob.Mu().Lock() - defer ob.Mu().Unlock() - idx, err = ob.findNonceMarkUTXO(nonce-1, preTxid) - if err != nil { - return nil, 0, 0, 0, err - } - } - - // select smallest possible UTXOs to make payment - total := 0.0 - left, right := 0, 0 - for total < amount && right < len(ob.utxos) { - if utxosToSpend > 0 { // expand sublist - total += ob.utxos[right].Amount - right++ - utxosToSpend-- - } else { // pop the smallest utxo and append the current one - total -= ob.utxos[left].Amount - total += ob.utxos[right].Amount - left++ - right++ - } - } - results := make([]btcjson.ListUnspentResult, right-left) - copy(results, ob.utxos[left:right]) - - // include nonce-mark as the 1st input - if idx >= 0 { // for nonce > 0 - if idx < left || idx >= right { - total += ob.utxos[idx].Amount - results = append([]btcjson.ListUnspentResult{ob.utxos[idx]}, results...) - } else { // move nonce-mark to left - for i := idx - left; i > 0; i-- { - results[i], results[i-1] = results[i-1], results[i] - } - } - } - if total < amount { - return nil, 0, 0, 0, fmt.Errorf( - "SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", - total, - amount, - ) - } - - // consolidate biggest possible UTXOs to maximize consolidated value - // consolidation happens only when there are more than (or equal to) consolidateRank (10) UTXOs - utxoRank, consolidatedUtxo, consolidatedValue := uint16(0), uint16(0), 0.0 - for i := len(ob.utxos) - 1; i >= 0 && utxosToSpend > 0; i-- { // iterate over UTXOs big-to-small - if i != idx && (i < left || i >= right) { // exclude nonce-mark and already selected UTXOs - utxoRank++ - if utxoRank >= consolidateRank { // consolication starts from the 10-ranked UTXO based on value - utxosToSpend-- - consolidatedUtxo++ - total += ob.utxos[i].Amount - consolidatedValue += ob.utxos[i].Amount - results = append(results, ob.utxos[i]) - } - } - } - - return results, total, consolidatedUtxo, consolidatedValue, nil -} - // refreshPendingNonce tries increasing the artificial pending nonce of outbound (if lagged behind). // There could be many (unpredictable) reasons for a pending nonce lagging behind, for example: // 1. The zetaclient gets restarted. @@ -315,162 +227,149 @@ func (ob *Observer) refreshPendingNonce(ctx context.Context) { } // increase pending nonce if lagged behind - ob.Mu().Lock() - pendingNonce := ob.pendingNonce - ob.Mu().Unlock() - // #nosec G115 always non-negative nonceLow := uint64(p.NonceLow) - if nonceLow > pendingNonce { + if nonceLow > ob.GetPendingNonce() { // get the last included outbound hash - txid, err := ob.getOutboundIDByNonce(ctx, nonceLow-1, false) + txid, err := ob.getOutboundHashByNonce(ctx, nonceLow-1, false) if err != nil { ob.logger.Chain.Error().Err(err).Msg("refreshPendingNonce: error getting last outbound txid") } // set 'NonceLow' as the new pending nonce - ob.Mu().Lock() - defer ob.Mu().Unlock() - ob.pendingNonce = nonceLow + ob.SetPendingNonce(nonceLow) ob.logger.Chain.Info(). - Msgf("refreshPendingNonce: increase pending nonce to %d with txid %s", ob.pendingNonce, txid) + Msgf("refreshPendingNonce: increase pending nonce to %d with txid %s", nonceLow, txid) } } -// getOutboundIDByNonce gets the outbound ID from the nonce of the outbound transaction +// getOutboundHashByNonce gets the outbound hash for given nonce. // test is true for unit test only -func (ob *Observer) getOutboundIDByNonce(ctx context.Context, nonce uint64, test bool) (string, error) { +func (ob *Observer) getOutboundHashByNonce(ctx context.Context, nonce uint64, test bool) (string, error) { // There are 2 types of txids an observer can trust // 1. The ones had been verified and saved by observer self. // 2. The ones had been finalized in zetacore based on majority vote. - if res := ob.getIncludedTx(nonce); res != nil { + if res := ob.GetIncludedTx(nonce); res != nil { return res.TxID, nil } if !test { // if not unit test, get cctx from zetacore send, err := ob.ZetacoreClient().GetCctxByNonce(ctx, ob.Chain().ChainId, nonce) if err != nil { - return "", errors.Wrapf(err, "getOutboundIDByNonce: error getting cctx for nonce %d", nonce) + return "", errors.Wrapf(err, "getOutboundHashByNonce: error getting cctx for nonce %d", nonce) } txid := send.GetCurrentOutboundParam().Hash if txid == "" { - return "", fmt.Errorf("getOutboundIDByNonce: cannot find outbound txid for nonce %d", nonce) + return "", fmt.Errorf("getOutboundHashByNonce: cannot find outbound txid for nonce %d", nonce) } // make sure it's a real Bitcoin txid _, getTxResult, err := ob.rpc.GetTransactionByStr(ctx, txid) if err != nil { return "", errors.Wrapf( err, - "getOutboundIDByNonce: error getting outbound result for nonce %d hash %s", + "getOutboundHashByNonce: error getting outbound result for nonce %d hash %s", nonce, txid, ) } if getTxResult.Confirmations <= 0 { // just a double check - return "", fmt.Errorf("getOutboundIDByNonce: outbound txid %s for nonce %d is not included", txid, nonce) + return "", fmt.Errorf("getOutboundHashByNonce: outbound txid %s for nonce %d is not included", txid, nonce) } return txid, nil } - return "", fmt.Errorf("getOutboundIDByNonce: cannot find outbound txid for nonce %d", nonce) + return "", fmt.Errorf("getOutboundHashByNonce: cannot find outbound txid for nonce %d", nonce) } -// findNonceMarkUTXO finds the nonce-mark UTXO in the list of UTXOs. -func (ob *Observer) findNonceMarkUTXO(nonce uint64, txid string) (int, error) { - tssAddress := ob.TSSAddressString() - amount := chains.NonceMarkAmount(nonce) - for i, utxo := range ob.utxos { - sats, err := common.GetSatoshis(utxo.Amount) - if err != nil { - ob.logger.Outbound.Error().Err(err).Msgf("findNonceMarkUTXO: error getting satoshis for utxo %v", utxo) - } - if utxo.Address == tssAddress && sats == amount && utxo.TxID == txid && utxo.Vout == 0 { - ob.logger.Outbound.Info(). - Msgf("findNonceMarkUTXO: found nonce-mark utxo with txid %s, amount %d satoshi", utxo.TxID, sats) - return i, nil - } - } - return -1, fmt.Errorf("findNonceMarkUTXO: cannot find nonce-mark utxo with nonce %d", nonce) -} - -// checkIncludedTx checks if a txHash is included and returns (txResult, inMempool) -// Note: if txResult is nil, then inMempool flag should be ignored. -func (ob *Observer) checkIncludedTx( +// checkTxInclusion checks if a txHash is included and returns (txResult, included) +func (ob *Observer) checkTxInclusion( ctx context.Context, cctx *crosschaintypes.CrossChainTx, txHash string, ) (*btcjson.GetTransactionResult, bool) { - outboundID := ob.OutboundID(cctx.GetCurrentOutboundParam().TssNonce) - hash, getTxResult, err := ob.rpc.GetTransactionByStr(ctx, txHash) + // logger fields + lf := map[string]any{ + logs.FieldMethod: "checkTxInclusion", + logs.FieldNonce: cctx.GetCurrentOutboundParam().TssNonce, + logs.FieldTx: txHash, + } + + // fetch tx result + hash, txResult, err := ob.rpc.GetTransactionByStr(ctx, txHash) if err != nil { - ob.logger.Outbound.Error().Err(err).Msgf("checkIncludedTx: error GetTxResultByHash: %s", txHash) + ob.logger.Outbound.Warn().Err(err).Fields(lf).Msg("GetTxResultByHash failed") return nil, false } - if txHash != getTxResult.TxID { // just in case, we'll use getTxResult.TxID later - ob.logger.Outbound.Error(). - Msgf("checkIncludedTx: inconsistent txHash %s and getTxResult.TxID %s", txHash, getTxResult.TxID) + // check minimum confirmations + if txResult.Confirmations < minTxConfirmations { + ob.logger.Outbound.Warn().Fields(lf).Msgf("invalid confirmations %d", txResult.Confirmations) return nil, false } - if getTxResult.Confirmations >= 0 { // check included tx only - err = ob.checkTssOutboundResult(ctx, cctx, hash, getTxResult) - if err != nil { - ob.logger.Outbound.Error(). - Err(err). - Msgf("checkIncludedTx: error verify bitcoin outbound %s outboundID %s", txHash, outboundID) - return nil, false - } - return getTxResult, false // included + // validate tx result + err = ob.checkTssOutboundResult(ctx, cctx, hash, txResult) + if err != nil { + ob.logger.Outbound.Error().Err(err).Fields(lf).Msg("checkTssOutboundResult failed") + return nil, false } - return getTxResult, true // in mempool + + // tx is valid and included + return txResult, true } -// setIncludedTx saves included tx result in memory -func (ob *Observer) setIncludedTx(nonce uint64, getTxResult *btcjson.GetTransactionResult) { - txHash := getTxResult.TxID - outboundID := ob.OutboundID(nonce) +// SetIncludedTx saves included tx result in memory. +// - the outbounds are chained (by nonce) txs sequentially included. +// - tx results may be set in arbitrary order as the method is called across goroutines, and it doesn't matter. +func (ob *Observer) SetIncludedTx(nonce uint64, getTxResult *btcjson.GetTransactionResult) { + var ( + txHash = getTxResult.TxID + outboundID = ob.OutboundID(nonce) + lf = map[string]any{ + logs.FieldMethod: "SetIncludedTx", + logs.FieldNonce: nonce, + logs.FieldTx: txHash, + logs.FieldOutboundID: outboundID, + } + ) ob.Mu().Lock() defer ob.Mu().Unlock() res, found := ob.includedTxResults[outboundID] - if !found { // not found. - ob.includedTxHashes[txHash] = true - ob.includedTxResults[outboundID] = getTxResult // include new outbound and enforce rigid 1-to-1 mapping: nonce <===> txHash - if nonce >= ob.pendingNonce { // try increasing pending nonce on every newly included outbound + if !found { + // for new hash: + // - include new outbound and enforce rigid 1-to-1 mapping: nonce <===> txHash + // - try increasing pending nonce on every newly included outbound + ob.tssOutboundHashes[txHash] = true + ob.includedTxResults[outboundID] = getTxResult + if nonce >= ob.pendingNonce { ob.pendingNonce = nonce + 1 } - ob.logger.Outbound.Info(). - Msgf("setIncludedTx: included new bitcoin outbound %s outboundID %s pending nonce %d", txHash, outboundID, ob.pendingNonce) - } else if txHash == res.TxID { // found same hash - ob.includedTxResults[outboundID] = getTxResult // update tx result as confirmations may increase + ob.logger.Outbound.Info().Fields(lf).Msgf("included new bitcoin outbound, pending nonce %d", ob.pendingNonce) + } else if txHash == res.TxID { + // for existing hash: + // - update tx result because confirmations may increase + ob.includedTxResults[outboundID] = getTxResult if getTxResult.Confirmations > res.Confirmations { - ob.logger.Outbound.Info().Msgf("setIncludedTx: bitcoin outbound %s got confirmations %d", txHash, getTxResult.Confirmations) + ob.logger.Outbound.Info().Msgf("bitcoin outbound got %d confirmations", getTxResult.Confirmations) } - } else { // found other hash. + } else { + // for other hash: // be alert for duplicate payment!!! As we got a new hash paying same cctx (for whatever reason). - delete(ob.includedTxResults, outboundID) // we can't tell which txHash is true, so we remove all to be safe + // we can't tell which txHash is true, so we remove all to be safe + delete(ob.tssOutboundHashes, res.TxID) + delete(ob.includedTxResults, outboundID) ob.logger.Outbound.Error().Msgf("setIncludedTx: duplicate payment by bitcoin outbound %s outboundID %s, prior outbound %s", txHash, outboundID, res.TxID) } } -// getIncludedTx gets the receipt and transaction from memory -func (ob *Observer) getIncludedTx(nonce uint64) *btcjson.GetTransactionResult { +// GetIncludedTx gets the receipt and transaction from memory +func (ob *Observer) GetIncludedTx(nonce uint64) *btcjson.GetTransactionResult { ob.Mu().Lock() defer ob.Mu().Unlock() return ob.includedTxResults[ob.OutboundID(nonce)] } -// removeIncludedTx removes included tx from memory -func (ob *Observer) removeIncludedTx(nonce uint64) { - ob.Mu().Lock() - defer ob.Mu().Unlock() - txResult, found := ob.includedTxResults[ob.OutboundID(nonce)] - if found { - delete(ob.includedTxHashes, txResult.TxID) - delete(ob.includedTxResults, ob.OutboundID(nonce)) - } -} - // Basic TSS outbound checks: +// - confirmations >= 0 // - should be able to query the raw tx // - check if all inputs are segwit && TSS inputs // @@ -485,7 +384,7 @@ func (ob *Observer) checkTssOutboundResult( nonce := params.TssNonce rawResult, err := ob.rpc.GetRawTransactionResult(ctx, hash, res) if err != nil { - return errors.Wrapf(err, "checkTssOutboundResult: error GetRawTxResultByHash %s", hash.String()) + return errors.Wrapf(err, "checkTssOutboundResult: error GetRawTransactionResult %s", hash.String()) } err = ob.checkTSSVin(ctx, rawResult.Vin, nonce) if err != nil { @@ -531,7 +430,7 @@ func (ob *Observer) checkTSSVin(ctx context.Context, vins []btcjson.Vin, nonce u } // 1st vin: nonce-mark MUST come from prior TSS outbound if nonce > 0 && i == 0 { - preTxid, err := ob.getOutboundIDByNonce(ctx, nonce-1, false) + preTxid, err := ob.getOutboundHashByNonce(ctx, nonce-1, false) if err != nil { return fmt.Errorf("checkTSSVin: error findTxIDByNonce %d", nonce-1) } diff --git a/zetaclient/chains/bitcoin/observer/outbound_test.go b/zetaclient/chains/bitcoin/observer/outbound_test.go index 843f1c0f84..b6ddb875df 100644 --- a/zetaclient/chains/bitcoin/observer/outbound_test.go +++ b/zetaclient/chains/bitcoin/observer/outbound_test.go @@ -263,85 +263,85 @@ func TestSelectUTXOs(t *testing.T) { // Case1: nonce = 0, bootstrap // input: utxoCap = 5, amount = 0.01, nonce = 0 // output: [0.01], 0.01 - result, amount, _, _, err := ob.SelectUTXOs(ctx, 0.01, 5, 0, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 0, math.MaxUint16, true) require.NoError(t, err) - require.Equal(t, 0.01, amount) - require.Equal(t, ob.utxos[0:1], result) + require.Equal(t, 0.01, selected.Value) + require.Equal(t, ob.utxos[0:1], selected.UTXOs) // Case2: nonce = 1, must FAIL and wait for previous transaction to be mined // input: utxoCap = 5, amount = 0.5, nonce = 1 // output: error - result, amount, _, _, err = ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) require.Error(t, err) - require.Nil(t, result) - require.Zero(t, amount) - require.Equal(t, "getOutboundIDByNonce: cannot find outbound txid for nonce 0", err.Error()) + require.Nil(t, selected.UTXOs) + require.Zero(t, selected.Value) + require.Equal(t, "getOutboundHashByNonce: cannot find outbound txid for nonce 0", err.Error()) mineTxNSetNonceMark(t, ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 // Case3: nonce = 1, should pass now // input: utxoCap = 5, amount = 0.5, nonce = 1 // output: [0.00002, 0.01, 0.12, 0.18, 0.24], 0.55002 - result, amount, _, _, err = ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) require.NoError(t, err) - require.Equal(t, 0.55002, amount) - require.Equal(t, ob.utxos[0:5], result) + require.Equal(t, 0.55002, selected.Value) + require.Equal(t, ob.utxos[0:5], selected.UTXOs) mineTxNSetNonceMark(t, ob, 1, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 1 // Case4: // input: utxoCap = 5, amount = 1.0, nonce = 2 // output: [0.00002001, 0.01, 0.12, 0.18, 0.24, 0.5], 1.05002001 - result, amount, _, _, err = ob.SelectUTXOs(ctx, 1.0, 5, 2, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 1.0, 5, 2, math.MaxUint16, true) require.NoError(t, err) - require.InEpsilon(t, 1.05002001, amount, 1e-8) - require.Equal(t, ob.utxos[0:6], result) + require.InEpsilon(t, 1.05002001, selected.Value, 1e-8) + require.Equal(t, ob.utxos[0:6], selected.UTXOs) mineTxNSetNonceMark(t, ob, 2, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 2 // Case5: should include nonce-mark utxo on the LEFT // input: utxoCap = 5, amount = 8.05, nonce = 3 // output: [0.00002002, 0.24, 0.5, 1.26, 2.97, 3.28], 8.25002002 - result, amount, _, _, err = ob.SelectUTXOs(ctx, 8.05, 5, 3, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 8.05, 5, 3, math.MaxUint16, true) require.NoError(t, err) - require.InEpsilon(t, 8.25002002, amount, 1e-8) + require.InEpsilon(t, 8.25002002, selected.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{ob.utxos[0]}, ob.utxos[4:9]...) - require.Equal(t, expected, result) + require.Equal(t, expected, selected.UTXOs) mineTxNSetNonceMark(t, ob, 24105431, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 24105431 // Case6: should include nonce-mark utxo on the RIGHT // input: utxoCap = 5, amount = 0.503, nonce = 24105432 // output: [0.24107432, 0.01, 0.12, 0.18, 0.24], 0.55002002 - result, amount, _, _, err = ob.SelectUTXOs(ctx, 0.503, 5, 24105432, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 0.503, 5, 24105432, math.MaxUint16, true) require.NoError(t, err) - require.InEpsilon(t, 0.79107431, amount, 1e-8) + require.InEpsilon(t, 0.79107431, selected.Value, 1e-8) expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:4]...) - require.Equal(t, expected, result) + require.Equal(t, expected, selected.UTXOs) mineTxNSetNonceMark(t, ob, 24105432, dummyTxID, 4) // mine a transaction and set nonce-mark utxo for nonce 24105432 // Case7: should include nonce-mark utxo in the MIDDLE // input: utxoCap = 5, amount = 1.0, nonce = 24105433 // output: [0.24107432, 0.12, 0.18, 0.24, 0.5], 1.28107432 - result, amount, _, _, err = ob.SelectUTXOs(ctx, 1.0, 5, 24105433, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 1.0, 5, 24105433, math.MaxUint16, true) require.NoError(t, err) - require.InEpsilon(t, 1.28107432, amount, 1e-8) + require.InEpsilon(t, 1.28107432, selected.Value, 1e-8) expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[1:4]...) expected = append(expected, ob.utxos[5]) - require.Equal(t, expected, result) + require.Equal(t, expected, selected.UTXOs) // Case8: should work with maximum amount // input: utxoCap = 5, amount = 16.03 // output: [0.24107432, 1.26, 2.97, 3.28, 5.16, 8.72], 21.63107432 - result, amount, _, _, err = ob.SelectUTXOs(ctx, 16.03, 5, 24105433, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 16.03, 5, 24105433, math.MaxUint16, true) require.NoError(t, err) - require.InEpsilon(t, 21.63107432, amount, 1e-8) + require.InEpsilon(t, 21.63107432, selected.Value, 1e-8) expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[6:11]...) - require.Equal(t, expected, result) + require.Equal(t, expected, selected.UTXOs) // Case9: must FAIL due to insufficient funds // input: utxoCap = 5, amount = 21.64 // output: error - result, amount, _, _, err = ob.SelectUTXOs(ctx, 21.64, 5, 24105433, math.MaxUint16, true) + selected, err = ob.SelectUTXOs(ctx, 21.64, 5, 24105433, math.MaxUint16, true) require.Error(t, err) - require.Nil(t, result) - require.Zero(t, amount) + require.Nil(t, selected.UTXOs) + require.Zero(t, selected.Value) require.Equal( t, "SelectUTXOs: not enough btc in reserve - available : 21.63107432 , tx amount : 21.64", @@ -360,12 +360,12 @@ func TestUTXOConsolidation(t *testing.T) { // input: utxoCap = 10, amount = 0.01, nonce = 1, rank = 10 // output: [0.00002, 0.01], 0.01002 - result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(ctx, 0.01, 10, 1, 10, true) + res, err := ob.SelectUTXOs(ctx, 0.01, 10, 1, 10, true) require.NoError(t, err) - require.Equal(t, 0.01002, amount) - require.Equal(t, ob.utxos[0:2], result) - require.Equal(t, uint16(0), clsdtUtxo) - require.Equal(t, 0.0, clsdtValue) + require.Equal(t, 0.01002, res.Value) + require.Equal(t, ob.utxos[0:2], res.UTXOs) + require.Equal(t, uint16(0), res.ConsolidatedUTXOs) + require.Equal(t, 0.0, res.ConsolidatedValue) }) t.Run("should consolidate 1 utxo", func(t *testing.T) { @@ -374,12 +374,12 @@ func TestUTXOConsolidation(t *testing.T) { // input: utxoCap = 9, amount = 0.01, nonce = 1, rank = 9 // output: [0.00002, 0.01, 0.12], 0.13002 - result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(ctx, 0.01, 9, 1, 9, true) + res, err := ob.SelectUTXOs(ctx, 0.01, 9, 1, 9, true) require.NoError(t, err) - require.Equal(t, 0.13002, amount) - require.Equal(t, ob.utxos[0:3], result) - require.Equal(t, uint16(1), clsdtUtxo) - require.Equal(t, 0.12, clsdtValue) + require.Equal(t, 0.13002, res.Value) + require.Equal(t, ob.utxos[0:3], res.UTXOs) + require.Equal(t, uint16(1), res.ConsolidatedUTXOs) + require.Equal(t, 0.12, res.ConsolidatedValue) }) t.Run("should consolidate 3 utxos", func(t *testing.T) { @@ -388,17 +388,17 @@ func TestUTXOConsolidation(t *testing.T) { // input: utxoCap = 5, amount = 0.01, nonce = 0, rank = 5 // output: [0.00002, 0.014, 1.26, 0.5, 0.2], 2.01002 - result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(ctx, 0.01, 5, 1, 5, true) + res, err := ob.SelectUTXOs(ctx, 0.01, 5, 1, 5, true) require.NoError(t, err) - require.Equal(t, 2.01002, amount) + require.Equal(t, 2.01002, res.Value) expected := make([]btcjson.ListUnspentResult, 2) copy(expected, ob.utxos[0:2]) for i := 6; i >= 4; i-- { // append consolidated utxos in descending order expected = append(expected, ob.utxos[i]) } - require.Equal(t, expected, result) - require.Equal(t, uint16(3), clsdtUtxo) - require.Equal(t, 2.0, clsdtValue) + require.Equal(t, expected, res.UTXOs) + require.Equal(t, uint16(3), res.ConsolidatedUTXOs) + require.Equal(t, 2.0, res.ConsolidatedValue) }) t.Run("should consolidate all utxos using rank 1", func(t *testing.T) { @@ -407,17 +407,17 @@ func TestUTXOConsolidation(t *testing.T) { // input: utxoCap = 12, amount = 0.01, nonce = 0, rank = 1 // output: [0.00002, 0.01, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18, 0.12], 22.44002 - result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(ctx, 0.01, 12, 1, 1, true) + res, err := ob.SelectUTXOs(ctx, 0.01, 12, 1, 1, true) require.NoError(t, err) - require.Equal(t, 22.44002, amount) + require.Equal(t, 22.44002, res.Value) expected := make([]btcjson.ListUnspentResult, 2) copy(expected, ob.utxos[0:2]) for i := 10; i >= 2; i-- { // append consolidated utxos in descending order expected = append(expected, ob.utxos[i]) } - require.Equal(t, expected, result) - require.Equal(t, uint16(9), clsdtUtxo) - require.Equal(t, 22.43, clsdtValue) + require.Equal(t, expected, res.UTXOs) + require.Equal(t, uint16(9), res.ConsolidatedUTXOs) + require.Equal(t, 22.43, res.ConsolidatedValue) }) t.Run("should consolidate 3 utxos sparse", func(t *testing.T) { @@ -432,16 +432,16 @@ func TestUTXOConsolidation(t *testing.T) { // input: utxoCap = 5, amount = 0.13, nonce = 24105432, rank = 5 // output: [0.24107431, 0.01, 0.12, 1.26, 0.5, 0.24], 2.37107431 - result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(ctx, 0.13, 5, 24105432, 5, true) + res, err := ob.SelectUTXOs(ctx, 0.13, 5, 24105432, 5, true) require.NoError(t, err) - require.InEpsilon(t, 2.37107431, amount, 1e-8) + require.InEpsilon(t, 2.37107431, res.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...) expected = append(expected, ob.utxos[6]) expected = append(expected, ob.utxos[5]) expected = append(expected, ob.utxos[3]) - require.Equal(t, expected, result) - require.Equal(t, uint16(3), clsdtUtxo) - require.Equal(t, 2.0, clsdtValue) + require.Equal(t, expected, res.UTXOs) + require.Equal(t, uint16(3), res.ConsolidatedUTXOs) + require.Equal(t, 2.0, res.ConsolidatedValue) }) t.Run("should consolidate all utxos sparse", func(t *testing.T) { @@ -456,17 +456,17 @@ func TestUTXOConsolidation(t *testing.T) { // input: utxoCap = 12, amount = 0.13, nonce = 24105432, rank = 1 // output: [0.24107431, 0.01, 0.12, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18], 22.68107431 - result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(ctx, 0.13, 12, 24105432, 1, true) + res, err := ob.SelectUTXOs(ctx, 0.13, 12, 24105432, 1, true) require.NoError(t, err) - require.InEpsilon(t, 22.68107431, amount, 1e-8) + require.InEpsilon(t, 22.68107431, res.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...) for i := 10; i >= 5; i-- { // append consolidated utxos in descending order expected = append(expected, ob.utxos[i]) } expected = append(expected, ob.utxos[3]) expected = append(expected, ob.utxos[2]) - require.Equal(t, expected, result) - require.Equal(t, uint16(8), clsdtUtxo) - require.Equal(t, 22.31, clsdtValue) + require.Equal(t, expected, res.UTXOs) + require.Equal(t, uint16(8), res.ConsolidatedUTXOs) + require.Equal(t, 22.31, res.ConsolidatedValue) }) } diff --git a/zetaclient/chains/bitcoin/observer/utxos.go b/zetaclient/chains/bitcoin/observer/utxos.go new file mode 100644 index 0000000000..3413d62511 --- /dev/null +++ b/zetaclient/chains/bitcoin/observer/utxos.go @@ -0,0 +1,199 @@ +package observer + +import ( + "context" + "fmt" + "sort" + + "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/btcutil" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" +) + +// SelectedUTXOs is a struct containing the selected UTXOs' details. +type SelectedUTXOs struct { + // A sublist of UTXOs selected for the outbound transaction. + UTXOs []btcjson.ListUnspentResult + + // The total value of the selected UTXOs. + Value float64 + + // The number of consolidated UTXOs. + ConsolidatedUTXOs uint16 + + // The total value of the consolidated UTXOs. + ConsolidatedValue float64 +} + +// FetchUTXOs fetches TSS-owned UTXOs from the Bitcoin node +func (ob *Observer) FetchUTXOs(ctx context.Context) error { + defer func() { + if err := recover(); err != nil { + ob.logger.UTXOs.Error().Msgf("BTC FetchUTXOs: caught panic error: %v", err) + } + }() + + // this is useful when a zetaclient's pending nonce lagged behind for whatever reason. + ob.refreshPendingNonce(ctx) + + // list all unspent UTXOs (160ms) + tssAddr, err := ob.TSS().PubKey().AddressBTC(ob.Chain().ChainId) + if err != nil { + return fmt.Errorf("error getting bitcoin tss address") + } + utxos, err := ob.rpc.ListUnspentMinMaxAddresses(ctx, 0, 9999999, []btcutil.Address{tssAddr}) + if err != nil { + return err + } + + // rigid sort to make utxo list deterministic + sort.SliceStable(utxos, func(i, j int) bool { + if utxos[i].Amount == utxos[j].Amount { + if utxos[i].TxID == utxos[j].TxID { + return utxos[i].Vout < utxos[j].Vout + } + return utxos[i].TxID < utxos[j].TxID + } + return utxos[i].Amount < utxos[j].Amount + }) + + // filter UTXOs good to spend for next TSS transaction + utxosFiltered := make([]btcjson.ListUnspentResult, 0) + for _, utxo := range utxos { + // UTXOs big enough to cover the cost of spending themselves + if utxo.Amount < common.DefaultDepositorFee { + continue + } + // we don't want to spend other people's unconfirmed UTXOs as they may not be safe to spend + if utxo.Confirmations == 0 { + if !ob.IsTSSTransaction(utxo.TxID) { + continue + } + } + utxosFiltered = append(utxosFiltered, utxo) + } + + ob.Mu().Lock() + ob.TelemetryServer().SetNumberOfUTXOs(len(utxosFiltered)) + ob.utxos = utxosFiltered + ob.Mu().Unlock() + return nil +} + +// SelectUTXOs selects a sublist of utxos to be used as inputs. +// +// Parameters: +// - amount: The desired minimum total value of the selected UTXOs. +// - utxos2Spend: The maximum number of UTXOs to spend. +// - nonce: The nonce of the outbound transaction. +// - consolidateRank: The rank below which UTXOs will be consolidated. +// - test: true for unit test only. +// +// Returns: a sublist (includes previous nonce-mark) of UTXOs or an error if the qualifying sublist cannot be found. +func (ob *Observer) SelectUTXOs( + ctx context.Context, + amount float64, + utxosToSpend uint16, + nonce uint64, + consolidateRank uint16, + test bool, +) (SelectedUTXOs, error) { + idx := -1 + if nonce == 0 { + // for nonce = 0; make exception; no need to include nonce-mark utxo + ob.Mu().Lock() + defer ob.Mu().Unlock() + } else { + // for nonce > 0; we proceed only when we see the nonce-mark utxo + preTxid, err := ob.getOutboundHashByNonce(ctx, nonce-1, test) + if err != nil { + return SelectedUTXOs{}, err + } + ob.Mu().Lock() + defer ob.Mu().Unlock() + idx, err = ob.findNonceMarkUTXO(nonce-1, preTxid) + if err != nil { + return SelectedUTXOs{}, err + } + } + + // select smallest possible UTXOs to make payment + total := 0.0 + left, right := 0, 0 + for total < amount && right < len(ob.utxos) { + if utxosToSpend > 0 { // expand sublist + total += ob.utxos[right].Amount + right++ + utxosToSpend-- + } else { // pop the smallest utxo and append the current one + total -= ob.utxos[left].Amount + total += ob.utxos[right].Amount + left++ + right++ + } + } + results := make([]btcjson.ListUnspentResult, right-left) + copy(results, ob.utxos[left:right]) + + // include nonce-mark as the 1st input + if idx >= 0 { // for nonce > 0 + if idx < left || idx >= right { + total += ob.utxos[idx].Amount + results = append([]btcjson.ListUnspentResult{ob.utxos[idx]}, results...) + } else { // move nonce-mark to left + for i := idx - left; i > 0; i-- { + results[i], results[i-1] = results[i-1], results[i] + } + } + } + if total < amount { + return SelectedUTXOs{}, fmt.Errorf( + "SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", + total, + amount, + ) + } + + // consolidate biggest possible UTXOs to maximize consolidated value + // consolidation happens only when there are more than (or equal to) consolidateRank (10) UTXOs + utxoRank, consolidatedUtxo, consolidatedValue := uint16(0), uint16(0), 0.0 + for i := len(ob.utxos) - 1; i >= 0 && utxosToSpend > 0; i-- { // iterate over UTXOs big-to-small + if i != idx && (i < left || i >= right) { // exclude nonce-mark and already selected UTXOs + utxoRank++ + if utxoRank >= consolidateRank { // consolication starts from the 10-ranked UTXO based on value + utxosToSpend-- + consolidatedUtxo++ + total += ob.utxos[i].Amount + consolidatedValue += ob.utxos[i].Amount + results = append(results, ob.utxos[i]) + } + } + } + + return SelectedUTXOs{ + UTXOs: results, + Value: total, + ConsolidatedUTXOs: consolidatedUtxo, + ConsolidatedValue: consolidatedValue, + }, nil +} + +// findNonceMarkUTXO finds the nonce-mark UTXO in the list of UTXOs. +func (ob *Observer) findNonceMarkUTXO(nonce uint64, txid string) (int, error) { + tssAddress := ob.TSSAddressString() + amount := chains.NonceMarkAmount(nonce) + for i, utxo := range ob.utxos { + sats, err := common.GetSatoshis(utxo.Amount) + if err != nil { + ob.logger.Outbound.Error().Err(err).Msgf("FindNonceMarkUTXO: error getting satoshis for utxo %v", utxo) + } + if utxo.Address == tssAddress && sats == amount && utxo.TxID == txid && utxo.Vout == 0 { + ob.logger.Outbound.Info(). + Msgf("FindNonceMarkUTXO: found nonce-mark utxo with txid %s, amount %d satoshi", utxo.TxID, sats) + return i, nil + } + } + return -1, fmt.Errorf("FindNonceMarkUTXO: cannot find nonce-mark utxo with nonce %d", nonce) +} diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go new file mode 100644 index 0000000000..fdffadc602 --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -0,0 +1,137 @@ +package signer + +import ( + "fmt" + "math" + "strconv" + + "github.com/btcsuite/btcd/btcutil" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/coin" + "github.com/zeta-chain/node/pkg/constant" + "github.com/zeta-chain/node/x/crosschain/types" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" + "github.com/zeta-chain/node/zetaclient/compliance" +) + +// OutboundData is a data structure containing necessary data to construct a BTC outbound transaction +type OutboundData struct { + // chainID is the external chain ID + chainID int64 + + // to is the recipient address + to btcutil.Address + + // amount is the amount in BTC + amount float64 + + // amountSats is the amount in satoshis + amountSats int64 + + // feeRate is the fee rate in satoshis/vByte + feeRate int64 + + // txSize is the average size of a BTC outbound transaction + // user is charged (in ZRC20 contract) at a static txSize on each withdrawal + txSize int64 + + // nonce is the nonce of the outbound + nonce uint64 + + // height is the ZetaChain block height + height uint64 + + // cancelTx is a flag to indicate if this outbound should be cancelled + cancelTx bool +} + +// NewOutboundData creates OutboundData from the given CCTX. +func NewOutboundData( + cctx *types.CrossChainTx, + chainID int64, + height uint64, + minRelayFee float64, + logger, loggerCompliance zerolog.Logger, +) (*OutboundData, error) { + if cctx == nil { + return nil, errors.New("cctx is nil") + } + params := cctx.GetCurrentOutboundParam() + + // support gas token only for Bitcoin outbound + if cctx.InboundParams.CoinType != coin.CoinType_Gas { + return nil, errors.New("can only send gas token to a Bitcoin network") + } + + // initial fee rate + feeRate, err := strconv.ParseInt(params.GasPrice, 10, 64) + if err != nil || feeRate <= 0 { + return nil, fmt.Errorf("invalid fee rate %s", params.GasPrice) + } + + // use current gas rate if fed by zetacore + newRate, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) + if err == nil && newRate > 0 && newRate != feeRate { + logger.Info().Msgf("use new fee rate %d sat/vB instead of %d sat/vB", newRate, feeRate) + feeRate = newRate + } + + // check receiver address + to, err := chains.DecodeBtcAddress(params.Receiver, params.ReceiverChainId) + if err != nil { + return nil, errors.Wrapf(err, "cannot decode receiver address %s", params.Receiver) + } + if !chains.IsBtcAddressSupported(to) { + return nil, fmt.Errorf("unsupported receiver address %s", params.Receiver) + } + + // amount in BTC and satoshis + amount := float64(params.Amount.Uint64()) / 1e8 + amountSats := params.Amount.BigInt().Int64() + + // check gas limit + if params.CallOptions.GasLimit > math.MaxInt64 { + return nil, fmt.Errorf("invalid gas limit %d", params.CallOptions.GasLimit) + } + + // add minimum relay fee (1000 satoshis/KB by default) to gasPrice to avoid minRelayTxFee error + // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/policy.h#L35 + satPerByte := client.FeeRateToSatPerByte(minRelayFee) + feeRate += satPerByte.Int64() + + // compliance check + restrictedCCTX := compliance.IsCctxRestricted(cctx) + if restrictedCCTX { + compliance.PrintComplianceLog(logger, loggerCompliance, + true, chainID, cctx.Index, cctx.InboundParams.Sender, params.Receiver, "BTC") + } + + // check dust amount + dustAmount := params.Amount.Uint64() < constant.BTCWithdrawalDustAmount + if dustAmount { + logger.Warn().Msgf("dust amount %d sats, canceling tx", params.Amount.Uint64()) + } + + // set the amount to 0 when the tx should be cancelled + cancelTx := restrictedCCTX || dustAmount + if cancelTx { + amount = 0.0 + amountSats = 0 + } + + return &OutboundData{ + chainID: chainID, + to: to, + amount: amount, + amountSats: amountSats, + feeRate: feeRate, + // #nosec G115 checked in range + txSize: int64(params.CallOptions.GasLimit), + nonce: params.TssNonce, + height: height, + cancelTx: cancelTx, + }, nil +} diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go new file mode 100644 index 0000000000..55b4eb7349 --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -0,0 +1,241 @@ +package signer + +import ( + "math" + "testing" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/coin" + "github.com/zeta-chain/node/pkg/constant" + "github.com/zeta-chain/node/testutil/sample" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" + "github.com/zeta-chain/node/zetaclient/config" +) + +func Test_NewOutboundData(t *testing.T) { + // sample address + chain := chains.BitcoinMainnet + receiver, err := chains.DecodeBtcAddress("bc1qaxf82vyzy8y80v000e7t64gpten7gawewzu42y", chain.ChainId) + require.NoError(t, err) + + // setup compliance config + cfg := config.Config{ + ComplianceConfig: sample.ComplianceConfig(), + } + config.LoadComplianceConfig(cfg) + + // test cases + tests := []struct { + name string + cctx *crosschaintypes.CrossChainTx + cctxModifier func(cctx *crosschaintypes.CrossChainTx) + chainID int64 + height uint64 + minRelayFee float64 + expected *OutboundData + errMsg string + }{ + { + name: "create new outbound data successfully", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().Receiver = receiver.String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId + cctx.GetCurrentOutboundParam().Amount = sdk.NewUint(1e7) // 0.1 BTC + cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes + cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().TssNonce = 1 + }, + chainID: chain.ChainId, + height: 101, + minRelayFee: 0.00001, // 1000 sat/KB + expected: &OutboundData{ + chainID: chain.ChainId, + to: receiver, + amount: 0.1, + amountSats: 10000000, + feeRate: 11, // 10 + 1 (minRelayFee) + txSize: 254, + nonce: 1, + height: 101, + cancelTx: false, + }, + errMsg: "", + }, + { + name: "create new outbound data using current gas rate instead of old rate", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().Receiver = receiver.String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId + cctx.GetCurrentOutboundParam().Amount = sdk.NewUint(1e7) // 0.1 BTC + cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes + cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().GasPriorityFee = "15" // 15 sats/vByte + cctx.GetCurrentOutboundParam().TssNonce = 1 + }, + chainID: chain.ChainId, + height: 101, + minRelayFee: 0.00001, // 1000 sat/KB + expected: &OutboundData{ + chainID: chain.ChainId, + to: receiver, + amount: 0.1, + amountSats: 10000000, + feeRate: 16, // 15 + 1 (minRelayFee) + txSize: 254, + nonce: 1, + height: 101, + cancelTx: false, + }, + errMsg: "", + }, + { + name: "cctx is nil", + cctx: nil, + cctxModifier: nil, + expected: nil, + errMsg: "cctx is nil", + }, + { + name: "coin type is not gas", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_ERC20 + }, + expected: nil, + errMsg: "can only send gas token to a Bitcoin network", + }, + { + name: "invalid gas price", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().GasPrice = "invalid" + }, + expected: nil, + errMsg: "invalid fee rate", + }, + { + name: "zero fee rate", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().GasPrice = "0" + }, + expected: nil, + errMsg: "invalid fee rate", + }, + { + name: "invalid receiver address", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().Receiver = "invalid" + }, + expected: nil, + errMsg: "cannot decode receiver address", + }, + { + name: "unsupported receiver address", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().Receiver = "035e4ae279bd416b5da724972c9061ec6298dac020d1e3ca3f06eae715135cdbec" + cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId + }, + expected: nil, + errMsg: "unsupported receiver address", + }, + { + name: "invalid gas limit", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().Receiver = receiver.String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId + cctx.GetCurrentOutboundParam().CallOptions.GasLimit = math.MaxInt64 + 1 + }, + expected: nil, + errMsg: "invalid gas limit", + }, + { + name: "should cancel restricted CCTX", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.InboundParams.Sender = sample.RestrictedEVMAddressTest + cctx.GetCurrentOutboundParam().Receiver = receiver.String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId + cctx.GetCurrentOutboundParam().Amount = sdk.NewUint(1e7) // 0.1 BTC + cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes + cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().TssNonce = 1 + }, + chainID: chain.ChainId, + height: 101, + minRelayFee: 0.00001, // 1000 sat/KB + expected: &OutboundData{ + chainID: chain.ChainId, + to: receiver, + amount: 0, // should cancel the tx + amountSats: 0, + feeRate: 11, // 10 + 1 (minRelayFee) + txSize: 254, + nonce: 1, + height: 101, + cancelTx: true, + }, + }, + { + name: "should cancel dust amount CCTX", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().Receiver = receiver.String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId + cctx.GetCurrentOutboundParam().Amount = sdk.NewUint(constant.BTCWithdrawalDustAmount - 1) + cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes + cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().TssNonce = 1 + }, + chainID: chain.ChainId, + height: 101, + minRelayFee: 0.00001, // 1000 sat/KB + expected: &OutboundData{ + chainID: chain.ChainId, + to: receiver, + amount: 0, // should cancel the tx + amountSats: 0, + feeRate: 11, // 10 + 1 (minRelayFee) + txSize: 254, + nonce: 1, + height: 101, + cancelTx: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // modify cctx if needed + if tt.cctxModifier != nil { + tt.cctxModifier(tt.cctx) + } + + outboundData, err := NewOutboundData(tt.cctx, tt.chainID, tt.height, tt.minRelayFee, log.Logger, log.Logger) + if tt.errMsg != "" { + require.Nil(t, outboundData) + require.ErrorContains(t, err, tt.errMsg) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, outboundData) + } + }) + } +} diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go new file mode 100644 index 0000000000..d20ca9fb0e --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -0,0 +1,246 @@ +package signer + +import ( + "context" + "fmt" + + "github.com/btcsuite/btcd/btcec/v2" + btcecdsa "github.com/btcsuite/btcd/btcec/v2/ecdsa" + "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/pkg/errors" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/constant" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" +) + +const ( + // the maximum number of inputs per outbound + MaxNoOfInputsPerTx = 20 + + // the rank below (or equal to) which we consolidate UTXOs + consolidationRank = 10 +) + +// SignWithdrawTx signs a BTC withdrawal tx and returns the signed tx +func (signer *Signer) SignWithdrawTx( + ctx context.Context, + txData *OutboundData, + ob *observer.Observer, +) (*wire.MsgTx, error) { + nonceMark := chains.NonceMarkAmount(txData.nonce) + estimateFee := float64(txData.feeRate*common.OutboundBytesMax) / 1e8 + totalAmount := txData.amount + estimateFee + float64(nonceMark)*1e-8 + + // refresh unspent UTXOs and continue with keysign regardless of error + if err := ob.FetchUTXOs(ctx); err != nil { + signer.Logger().Std.Error().Err(err).Uint64("nonce", txData.nonce).Msg("SignWithdrawTx: FetchUTXOs failed") + } + + // select N UTXOs to cover the total expense + selected, err := ob.SelectUTXOs( + ctx, + totalAmount, + MaxNoOfInputsPerTx, + txData.nonce, + consolidationRank, + false, + ) + if err != nil { + return nil, err + } + + // build tx and add inputs + tx := wire.NewMsgTx(wire.TxVersion) + inAmounts, err := signer.AddTxInputs(tx, selected.UTXOs) + if err != nil { + return nil, err + } + + // size checking + // #nosec G115 always positive + txSize, err := common.EstimateOutboundSize(int64(len(selected.UTXOs)), []btcutil.Address{txData.to}) + if err != nil { + return nil, err + } + if txData.txSize < common.BtcOutboundBytesWithdrawer { // ZRC20 'withdraw' charged less fee from end user + signer.Logger().Std.Info(). + Msgf("txSize %d is less than BtcOutboundBytesWithdrawer %d for nonce %d", txData.txSize, txSize, txData.nonce) + } + if txSize < common.OutboundBytesMin { // outbound shouldn't be blocked by low sizeLimit + signer.Logger().Std.Warn(). + Msgf("txSize %d is less than outboundBytesMin %d; use outboundBytesMin", txSize, common.OutboundBytesMin) + txSize = common.OutboundBytesMin + } + if txSize > common.OutboundBytesMax { // in case of accident + signer.Logger().Std.Warn(). + Msgf("txSize %d is greater than outboundBytesMax %d; use outboundBytesMax", txSize, common.OutboundBytesMax) + txSize = common.OutboundBytesMax + } + + // fee calculation + // #nosec G115 always in range (checked above) + fees := txSize * txData.feeRate + signer.Logger(). + Std.Info(). + Msgf("bitcoin outbound nonce %d feeRate %d size %d fees %d consolidated %d utxos of value %v", + txData.nonce, txData.feeRate, txSize, fees, selected.ConsolidatedUTXOs, selected.ConsolidatedValue) + + // add tx outputs + err = signer.AddWithdrawTxOutputs( + tx, + txData.to, + selected.Value, + txData.amountSats, + nonceMark, + fees, + txData.cancelTx, + ) + if err != nil { + return nil, err + } + + // sign the tx + err = signer.SignTx(ctx, tx, inAmounts, txData.height, txData.nonce) + if err != nil { + return nil, errors.Wrap(err, "SignTx failed") + } + + return tx, nil +} + +// AddTxInputs adds the inputs to the tx and returns input amounts +func (signer *Signer) AddTxInputs(tx *wire.MsgTx, utxos []btcjson.ListUnspentResult) ([]int64, error) { + amounts := make([]int64, len(utxos)) + for i, utxo := range utxos { + hash, err := chainhash.NewHashFromStr(utxo.TxID) + if err != nil { + return nil, err + } + + outpoint := wire.NewOutPoint(hash, utxo.Vout) + txIn := wire.NewTxIn(outpoint, nil, nil) + tx.AddTxIn(txIn) + + // store the amount for later signing use + amount, err := common.GetSatoshis(utxos[i].Amount) + if err != nil { + return nil, err + } + amounts[i] = amount + } + + return amounts, nil +} + +// AddWithdrawTxOutputs adds the 3 outputs to the withdraw tx +// 1st output: the nonce-mark btc to TSS itself +// 2nd output: the payment to the recipient +// 3rd output: the remaining btc to TSS itself +func (signer *Signer) AddWithdrawTxOutputs( + tx *wire.MsgTx, + to btcutil.Address, + inputValue float64, + amountSats int64, + nonceMark int64, + fees int64, + cancelTx bool, +) error { + // convert withdraw amount to BTC + amount := float64(amountSats) / 1e8 + + // calculate remaining btc (the change) to TSS self + remaining := inputValue - amount + remainingSats, err := common.GetSatoshis(remaining) + if err != nil { + return err + } + remainingSats -= fees + remainingSats -= nonceMark + if remainingSats < 0 { + return fmt.Errorf("remainder value is negative: %d", remainingSats) + } else if remainingSats == nonceMark { + signer.Logger().Std.Info().Msgf("adjust remainder value to avoid duplicate nonce-mark: %d", remainingSats) + remainingSats-- + } + + // 1st output: the nonce-mark btc to TSS self + payToSelfScript, err := signer.TSSToPkScript() + if err != nil { + return err + } + txOut1 := wire.NewTxOut(nonceMark, payToSelfScript) + tx.AddTxOut(txOut1) + + // 2nd output: the payment to the recipient + if !cancelTx { + pkScript, err := txscript.PayToAddrScript(to) + if err != nil { + return err + } + txOut2 := wire.NewTxOut(amountSats, pkScript) + tx.AddTxOut(txOut2) + } else { + // send the amount to TSS self if tx is cancelled + remainingSats += amountSats + } + + // 3rd output: the remaining btc to TSS self + if remainingSats >= constant.BTCWithdrawalDustAmount { + txOut3 := wire.NewTxOut(remainingSats, payToSelfScript) + tx.AddTxOut(txOut3) + } + return nil +} + +// SignTx signs the given tx with TSS +func (signer *Signer) SignTx( + ctx context.Context, + tx *wire.MsgTx, + inputAmounts []int64, + height uint64, + nonce uint64, +) error { + pkScript, err := signer.TSSToPkScript() + if err != nil { + return err + } + + // calculate sighashes to sign + sigHashes := txscript.NewTxSigHashes(tx, txscript.NewCannedPrevOutputFetcher([]byte{}, 0)) + witnessHashes := make([][]byte, len(tx.TxIn)) + for ix := range tx.TxIn { + amount := inputAmounts[ix] + witnessHashes[ix], err = txscript.CalcWitnessSigHash(pkScript, sigHashes, txscript.SigHashAll, tx, ix, amount) + if err != nil { + return err + } + } + + // sign the tx with TSS + sig65Bs, err := signer.TSS().SignBatch(ctx, witnessHashes, height, nonce, signer.Chain().ChainId) + if err != nil { + return fmt.Errorf("SignBatch failed: %v", err) + } + + for ix := range tx.TxIn { + sig65B := sig65Bs[ix] + R := &btcec.ModNScalar{} + R.SetBytes((*[32]byte)(sig65B[:32])) + S := &btcec.ModNScalar{} + S.SetBytes((*[32]byte)(sig65B[32:64])) + sig := btcecdsa.NewSignature(R, S) + + pkCompressed := signer.TSS().PubKey().Bytes(true) + hashType := txscript.SigHashAll + txWitness := wire.TxWitness{append(sig.Serialize(), byte(hashType)), pkCompressed} + tx.TxIn[ix].Witness = txWitness + } + + return nil +} diff --git a/zetaclient/chains/bitcoin/signer/sign_test.go b/zetaclient/chains/bitcoin/signer/sign_test.go new file mode 100644 index 0000000000..f550204b0f --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/sign_test.go @@ -0,0 +1,326 @@ +package signer_test + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/testutil/sample" + "github.com/zeta-chain/node/zetaclient/testutils" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/zetaclient/chains/base" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" + "github.com/zeta-chain/node/zetaclient/testutils/mocks" +) + +func Test_AddTxInputs(t *testing.T) { + net := &chaincfg.MainNetParams + + tests := []struct { + name string + utxos []btcjson.ListUnspentResult + expectedAmounts []int64 + fail bool + }{ + { + name: "should add tx inputs successfully", + utxos: []btcjson.ListUnspentResult{ + { + TxID: sample.BtcHash().String(), + Vout: 0, + Address: sample.BtcAddressP2WPKH(t, net).String(), + Amount: 0.1, + }, + { + TxID: sample.BtcHash().String(), + Vout: 1, + Address: sample.BtcAddressP2WPKH(t, net).String(), + Amount: 0.2, + }, + }, + expectedAmounts: []int64{10000000, 20000000}, + }, + { + name: "should fail on invalid txid", + utxos: []btcjson.ListUnspentResult{ + { + TxID: "invalid txid", + }, + }, + fail: true, + }, + { + name: "should fail on invalid amount", + utxos: []btcjson.ListUnspentResult{ + { + TxID: sample.BtcHash().String(), + Amount: -0.1, + }, + }, + fail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // setup signer + s := newTestSuite(t, chains.BitcoinMainnet) + + // create tx msg and add inputs + tx := wire.NewMsgTx(wire.TxVersion) + inAmounts, err := s.AddTxInputs(tx, tt.utxos) + + // assert + if tt.fail { + require.Error(t, err) + require.Nil(t, inAmounts) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedAmounts, inAmounts) + } + }) + } +} + +func Test_AddWithdrawTxOutputs(t *testing.T) { + // Create test signer and receiver address + signer := signer.New( + chains.BitcoinMainnet, + mocks.NewTSS(t).FakePubKey(testutils.TSSPubKeyMainnet), + mocks.NewBitcoinClient(t), + base.DefaultLogger(), + ) + + // tss address and script + tssAddr, err := signer.TSS().PubKey().AddressBTC(chains.BitcoinMainnet.ChainId) + require.NoError(t, err) + tssScript, err := txscript.PayToAddrScript(tssAddr) + require.NoError(t, err) + fmt.Printf("tss address: %s", tssAddr.EncodeAddress()) + + // receiver addresses + receiver := "bc1qaxf82vyzy8y80v000e7t64gpten7gawewzu42y" + to, err := chains.DecodeBtcAddress(receiver, chains.BitcoinMainnet.ChainId) + require.NoError(t, err) + toScript, err := txscript.PayToAddrScript(to) + require.NoError(t, err) + + // test cases + tests := []struct { + name string + tx *wire.MsgTx + to btcutil.Address + total float64 + amountSats int64 + nonceMark int64 + fees int64 + cancelTx bool + fail bool + message string + txout []*wire.TxOut + }{ + { + name: "should add outputs successfully", + tx: wire.NewMsgTx(wire.TxVersion), + to: to, + total: 1.00012000, + amountSats: 20000000, + nonceMark: 10000, + fees: 2000, + fail: false, + txout: []*wire.TxOut{ + {Value: 10000, PkScript: tssScript}, + {Value: 20000000, PkScript: toScript}, + {Value: 80000000, PkScript: tssScript}, + }, + }, + { + name: "should add outputs without change successfully", + tx: wire.NewMsgTx(wire.TxVersion), + to: to, + total: 0.20012000, + amountSats: 20000000, + nonceMark: 10000, + fees: 2000, + fail: false, + txout: []*wire.TxOut{ + {Value: 10000, PkScript: tssScript}, + {Value: 20000000, PkScript: toScript}, + }, + }, + { + name: "should cancel tx successfully", + tx: wire.NewMsgTx(wire.TxVersion), + to: to, + total: 1.00012000, + amountSats: 20000000, + nonceMark: 10000, + fees: 2000, + cancelTx: true, + fail: false, + txout: []*wire.TxOut{ + {Value: 10000, PkScript: tssScript}, + {Value: 100000000, PkScript: tssScript}, + }, + }, + { + name: "should fail when total < amount", + tx: wire.NewMsgTx(wire.TxVersion), + to: to, + total: 0.00012000, + amountSats: 20000000, + fail: true, + }, + { + name: "should fail when total < fees + amount + nonce", + tx: wire.NewMsgTx(wire.TxVersion), + to: to, + total: 0.20011000, + amountSats: 20000000, + nonceMark: 10000, + fees: 2000, + fail: true, + message: "remainder value is negative", + }, + { + name: "should not produce duplicate nonce mark", + tx: wire.NewMsgTx(wire.TxVersion), + to: to, + total: 0.20022000, // 0.2 + fee + nonceMark * 2 + amountSats: 20000000, + nonceMark: 10000, + fees: 2000, + fail: false, + txout: []*wire.TxOut{ + {Value: 10000, PkScript: tssScript}, + {Value: 20000000, PkScript: toScript}, + {Value: 9999, PkScript: tssScript}, // nonceMark - 1 + }, + }, + { + name: "should not produce dust change to TSS self", + tx: wire.NewMsgTx(wire.TxVersion), + to: to, + total: 0.20012999, // 0.2 + fee + nonceMark + amountSats: 20000000, + nonceMark: 10000, + fees: 2000, + fail: false, + txout: []*wire.TxOut{ // 3rd output 999 is dust and removed + {Value: 10000, PkScript: tssScript}, + {Value: 20000000, PkScript: toScript}, + }, + }, + { + name: "should fail on invalid to address", + tx: wire.NewMsgTx(wire.TxVersion), + to: nil, + total: 1.00012000, + amountSats: 20000000, + nonceMark: 10000, + fees: 2000, + fail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := signer.AddWithdrawTxOutputs( + tt.tx, + tt.to, + tt.total, + tt.amountSats, + tt.nonceMark, + tt.fees, + tt.cancelTx, + ) + if tt.fail { + require.Error(t, err) + if tt.message != "" { + require.ErrorContains(t, err, tt.message) + } + return + } else { + require.NoError(t, err) + require.True(t, reflect.DeepEqual(tt.txout, tt.tx.TxOut)) + } + }) + } +} + +func Test_SignTx(t *testing.T) { + tests := []struct { + name string + chain chains.Chain + net *chaincfg.Params + inputs []float64 + outputs []int64 + height uint64 + nonce uint64 + }{ + { + name: "should sign tx successfully", + chain: chains.BitcoinMainnet, + net: &chaincfg.MainNetParams, + inputs: []float64{ + 0.0001, + 0.0002, + }, + outputs: []int64{ + 5000, + 20000, + }, + nonce: 100, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // setup signer + s := newTestSuite(t, tt.chain) + address, err := s.TSS().PubKey().AddressBTC(tt.chain.ChainId) + require.NoError(t, err) + + // create tx msg + tx := wire.NewMsgTx(wire.TxVersion) + + // add inputs + utxos := []btcjson.ListUnspentResult{} + for i, amount := range tt.inputs { + utxos = append(utxos, btcjson.ListUnspentResult{ + TxID: sample.BtcHash().String(), + Vout: uint32(i), + Address: address.EncodeAddress(), + Amount: amount, + }) + } + inAmounts, err := s.AddTxInputs(tx, utxos) + require.NoError(t, err) + require.Len(t, inAmounts, len(tt.inputs)) + + // add outputs + for _, amount := range tt.outputs { + pkScript := sample.BtcAddressP2WPKHScript(t, tt.net) + tx.AddTxOut(wire.NewTxOut(amount, pkScript)) + } + + // sign tx + ctx := context.Background() + err = s.SignTx(ctx, tx, inAmounts, tt.height, tt.nonce) + require.NoError(t, err) + + // check tx signature + for i := range tx.TxIn { + require.Len(t, tx.TxIn[i].Witness, 2) + } + }) + } +} diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index b59d9f1232..99289acc03 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -5,12 +5,8 @@ import ( "bytes" "context" "encoding/hex" - "fmt" - "math/big" "time" - "github.com/btcsuite/btcd/btcec/v2" - btcecdsa "github.com/btcsuite/btcd/btcec/v2/ecdsa" "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg/chainhash" @@ -19,26 +15,15 @@ import ( "github.com/pkg/errors" "github.com/zeta-chain/node/pkg/chains" - "github.com/zeta-chain/node/pkg/coin" - "github.com/zeta-chain/node/pkg/constant" "github.com/zeta-chain/node/x/crosschain/types" - observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/node/zetaclient/chains/interfaces" - "github.com/zeta-chain/node/zetaclient/compliance" "github.com/zeta-chain/node/zetaclient/logs" "github.com/zeta-chain/node/zetaclient/outboundprocessor" ) const ( - // the maximum number of inputs per outbound - MaxNoOfInputsPerTx = 20 - - // the rank below (or equal to) which we consolidate UTXOs - consolidationRank = 10 - // broadcastBackoff is the initial backoff duration for retrying broadcast broadcastBackoff = 1000 * time.Millisecond @@ -48,6 +33,8 @@ const ( type RPC interface { GetNetworkInfo(ctx context.Context) (*btcjson.GetNetworkInfoResult, error) + GetRawTransaction(ctx context.Context, hash *chainhash.Hash) (*btcutil.Tx, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*chainhash.Hash, error) } @@ -65,196 +52,6 @@ func New(chain chains.Chain, tss interfaces.TSSSigner, rpc RPC, logger base.Logg } } -// AddWithdrawTxOutputs adds the 3 outputs to the withdraw tx -// 1st output: the nonce-mark btc to TSS itself -// 2nd output: the payment to the recipient -// 3rd output: the remaining btc to TSS itself -func (signer *Signer) AddWithdrawTxOutputs( - tx *wire.MsgTx, - to btcutil.Address, - total float64, - amount float64, - nonceMark int64, - fees *big.Int, - cancelTx bool, -) error { - // convert withdraw amount to satoshis - amountSatoshis, err := common.GetSatoshis(amount) - if err != nil { - return err - } - - // calculate remaining btc (the change) to TSS self - remaining := total - amount - remainingSats, err := common.GetSatoshis(remaining) - if err != nil { - return err - } - remainingSats -= fees.Int64() - remainingSats -= nonceMark - if remainingSats < 0 { - return fmt.Errorf("remainder value is negative: %d", remainingSats) - } else if remainingSats == nonceMark { - signer.Logger().Std.Info().Msgf("adjust remainder value to avoid duplicate nonce-mark: %d", remainingSats) - remainingSats-- - } - - // 1st output: the nonce-mark btc to TSS self - tssAddrP2WPKH, err := signer.TSS().PubKey().AddressBTC(signer.Chain().ChainId) - if err != nil { - return err - } - payToSelfScript, err := txscript.PayToAddrScript(tssAddrP2WPKH) - if err != nil { - return err - } - txOut1 := wire.NewTxOut(nonceMark, payToSelfScript) - tx.AddTxOut(txOut1) - - // 2nd output: the payment to the recipient - if !cancelTx { - pkScript, err := txscript.PayToAddrScript(to) - if err != nil { - return err - } - txOut2 := wire.NewTxOut(amountSatoshis, pkScript) - tx.AddTxOut(txOut2) - } else { - // send the amount to TSS self if tx is cancelled - remainingSats += amountSatoshis - } - - // 3rd output: the remaining btc to TSS self - if remainingSats > 0 { - txOut3 := wire.NewTxOut(remainingSats, payToSelfScript) - tx.AddTxOut(txOut3) - } - return nil -} - -// SignWithdrawTx receives utxos sorted by value, amount in BTC, feeRate in BTC per Kb -// TODO(revamp): simplify the function -func (signer *Signer) SignWithdrawTx( - ctx context.Context, - to btcutil.Address, - amount float64, - gasPrice *big.Int, - sizeLimit uint64, - observer *observer.Observer, - height uint64, - nonce uint64, - chain chains.Chain, - cancelTx bool, -) (*wire.MsgTx, error) { - estimateFee := float64(gasPrice.Uint64()*common.OutboundBytesMax) / 1e8 - nonceMark := chains.NonceMarkAmount(nonce) - - // refresh unspent UTXOs and continue with keysign regardless of error - if err := observer.FetchUTXOs(ctx); err != nil { - signer.Logger().Std.Error().Err(err).Uint64("nonce", nonce).Msg("SignWithdrawTx: FetchUTXOs failed") - } - - // select N UTXOs to cover the total expense - prevOuts, total, consolidatedUtxo, consolidatedValue, err := observer.SelectUTXOs( - ctx, - amount+estimateFee+float64(nonceMark)*1e-8, - MaxNoOfInputsPerTx, - nonce, - consolidationRank, - false, - ) - if err != nil { - return nil, errors.Wrap(err, "unable to select UTXOs") - } - - // build tx with selected unspents - tx := wire.NewMsgTx(wire.TxVersion) - for _, prevOut := range prevOuts { - hash, err := chainhash.NewHashFromStr(prevOut.TxID) - if err != nil { - return nil, errors.Wrap(err, "unable to construct hash") - } - - outpoint := wire.NewOutPoint(hash, prevOut.Vout) - txIn := wire.NewTxIn(outpoint, nil, nil) - tx.AddTxIn(txIn) - } - - // size checking - // #nosec G115 always positive - txSize, err := common.EstimateOutboundSize(uint64(len(prevOuts)), []btcutil.Address{to}) - if err != nil { - return nil, errors.Wrap(err, "unable to estimate tx size") - } - if sizeLimit < common.BtcOutboundBytesWithdrawer { // ZRC20 'withdraw' charged less fee from end user - signer.Logger().Std.Info(). - Msgf("sizeLimit %d is less than BtcOutboundBytesWithdrawer %d for nonce %d", sizeLimit, txSize, nonce) - } - if txSize < common.OutboundBytesMin { // outbound shouldn't be blocked a low sizeLimit - signer.Logger().Std.Warn(). - Msgf("txSize %d is less than outboundBytesMin %d; use outboundBytesMin", txSize, common.OutboundBytesMin) - txSize = common.OutboundBytesMin - } - if txSize > common.OutboundBytesMax { // in case of accident - signer.Logger().Std.Warn(). - Msgf("txSize %d is greater than outboundBytesMax %d; use outboundBytesMax", txSize, common.OutboundBytesMax) - txSize = common.OutboundBytesMax - } - - // fee calculation - // #nosec G115 always in range (checked above) - fees := new(big.Int).Mul(big.NewInt(int64(txSize)), gasPrice) - signer.Logger(). - Std.Info(). - Msgf("bitcoin outbound nonce %d gasPrice %s size %d fees %s consolidated %d utxos of value %v", - nonce, gasPrice.String(), txSize, fees.String(), consolidatedUtxo, consolidatedValue) - - // add tx outputs - err = signer.AddWithdrawTxOutputs(tx, to, total, amount, nonceMark, fees, cancelTx) - if err != nil { - return nil, errors.Wrap(err, "unable to add withdrawal tx outputs") - } - - // sign the tx - sigHashes := txscript.NewTxSigHashes(tx, txscript.NewCannedPrevOutputFetcher([]byte{}, 0)) - witnessHashes := make([][]byte, len(tx.TxIn)) - for ix := range tx.TxIn { - amt, err := common.GetSatoshis(prevOuts[ix].Amount) - if err != nil { - return nil, err - } - pkScript, err := hex.DecodeString(prevOuts[ix].ScriptPubKey) - if err != nil { - return nil, err - } - witnessHashes[ix], err = txscript.CalcWitnessSigHash(pkScript, sigHashes, txscript.SigHashAll, tx, ix, amt) - if err != nil { - return nil, err - } - } - - sig65Bs, err := signer.TSS().SignBatch(ctx, witnessHashes, height, nonce, chain.ChainId) - if err != nil { - return nil, errors.Wrap(err, "unable to batch sign") - } - - for ix := range tx.TxIn { - sig65B := sig65Bs[ix] - R := &btcec.ModNScalar{} - R.SetBytes((*[32]byte)(sig65B[:32])) - S := &btcec.ModNScalar{} - S.SetBytes((*[32]byte)(sig65B[32:64])) - sig := btcecdsa.NewSignature(R, S) - - pkCompressed := signer.TSS().PubKey().Bytes(true) - hashType := txscript.SigHashAll - txWitness := wire.TxWitness{append(sig.Serialize(), byte(hashType)), pkCompressed} - tx.TxIn[ix].Witness = txWitness - } - - return tx, nil -} - // Broadcast sends the signed transaction to the network func (signer *Signer) Broadcast(ctx context.Context, signedTx *wire.MsgTx) error { var outBuff bytes.Buffer @@ -263,7 +60,7 @@ func (signer *Signer) Broadcast(ctx context.Context, signedTx *wire.MsgTx) error } signer.Logger().Std.Info(). - Stringer("signer.tx_hash", signedTx.TxHash()). + Str(logs.FieldTx, signedTx.TxHash().String()). Str("signer.tx_payload", hex.EncodeToString(outBuff.Bytes())). Msg("Broadcasting transaction") @@ -275,8 +72,16 @@ func (signer *Signer) Broadcast(ctx context.Context, signedTx *wire.MsgTx) error return nil } +// TSSToPkScript returns the TSS pkScript +func (signer *Signer) TSSToPkScript() ([]byte, error) { + tssAddrP2WPKH, err := signer.TSS().PubKey().AddressBTC(signer.Chain().ChainId) + if err != nil { + return nil, err + } + return txscript.PayToAddrScript(tssAddrP2WPKH) +} + // TryProcessOutbound signs and broadcasts a BTC transaction from a new outbound -// TODO(revamp): simplify the function func (signer *Signer) TryProcessOutbound( ctx context.Context, cctx *types.CrossChainTx, @@ -295,136 +100,103 @@ func (signer *Signer) TryProcessOutbound( }() // prepare logger + chain := signer.Chain() params := cctx.GetCurrentOutboundParam() - // prepare logger fields lf := map[string]any{ logs.FieldMethod: "TryProcessOutbound", logs.FieldCctx: cctx.Index, logs.FieldNonce: params.TssNonce, } - logger := signer.Logger().Std.With().Fields(lf).Logger() - - // support gas token only for Bitcoin outbound - coinType := cctx.InboundParams.CoinType - if coinType == coin.CoinType_Zeta || coinType == coin.CoinType_ERC20 { - logger.Error().Msg("can only send BTC to a BTC network") - return + signerAddress, err := zetacoreClient.GetKeys().GetAddress() + if err == nil { + lf["signer"] = signerAddress.String() } + logger := signer.Logger().Std.With().Fields(lf).Logger() - chain := observer.Chain() - outboundTssNonce := params.TssNonce - signerAddress, err := zetacoreClient.GetKeys().GetAddress() + // query network info to get minRelayFee (typically 1000 satoshis) + networkInfo, err := signer.rpc.GetNetworkInfo(ctx) if err != nil { - logger.Error().Err(err).Msg("cannot get signer address") + logger.Error().Err(err).Msgf("failed get bitcoin network info") return } - lf["signer"] = signerAddress.String() - - // get size limit and gas price - sizelimit := params.CallOptions.GasLimit - gasprice, ok := new(big.Int).SetString(params.GasPrice, 10) - if !ok || gasprice.Cmp(big.NewInt(0)) < 0 { - logger.Error().Msgf("cannot convert gas price %s ", params.GasPrice) + minRelayFee := networkInfo.RelayFee + if minRelayFee <= 0 { + logger.Error().Msgf("invalid minimum relay fee: %f", minRelayFee) return } - // Check receiver P2WPKH address - to, err := chains.DecodeBtcAddress(params.Receiver, params.ReceiverChainId) + // setup outbound data + txData, err := NewOutboundData(cctx, chain.ChainId, height, minRelayFee, logger, signer.Logger().Compliance) if err != nil { - logger.Error().Err(err).Msgf("cannot decode address %s ", params.Receiver) - return - } - if !chains.IsBtcAddressSupported(to) { - logger.Error().Msgf("unsupported address %s", params.Receiver) + logger.Error().Err(err).Msg("failed to setup Bitcoin outbound data") return } - amount := float64(params.Amount.Uint64()) / 1e8 - // Add 1 satoshi/byte to gasPrice to avoid minRelayTxFee issue - networkInfo, err := signer.rpc.GetNetworkInfo(ctx) + // sign withdraw tx + signedTx, err := signer.SignWithdrawTx(ctx, txData, observer) if err != nil { - logger.Error().Err(err).Msgf("cannot get bitcoin network info") + logger.Error().Err(err).Msg("SignWithdrawTx failed") return } - satPerByte := common.FeeRateToSatPerByte(networkInfo.RelayFee) - gasprice.Add(gasprice, satPerByte) + logger.Info().Str(logs.FieldTx, signedTx.TxID()).Msg("SignWithdrawTx succeed") - // compliance check - restrictedCCTX := compliance.IsCctxRestricted(cctx) - if restrictedCCTX { - compliance.PrintComplianceLog(logger, signer.Logger().Compliance, - true, chain.ChainId, cctx.Index, cctx.InboundParams.Sender, params.Receiver, "BTC") - } + // broadcast signed outbound + signer.BroadcastOutbound(ctx, signedTx, params.TssNonce, cctx, observer, zetacoreClient) +} - // check dust amount - dustAmount := params.Amount.Uint64() < constant.BTCWithdrawalDustAmount - if dustAmount { - logger.Warn().Msgf("dust amount %d sats, canceling tx", params.Amount.Uint64()) - } +// BroadcastOutbound sends the signed transaction to the Bitcoin network +func (signer *Signer) BroadcastOutbound( + ctx context.Context, + tx *wire.MsgTx, + nonce uint64, + cctx *types.CrossChainTx, + ob *observer.Observer, + zetacoreClient interfaces.ZetacoreClient, +) { + txHash := tx.TxID() - // set the amount to 0 when the tx should be cancelled - cancelTx := restrictedCCTX || dustAmount - if cancelTx { - amount = 0.0 + // prepare logger fields + lf := map[string]any{ + logs.FieldMethod: "broadcastOutbound", + logs.FieldNonce: nonce, + logs.FieldTx: txHash, + logs.FieldCctx: cctx.Index, } + logger := signer.Logger().Std - // sign withdraw tx - tx, err := signer.SignWithdrawTx( - ctx, - to, - amount, - gasprice, - sizelimit, - observer, - height, - outboundTssNonce, - chain, - cancelTx, - ) - if err != nil { - logger.Warn().Err(err).Msg("SignWithdrawTx failed") - return - } - logger.Info().Msg("Key-sign success") + // try broacasting tx with increasing backoff (1s, 2s, 4s, 8s, 16s) in case of RPC error + backOff := broadcastBackoff + for i := 0; i < broadcastRetries; i++ { + time.Sleep(backOff) - // FIXME: add prometheus metrics - _, err = zetacoreClient.GetObserverList(ctx) - if err != nil { - logger.Warn(). - Err(err).Stringer("observation_type", observertypes.ObservationType_OutboundTx). - Msg("unable to get observer list, observation") - } - if tx != nil { - outboundHash := tx.TxHash().String() - lf[logs.FieldTx] = outboundHash + // broadcast tx + err := signer.Broadcast(ctx, tx) + if err != nil { + logger.Warn().Err(err).Fields(lf).Msgf("broadcasting Bitcoin outbound, retry %d", i) + backOff *= 2 + continue + } + logger.Info().Fields(lf).Msg("broadcasted Bitcoin outbound successfully") - // try broacasting tx with increasing backoff (1s, 2s, 4s, 8s, 16s) in case of RPC error - backOff := broadcastBackoff - for i := 0; i < broadcastRetries; i++ { - time.Sleep(backOff) - err := signer.Broadcast(ctx, tx) - if err != nil { - logger.Warn().Err(err).Fields(lf).Msgf("Broadcasting Bitcoin tx, retry %d", i) - backOff *= 2 - continue - } - logger.Info().Fields(lf).Msgf("Broadcast Bitcoin tx successfully") - zetaHash, err := zetacoreClient.PostOutboundTracker( - ctx, - chain.ChainId, - outboundTssNonce, - outboundHash, - ) - if err != nil { - logger.Err(err).Fields(lf).Msgf("Unable to add Bitcoin outbound tracker") - } - lf[logs.FieldZetaTx] = zetaHash - logger.Info().Fields(lf).Msgf("Add Bitcoin outbound tracker successfully") + // save tx local db + ob.SaveBroadcastedTx(txHash, nonce) - // Save successfully broadcasted transaction to btc chain observer - observer.SaveBroadcastedTx(outboundHash, outboundTssNonce) + // add tx to outbound tracker so that all observers know about it + zetaHash, err := zetacoreClient.PostOutboundTracker(ctx, ob.Chain().ChainId, nonce, txHash) + if err != nil { + logger.Err(err).Fields(lf).Msg("unable to add Bitcoin outbound tracker") + } else { + lf[logs.FieldZetaTx] = zetaHash + logger.Info().Fields(lf).Msg("add Bitcoin outbound tracker successfully") + } - break // successful broadcast; no need to retry + // try including this outbound as early as possible + _, included := ob.TryIncludeOutbound(ctx, cctx, txHash) + if included { + logger.Info().Fields(lf).Msg("included newly broadcasted Bitcoin outbound") } + + // successful broadcast; no need to retry + break } } diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index 10363472bc..1d9202b7b8 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -1,74 +1,186 @@ -package signer +package signer_test import ( + "context" "encoding/hex" - "fmt" - "math/big" - "reflect" "testing" "github.com/btcsuite/btcd/btcec/v2" - btcecdsa "github.com/btcsuite/btcd/btcec/v2/ecdsa" + "github.com/btcsuite/btcd/btcec/v2/ecdsa" + "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" "github.com/ethereum/go-ethereum/crypto" + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/zeta-chain/node/zetaclient/testutils" - . "gopkg.in/check.v1" "github.com/zeta-chain/node/pkg/chains" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" + observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" + "github.com/zeta-chain/node/zetaclient/config" + zctx "github.com/zeta-chain/node/zetaclient/context" + "github.com/zeta-chain/node/zetaclient/db" + "github.com/zeta-chain/node/zetaclient/keys" + "github.com/zeta-chain/node/zetaclient/metrics" + "github.com/zeta-chain/node/zetaclient/outboundprocessor" + "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/zetaclient/testutils/mocks" ) -type BTCSignerSuite struct { - btcSigner *Signer +// the relative path to the testdata directory +var TestDataDir = "../../../" + +type testSuite struct { + *signer.Signer + tss *mocks.TSS + client *mocks.BitcoinClient + zetacoreClient *mocks.ZetacoreClient } -var _ = Suite(&BTCSignerSuite{}) +func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { + // mock BTC RPC client + rpcClient := mocks.NewBitcoinClient(t) + rpcClient.On("GetBlockCount", mock.Anything).Maybe().Return(int64(101), nil) + + // mock TSS + var tss *mocks.TSS + if chains.IsBitcoinMainnet(chain.ChainId) { + tss = mocks.NewTSS(t).FakePubKey(testutils.TSSPubKeyMainnet) + } else { + tss = mocks.NewTSS(t).FakePubKey(testutils.TSSPubkeyAthens3) + } -type cWrapper struct{ *C } + // mock Zetacore client + zetacoreClient := mocks.NewZetacoreClient(t). + WithKeys(&keys.Keys{}). + WithZetaChain() -func (cWrapper) Cleanup(func()) { /* noop */ } + // create logger + testLogger := zerolog.New(zerolog.NewTestWriter(t)) + logger := base.Logger{Std: testLogger, Compliance: testLogger} -func (s *BTCSignerSuite) SetUpTest(c *C) { + // create signer + signer := signer.New( + chain, + tss, + rpcClient, + logger, + ) + + return &testSuite{ + Signer: signer, + tss: tss, + client: rpcClient, + zetacoreClient: zetacoreClient, + } +} + +func Test_NewSigner(t *testing.T) { // test private key with EVM address - //// EVM: 0x236C7f53a90493Bb423411fe4117Cb4c2De71DfB - // BTC testnet3: muGe9prUBjQwEnX19zG26fVRHNi8z7kSPo + // EVM: 0x236C7f53a90493Bb423411fe4117Cb4c2De71DfB + // BTC testnet: muGe9prUBjQwEnX19zG26fVRHNi8z7kSPo skHex := "7b8507ba117e069f4a3f456f505276084f8c92aee86ac78ae37b4d1801d35fa8" privateKey, err := crypto.HexToECDSA(skHex) - pkBytes := crypto.FromECDSAPub(&privateKey.PublicKey) - c.Logf("pubkey: %d", len(pkBytes)) - // Uncomment the following code to generate new random private key pairs - //privateKey, err := crypto.GenerateKey() - //privkeyBytes := crypto.FromECDSA(privateKey) - //c.Logf("privatekey %s", hex.EncodeToString(privkeyBytes)) - c.Assert(err, IsNil) - - tss := mocks.NewTSSFromPrivateKey(c, privateKey) - - s.btcSigner = New( - chains.Chain{}, - tss, - mocks.NewBitcoinClient(cWrapper{c}), - base.DefaultLogger(), - ) + require.NoError(t, err) + tss := mocks.NewTSSFromPrivateKey(t, privateKey) + signer := signer.New(chains.BitcoinMainnet, tss, mocks.NewBitcoinClient(t), base.DefaultLogger()) + require.NotNil(t, signer) +} + +func Test_BroadcastOutbound(t *testing.T) { + // test cases + tests := []struct { + name string + chain chains.Chain + nonce uint64 + failTracker bool + }{ + { + name: "should successfully broadcast and include outbound", + chain: chains.BitcoinMainnet, + nonce: uint64(148), + }, + { + name: "should successfully broadcast and include RBF outbound", + chain: chains.BitcoinMainnet, + nonce: uint64(148), + }, + { + name: "should successfully broadcast and include outbound, but fail to post outbound tracker", + chain: chains.BitcoinMainnet, + nonce: uint64(148), + failTracker: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // setup signer and observer + s := newTestSuite(t, tt.chain) + observer := s.getNewObserver(t) + + // load tx and result + chainID := tt.chain.ChainId + rawResult, cctx := testutils.LoadBTCTxRawResultNCctx(t, TestDataDir, chainID, tt.nonce) + txResult := testutils.LoadBTCTransaction(t, TestDataDir, chainID, rawResult.Txid) + msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chainID, rawResult.Txid) + hash := hashFromTXID(t, rawResult.Txid) + + // mock RPC response + s.client.On("SendRawTransaction", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil, nil) + s.client.On("GetTransactionByStr", mock.Anything, mock.Anything).Maybe().Return(hash, txResult, nil) + s.client.On("GetRawTransactionResult", mock.Anything, mock.Anything, mock.Anything). + Maybe(). + Return(*rawResult, nil) + + // mock Zetacore client response + if tt.failTracker { + s.zetacoreClient.WithPostOutboundTracker("") + } else { + s.zetacoreClient.WithPostOutboundTracker("0x123") + } + + // mock the previous tx as included + // this is necessary to allow the 'checkTSSVin' function to pass + observer.SetIncludedTx(tt.nonce-1, &btcjson.GetTransactionResult{ + TxID: rawResult.Vin[0].Txid, + }) + + ctx := makeCtx(t) + s.BroadcastOutbound( + ctx, + msgTx, + tt.nonce, + cctx, + observer, + s.zetacoreClient, + ) + + // check if outbound is included + gotResult := observer.GetIncludedTx(tt.nonce) + require.Equal(t, txResult, gotResult) + }) + } } -func (s *BTCSignerSuite) TestP2PH(c *C) { +func Test_P2PH(t *testing.T) { // Ordinarily the private key would come from whatever storage mechanism // is being used, but for this example just hard code it. privKeyBytes, err := hex.DecodeString("22a47fa09a223f2aa079edf85a7c2" + "d4f8720ee63e502ee2869afab7de234b80c") - c.Assert(err, IsNil) + require.NoError(t, err) privKey, pubKey := btcec.PrivKeyFromBytes(privKeyBytes) pubKeyHash := btcutil.Hash160(pubKey.SerializeCompressed()) addr, err := btcutil.NewAddressPubKeyHash(pubKeyHash, &chaincfg.RegressionNetParams) - c.Assert(err, IsNil) + require.NoError(t, err) // For this example, create a fake transaction that represents what // would ordinarily be the real transaction that is being spent. It @@ -78,8 +190,7 @@ func (s *BTCSignerSuite) TestP2PH(c *C) { txIn := wire.NewTxIn(prevOut, []byte{txscript.OP_0, txscript.OP_0}, nil) originTx.AddTxIn(txIn) pkScript, err := txscript.PayToAddrScript(addr) - - c.Assert(err, IsNil) + require.NoError(t, err) txOut := wire.NewTxOut(100000000, pkScript) originTx.AddTxOut(txOut) @@ -110,7 +221,7 @@ func (s *BTCSignerSuite) TestP2PH(c *C) { sigScript, err := txscript.SignTxOutput(&chaincfg.MainNetParams, redeemTx, 0, originTx.TxOut[0].PkScript, txscript.SigHashAll, txscript.KeyClosure(lookupKey), nil, nil) - c.Assert(err, IsNil) + require.NoError(t, err) redeemTx.TxIn[0].SignatureScript = sigScript @@ -121,26 +232,24 @@ func (s *BTCSignerSuite) TestP2PH(c *C) { txscript.ScriptDiscourageUpgradableNops vm, err := txscript.NewEngine(originTx.TxOut[0].PkScript, redeemTx, 0, flags, nil, nil, -1, txscript.NewMultiPrevOutFetcher(nil)) - c.Assert(err, IsNil) + require.NoError(t, err) err = vm.Execute() - c.Assert(err, IsNil) - - fmt.Println("Transaction successfully signed") + require.NoError(t, err) } -func (s *BTCSignerSuite) TestP2WPH(c *C) { +func Test_P2WPH(t *testing.T) { // Ordinarily the private key would come from whatever storage mechanism // is being used, but for this example just hard code it. privKeyBytes, err := hex.DecodeString("22a47fa09a223f2aa079edf85a7c2" + "d4f8720ee63e502ee2869afab7de234b80c") - c.Assert(err, IsNil) + require.NoError(t, err) privKey, pubKey := btcec.PrivKeyFromBytes(privKeyBytes) pubKeyHash := btcutil.Hash160(pubKey.SerializeCompressed()) //addr, err := btcutil.NewAddressPubKeyHash(pubKeyHash, &chaincfg.RegressionNetParams) addr, err := btcutil.NewAddressWitnessPubKeyHash(pubKeyHash, &chaincfg.RegressionNetParams) - c.Assert(err, IsNil) + require.NoError(t, err) // For this example, create a fake transaction that represents what // would ordinarily be the real transaction that is being spent. It @@ -150,7 +259,7 @@ func (s *BTCSignerSuite) TestP2WPH(c *C) { txIn := wire.NewTxIn(prevOut, []byte{txscript.OP_0, txscript.OP_0}, nil) originTx.AddTxIn(txIn) pkScript, err := txscript.PayToAddrScript(addr) - c.Assert(err, IsNil) + require.NoError(t, err) txOut := wire.NewTxOut(100000000, pkScript) originTx.AddTxOut(txOut) originTxHash := originTx.TxHash() @@ -171,7 +280,7 @@ func (s *BTCSignerSuite) TestP2WPH(c *C) { redeemTx.AddTxOut(txOut) txSigHashes := txscript.NewTxSigHashes(redeemTx, txscript.NewCannedPrevOutputFetcher([]byte{}, 0)) pkScript, err = txscript.PayToAddrScript(addr) - c.Assert(err, IsNil) + require.NoError(t, err) { txWitness, err := txscript.WitnessSignature( @@ -184,7 +293,7 @@ func (s *BTCSignerSuite) TestP2WPH(c *C) { privKey, true, ) - c.Assert(err, IsNil) + require.NoError(t, err) redeemTx.TxIn[0].Witness = txWitness // Prove that the transaction has been validly signed by executing the // script pair. @@ -193,10 +302,10 @@ func (s *BTCSignerSuite) TestP2WPH(c *C) { txscript.ScriptDiscourageUpgradableNops vm, err := txscript.NewEngine(originTx.TxOut[0].PkScript, redeemTx, 0, flags, nil, nil, -1, txscript.NewCannedPrevOutputFetcher([]byte{}, 0)) - c.Assert(err, IsNil) + require.NoError(t, err) err = vm.Execute() - c.Assert(err, IsNil) + require.NoError(t, err) } { @@ -208,8 +317,8 @@ func (s *BTCSignerSuite) TestP2WPH(c *C) { 0, 100000000, ) - c.Assert(err, IsNil) - sig := btcecdsa.Sign(privKey, witnessHash) + require.NoError(t, err) + sig := ecdsa.Sign(privKey, witnessHash) txWitness := wire.TxWitness{append(sig.Serialize(), byte(txscript.SigHashAll)), pubKeyHash} redeemTx.TxIn[0].Witness = txWitness @@ -218,163 +327,73 @@ func (s *BTCSignerSuite) TestP2WPH(c *C) { txscript.ScriptDiscourageUpgradableNops vm, err := txscript.NewEngine(originTx.TxOut[0].PkScript, redeemTx, 0, flags, nil, nil, -1, txscript.NewMultiPrevOutFetcher(nil)) - c.Assert(err, IsNil) + require.NoError(t, err) err = vm.Execute() - c.Assert(err, IsNil) + require.NoError(t, err) } - - fmt.Println("Transaction successfully signed") } -func TestAddWithdrawTxOutputs(t *testing.T) { - // Create test signer and receiver address - signer := New( - chains.BitcoinMainnet, - mocks.NewTSS(t).FakePubKey(testutils.TSSPubKeyMainnet), - mocks.NewBitcoinClient(t), - base.DefaultLogger(), +func makeCtx(t *testing.T) context.Context { + app := zctx.New(config.New(false), nil, zerolog.Nop()) + + chain := chains.BitcoinMainnet + btcParams := mocks.MockChainParams(chain.ChainId, 2) + + err := app.Update( + []chains.Chain{chain, chains.ZetaChainMainnet}, + nil, + map[int64]*observertypes.ChainParams{ + chain.ChainId: &btcParams, + }, + observertypes.CrosschainFlags{}, ) + require.NoError(t, err, "unable to update app context") - // tss address and script - tssAddr, err := signer.TSS().PubKey().AddressBTC(chains.BitcoinMainnet.ChainId) - require.NoError(t, err) - tssScript, err := txscript.PayToAddrScript(tssAddr) - require.NoError(t, err) - fmt.Printf("tss address: %s", tssAddr.EncodeAddress()) + return zctx.WithAppContext(context.Background(), app) +} - // receiver addresses - receiver := "bc1qaxf82vyzy8y80v000e7t64gpten7gawewzu42y" - to, err := chains.DecodeBtcAddress(receiver, chains.BitcoinMainnet.ChainId) - require.NoError(t, err) - toScript, err := txscript.PayToAddrScript(to) +// getCCTX returns a CCTX for testing +func getCCTX(t *testing.T) *crosschaintypes.CrossChainTx { + return testutils.LoadCctxByNonce(t, 8332, 148) +} + +// getNewOutboundProcessor creates a new outbound processor for testing +func getNewOutboundProcessor() *outboundprocessor.Processor { + logger := zerolog.Logger{} + return outboundprocessor.NewProcessor(logger) +} + +// getNewObserver creates a new BTC chain observer for testing +func (s *testSuite) getNewObserver(t *testing.T) *observer.Observer { + // prepare mock arguments to create observer + params := mocks.MockChainParams(s.Chain().ChainId, 2) + ts := &metrics.TelemetryServer{} + + // create in-memory db + database, err := db.NewFromSqliteInMemory(true) require.NoError(t, err) - // test cases - tests := []struct { - name string - tx *wire.MsgTx - to btcutil.Address - total float64 - amount float64 - nonce int64 - fees *big.Int - cancelTx bool - fail bool - message string - txout []*wire.TxOut - }{ - { - name: "should add outputs successfully", - tx: wire.NewMsgTx(wire.TxVersion), - to: to, - total: 1.00012000, - amount: 0.2, - nonce: 10000, - fees: big.NewInt(2000), - fail: false, - txout: []*wire.TxOut{ - {Value: 10000, PkScript: tssScript}, - {Value: 20000000, PkScript: toScript}, - {Value: 80000000, PkScript: tssScript}, - }, - }, - { - name: "should add outputs without change successfully", - tx: wire.NewMsgTx(wire.TxVersion), - to: to, - total: 0.20012000, - amount: 0.2, - nonce: 10000, - fees: big.NewInt(2000), - fail: false, - txout: []*wire.TxOut{ - {Value: 10000, PkScript: tssScript}, - {Value: 20000000, PkScript: toScript}, - }, - }, - { - name: "should cancel tx successfully", - tx: wire.NewMsgTx(wire.TxVersion), - to: to, - total: 1.00012000, - amount: 0.2, - nonce: 10000, - fees: big.NewInt(2000), - cancelTx: true, - fail: false, - txout: []*wire.TxOut{ - {Value: 10000, PkScript: tssScript}, - {Value: 100000000, PkScript: tssScript}, - }, - }, - { - name: "should fail on invalid amount", - tx: wire.NewMsgTx(wire.TxVersion), - to: to, - total: 1.00012000, - amount: -0.5, - fail: true, - }, - { - name: "should fail when total < amount", - tx: wire.NewMsgTx(wire.TxVersion), - to: to, - total: 0.00012000, - amount: 0.2, - fail: true, - }, - { - name: "should fail when total < fees + amount + nonce", - tx: wire.NewMsgTx(wire.TxVersion), - to: to, - total: 0.20011000, - amount: 0.2, - nonce: 10000, - fees: big.NewInt(2000), - fail: true, - message: "remainder value is negative", - }, - { - name: "should not produce duplicate nonce mark", - tx: wire.NewMsgTx(wire.TxVersion), - to: to, - total: 0.20022000, // 0.2 + fee + nonceMark * 2 - amount: 0.2, - nonce: 10000, - fees: big.NewInt(2000), - fail: false, - txout: []*wire.TxOut{ - {Value: 10000, PkScript: tssScript}, - {Value: 20000000, PkScript: toScript}, - {Value: 9999, PkScript: tssScript}, // nonceMark - 1 - }, - }, - { - name: "should fail on invalid to address", - tx: wire.NewMsgTx(wire.TxVersion), - to: nil, - total: 1.00012000, - amount: 0.2, - nonce: 10000, - fees: big.NewInt(2000), - fail: true, - }, - } + // create logger + testLogger := zerolog.New(zerolog.NewTestWriter(t)) + logger := base.Logger{Std: testLogger, Compliance: testLogger} + + ob, err := observer.NewObserver( + s.Chain(), + s.client, + params, + s.zetacoreClient, + s.tss, + database, + logger, + ts, + ) + require.NoError(t, err) + return ob +} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := signer.AddWithdrawTxOutputs(tt.tx, tt.to, tt.total, tt.amount, tt.nonce, tt.fees, tt.cancelTx) - if tt.fail { - require.Error(t, err) - if tt.message != "" { - require.ErrorContains(t, err, tt.message) - } - return - } else { - require.NoError(t, err) - require.True(t, reflect.DeepEqual(tt.txout, tt.tx.TxOut)) - } - }) - } +func hashFromTXID(t *testing.T, txid string) *chainhash.Hash { + h, err := chainhash.NewHashFromStr(txid) + require.NoError(t, err) + return h } diff --git a/zetaclient/chains/evm/observer/observer_test.go b/zetaclient/chains/evm/observer/observer_test.go index 824ee94bb7..c40c7144c7 100644 --- a/zetaclient/chains/evm/observer/observer_test.go +++ b/zetaclient/chains/evm/observer/observer_test.go @@ -201,49 +201,6 @@ func Test_NewObserver(t *testing.T) { } } -func Test_LoadLastBlockScanned(t *testing.T) { - ctx := context.Background() - - // create observer using mock evm client - ob := newTestSuite(t) - ob.evmClient.On("BlockNumber", mock.Anything).Return(uint64(100), nil) - - t.Run("should load last block scanned", func(t *testing.T) { - // create db and write 123 as last block scanned - ob.WriteLastBlockScannedToDB(123) - - // load last block scanned - err := ob.LoadLastBlockScanned(ctx) - require.NoError(t, err) - require.EqualValues(t, 123, ob.LastBlockScanned()) - }) - t.Run("should fail on invalid env var", func(t *testing.T) { - // set invalid environment variable - envvar := base.EnvVarLatestBlockByChain(ob.Chain()) - os.Setenv(envvar, "invalid") - defer os.Unsetenv(envvar) - - // load last block scanned - err := ob.LoadLastBlockScanned(ctx) - require.ErrorContains(t, err, "error LoadLastBlockScanned") - }) - t.Run("should fail on RPC error", func(t *testing.T) { - // create observer on separate path, as we need to reset last block scanned - obOther := newTestSuite(t) - - // reset last block scanned to 0 so that it will be loaded from RPC - obOther.WithLastBlockScanned(0) - - // attach mock evm client to observer - obOther.evmClient.On("BlockNumber", mock.Anything).Unset() - obOther.evmClient.On("BlockNumber", mock.Anything).Return(uint64(0), fmt.Errorf("error RPC")) - - // load last block scanned - err := obOther.LoadLastBlockScanned(ctx) - require.ErrorContains(t, err, "error RPC") - }) -} - func Test_BlockCache(t *testing.T) { t.Run("should get block from cache", func(t *testing.T) { // create observer diff --git a/zetaclient/chains/evm/rpc/rpc_live_test.go b/zetaclient/chains/evm/rpc/rpc_live_test.go index ec99fe6ebd..7202ecca38 100644 --- a/zetaclient/chains/evm/rpc/rpc_live_test.go +++ b/zetaclient/chains/evm/rpc/rpc_live_test.go @@ -3,6 +3,7 @@ package rpc_test import ( "context" "math" + "math/big" "github.com/ethereum/go-ethereum/ethclient" "github.com/stretchr/testify/require" @@ -17,6 +18,7 @@ const ( URLEthSepolia = "https://rpc.ankr.com/eth_sepolia" URLBscMainnet = "https://rpc.ankr.com/bsc" URLPolygonMainnet = "https://rpc.ankr.com/polygon" + URLBaseMainnet = "https://rpc.ankr.com/base" ) // Test_EVMRPCLive is a phony test to run each live test individually @@ -27,6 +29,7 @@ func Test_EVMRPCLive(t *testing.T) { LiveTest_IsTxConfirmed(t) LiveTest_CheckRPCStatus(t) + LiveTest_SuggestGasPrice(t) } func LiveTest_IsTxConfirmed(t *testing.T) { @@ -58,3 +61,13 @@ func LiveTest_CheckRPCStatus(t *testing.T) { _, err = rpc.CheckRPCStatus(ctx, client) require.NoError(t, err) } + +func LiveTest_SuggestGasPrice(t *testing.T) { + client, err := ethclient.Dial(URLBaseMainnet) + require.NoError(t, err) + + ctx := context.Background() + gasPrice, err := client.SuggestGasPrice(ctx) + require.NoError(t, err) + require.True(t, gasPrice.Cmp(big.NewInt(0)) > 0) +} diff --git a/zetaclient/chains/evm/signer/signer.go b/zetaclient/chains/evm/signer/signer.go index 4a6c194b21..98ef49214e 100644 --- a/zetaclient/chains/evm/signer/signer.go +++ b/zetaclient/chains/evm/signer/signer.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "math/big" - "strconv" "strings" "time" @@ -520,8 +519,8 @@ func (signer *Signer) BroadcastOutbound( outboundHash, toChain.ID(), cctx.GetCurrentOutboundParam().TssNonce, i, myID) retry, report := zetacore.HandleBroadcastError( err, - strconv.FormatUint(cctx.GetCurrentOutboundParam().TssNonce, 10), - fmt.Sprintf("%d", toChain.ID()), + cctx.GetCurrentOutboundParam().TssNonce, + toChain.ID(), outboundHash, ) if report { diff --git a/zetaclient/logs/fields.go b/zetaclient/logs/fields.go index 58880543af..c42314241b 100644 --- a/zetaclient/logs/fields.go +++ b/zetaclient/logs/fields.go @@ -9,6 +9,7 @@ const ( FieldChainNetwork = "chain_network" FieldNonce = "nonce" FieldTx = "tx" + FieldOutboundID = "outbound_id" FieldCctx = "cctx" FieldZetaTx = "zeta_tx" FieldBallot = "ballot" diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index 122454db3b..be71824e19 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -408,12 +408,12 @@ func (oc *Orchestrator) runScheduler(ctx context.Context) error { switch { case chain.IsEVM(): - oc.ScheduleCctxEVM(ctx, zetaHeight, chainID, cctxList, ob, signer) + oc.ScheduleCCTXEVM(ctx, zetaHeight, chainID, cctxList, ob, signer) case chain.IsBitcoin(): // Managed by orchestrator V2 continue case chain.IsSolana(): - oc.ScheduleCctxSolana(ctx, zetaHeight, chainID, cctxList, ob, signer) + oc.ScheduleCCTXSolana(ctx, zetaHeight, chainID, cctxList, ob, signer) case chain.IsTON(): oc.ScheduleCCTXTON(ctx, zetaHeight, chainID, cctxList, ob, signer) default: @@ -428,8 +428,8 @@ func (oc *Orchestrator) runScheduler(ctx context.Context) error { } } -// ScheduleCctxEVM schedules evm outbound keysign on each ZetaChain block (the ticker) -func (oc *Orchestrator) ScheduleCctxEVM( +// ScheduleCCTXEVM schedules evm outbound keysign on each ZetaChain block (the ticker) +func (oc *Orchestrator) ScheduleCCTXEVM( ctx context.Context, zetaHeight uint64, chainID int64, @@ -439,7 +439,7 @@ func (oc *Orchestrator) ScheduleCctxEVM( ) { res, err := oc.zetacoreClient.GetAllOutboundTrackerByChain(ctx, chainID, interfaces.Ascending) if err != nil { - oc.logger.Warn().Err(err).Msgf("ScheduleCctxEVM: GetAllOutboundTrackerByChain failed for chain %d", chainID) + oc.logger.Warn().Err(err).Msgf("ScheduleCCTXEVM: GetAllOutboundTrackerByChain failed for chain %d", chainID) return } trackerMap := make(map[uint64]bool) @@ -461,11 +461,11 @@ func (oc *Orchestrator) ScheduleCctxEVM( if params.ReceiverChainId != chainID { oc.logger.Error(). - Msgf("ScheduleCctxEVM: outbound %s chainid mismatch: want %d, got %d", outboundID, chainID, params.ReceiverChainId) + Msgf("ScheduleCCTXEVM: outbound %s chainid mismatch: want %d, got %d", outboundID, chainID, params.ReceiverChainId) continue } if params.TssNonce > cctxList[0].GetCurrentOutboundParam().TssNonce+outboundScheduleLookback { - oc.logger.Error().Msgf("ScheduleCctxEVM: nonce too high: signing %d, earliest pending %d, chain %d", + oc.logger.Error().Msgf("ScheduleCCTXEVM: nonce too high: signing %d, earliest pending %d, chain %d", params.TssNonce, cctxList[0].GetCurrentOutboundParam().TssNonce, chainID) break } @@ -475,12 +475,12 @@ func (oc *Orchestrator) ScheduleCctxEVM( if err != nil { oc.logger.Error(). Err(err). - Msgf("ScheduleCctxEVM: VoteOutboundIfConfirmed failed for chain %d nonce %d", chainID, nonce) + Msgf("ScheduleCCTXEVM: VoteOutboundIfConfirmed failed for chain %d nonce %d", chainID, nonce) continue } if !continueKeysign { oc.logger.Info(). - Msgf("ScheduleCctxEVM: outbound %s already processed; do not schedule keysign", outboundID) + Msgf("ScheduleCCTXEVM: outbound %s already processed; do not schedule keysign", outboundID) continue } @@ -508,7 +508,7 @@ func (oc *Orchestrator) ScheduleCctxEVM( !oc.outboundProc.IsOutboundActive(outboundID) { oc.outboundProc.StartTryProcess(outboundID) oc.logger.Debug(). - Msgf("ScheduleCctxEVM: sign outbound %s with value %d", outboundID, cctx.GetCurrentOutboundParam().Amount) + Msgf("ScheduleCCTXEVM: sign outbound %s with value %d", outboundID, cctx.GetCurrentOutboundParam().Amount) go signer.TryProcessOutbound( ctx, cctx, @@ -527,8 +527,8 @@ func (oc *Orchestrator) ScheduleCctxEVM( } } -// ScheduleCctxSolana schedules solana outbound keysign on each ZetaChain block (the ticker) -func (oc *Orchestrator) ScheduleCctxSolana( +// ScheduleCCTXSolana schedules solana outbound keysign on each ZetaChain block (the ticker) +func (oc *Orchestrator) ScheduleCCTXSolana( ctx context.Context, zetaHeight uint64, chainID int64, @@ -538,7 +538,7 @@ func (oc *Orchestrator) ScheduleCctxSolana( ) { solObserver, ok := observer.(*solanaobserver.Observer) if !ok { // should never happen - oc.logger.Error().Msgf("ScheduleCctxSolana: chain observer is not a solana observer") + oc.logger.Error().Msgf("ScheduleCCTXSolana: chain observer is not a solana observer") return } @@ -556,11 +556,11 @@ func (oc *Orchestrator) ScheduleCctxSolana( if params.ReceiverChainId != chainID { oc.logger.Error(). - Msgf("ScheduleCctxSolana: outbound %s chainid mismatch: want %d, got %d", outboundID, chainID, params.ReceiverChainId) + Msgf("ScheduleCCTXSolana: outbound %s chainid mismatch: want %d, got %d", outboundID, chainID, params.ReceiverChainId) continue } if params.TssNonce > cctxList[0].GetCurrentOutboundParam().TssNonce+outboundScheduleLookback { - oc.logger.Warn().Msgf("ScheduleCctxSolana: nonce too high: signing %d, earliest pending %d", + oc.logger.Warn().Msgf("ScheduleCCTXSolana: nonce too high: signing %d, earliest pending %d", params.TssNonce, cctxList[0].GetCurrentOutboundParam().TssNonce) break } @@ -570,19 +570,19 @@ func (oc *Orchestrator) ScheduleCctxSolana( if err != nil { oc.logger.Error(). Err(err). - Msgf("ScheduleCctxSolana: VoteOutboundIfConfirmed failed for chain %d nonce %d", chainID, nonce) + Msgf("ScheduleCCTXSolana: VoteOutboundIfConfirmed failed for chain %d nonce %d", chainID, nonce) continue } if !continueKeysign { oc.logger.Info(). - Msgf("ScheduleCctxSolana: outbound %s already processed; do not schedule keysign", outboundID) + Msgf("ScheduleCCTXSolana: outbound %s already processed; do not schedule keysign", outboundID) continue } // schedule a TSS keysign if nonce%interval == zetaHeight%interval && !oc.outboundProc.IsOutboundActive(outboundID) { oc.outboundProc.StartTryProcess(outboundID) - oc.logger.Debug().Msgf("ScheduleCctxSolana: sign outbound %s with value %d", outboundID, params.Amount) + oc.logger.Debug().Msgf("ScheduleCCTXSolana: sign outbound %s with value %d", outboundID, params.Amount) go signer.TryProcessOutbound( ctx, cctx, diff --git a/zetaclient/orchestrator/v2_bootstrap.go b/zetaclient/orchestrator/v2_bootstrap.go index e05375f6f3..95ddb7179c 100644 --- a/zetaclient/orchestrator/v2_bootstrap.go +++ b/zetaclient/orchestrator/v2_bootstrap.go @@ -49,7 +49,6 @@ func (oc *V2) bootstrapBitcoin(ctx context.Context, chain zctx.Chain) (*bitcoin. // TODO extract base observer // TODO extract base signer // https://github.com/zeta-chain/node/issues/3331 - observer, err := btcobserver.NewObserver( *rawChain, rpcClient, diff --git a/zetaclient/testdata/btc/chain_8332_msgtx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json b/zetaclient/testdata/btc/chain_8332_msgtx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json new file mode 100644 index 0000000000..a96ef05a83 --- /dev/null +++ b/zetaclient/testdata/btc/chain_8332_msgtx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json @@ -0,0 +1,95 @@ +{ + "Version": 1, + "TxIn": [ + { + "PreviousOutPoint": { + "Hash": "efca302a18bd8cebb3b8afef13e98ecaac47157755a62ab241ef3848140cfe92", + "Index": 0 + }, + "SignatureScript": "", + "Witness": [ + "MEUCIQD+vXB5QFfmG4PiqsCTAtiZEOO3mgMbCEtPFIxVKaGJxgIgfGCjg07rfdmJwQjHNbwX4NU853oBbowIkNvB5dxXO2wB", + "AvrTNIt8fXPoWnz+U69CtTW3UEGe/FXDbIB/ON0LoG7c" + ], + "Sequence": 4294967295 + }, + { + "PreviousOutPoint": { + "Hash": "3dc005eb0c1d393e717070ea84aa13e334a458a4fb7c7f9f98dbf8b231b5ceef", + "Index": 0 + }, + "SignatureScript": "", + "Witness": [ + "MEUCIQDip9szSAtGI8GRvjSFJfSNLGx/2MepdquH1Vaj2fG/DAIgYMUfOFQvE8MywRSqqiCTcoNDqVUGkw1cgQvd3koxIVMB", + "AvrTNIt8fXPoWnz+U69CtTW3UEGe/FXDbIB/ON0LoG7c" + ], + "Sequence": 4294967295 + }, + { + "PreviousOutPoint": { + "Hash": "74c3aca825f3b21b82ee344d939c40d4c1e836a89c18abbd521bfa69f5f6e5d7", + "Index": 0 + }, + "SignatureScript": "", + "Witness": [ + "MEQCIDzr8YVsCvLFwtjs5DVBjpmecAUH6mR7tc8QmUmzN9VzAiBnU/AbfIG3MQRrGK/3WJ6EcVJK7+Y0mjRocLwJyh3o1wE=", + "AvrTNIt8fXPoWnz+U69CtTW3UEGe/FXDbIB/ON0LoG7c" + ], + "Sequence": 4294967295 + }, + { + "PreviousOutPoint": { + "Hash": "87264cef0e581f4aab3c99c53221bec3219686b48088d651a8cf8a98e4c2c5bf", + "Index": 0 + }, + "SignatureScript": "", + "Witness": [ + "MEQCIFM1gzPXKK/6SpXiP2ZZn2bJQB5PgCu7E/AUrPrighdoAiB5PFg1YmenwAUoiafag9N+sBMGJ3SWs+BE5KW0q9xEYQE=", + "AvrTNIt8fXPoWnz+U69CtTW3UEGe/FXDbIB/ON0LoG7c" + ], + "Sequence": 4294967295 + }, + { + "PreviousOutPoint": { + "Hash": "5af24933973df03d96624ae1341d79a860e8dbc2ffc841420aa6710f3abc0074", + "Index": 0 + }, + "SignatureScript": "", + "Witness": [ + "MEQCICFVKukAkYOXm1NN7bJR1VWqyqaPFAwBbr+x5nh6NcXgAiAwnfdr1RESQ1LDlV+S0NscurZQT+VkmwWFsMdABANXCwE=", + "AvrTNIt8fXPoWnz+U69CtTW3UEGe/FXDbIB/ON0LoG7c" + ], + "Sequence": 4294967295 + }, + { + "PreviousOutPoint": { + "Hash": "efca302a18bd8cebb3b8afef13e98ecaac47157755a62ab241ef3848140cfe92", + "Index": 2 + }, + "SignatureScript": "", + "Witness": [ + "MEUCIQD6vL28zA0kaK9gdD+oFxWf3Qmj+XGT8Rl4DulatAFMkgIgX3KMst6jqScmUdCcI4ImSbOMFg0MwiJhPLddsbzeXhgB", + "AvrTNIt8fXPoWnz+U69CtTW3UEGe/FXDbIB/ON0LoG7c" + ], + "Sequence": 4294967295 + }, + { + "PreviousOutPoint": { + "Hash": "b85755938ac026b2d13e5fbacf015288f453712b4eb4a02d7e4c98ee76ada530", + "Index": 0 + }, + "SignatureScript": "", + "Witness": [ + "MEUCIQCFNdqVZQvNeGSV8/2/GRA/wNZAjQAtYCErth+8e/aJRQIgK6Xl4ymJrD7yk/VWGWwmM+bnN1DjJT7UdONmxWSawd0B", + "AvrTNIt8fXPoWnz+U69CtTW3UEGe/FXDbIB/ON0LoG7c" + ], + "Sequence": 4294967295 + } + ], + "TxOut": [ + { "Value": 2148, "PkScript": "ABTaquDT3p2P3uMWYeYa6oKLWb54ZA==" }, + { "Value": 12000, "PkScript": "ABQMG/t9ON/wlG/exWJtUa1Y1+m8VA==" }, + { "Value": 39041489, "PkScript": "ABTaquDT3p2P3uMWYeYa6oKLWb54ZA==" } + ], + "LockTime": 0 +} diff --git a/zetaclient/testdata/btc/chain_8332_tx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json b/zetaclient/testdata/btc/chain_8332_tx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json new file mode 100644 index 0000000000..554dcbdd1b --- /dev/null +++ b/zetaclient/testdata/btc/chain_8332_tx_030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0.json @@ -0,0 +1,58 @@ +{ + "amount": -0.00012, + "fee": -0.00027213, + "confirmations": 0, + "blockhash": "000000000000000000019881b7ae81a9bfac866989c8b976b1aff7ace01b85e7", + "blockindex": 150, + "blocktime": 1708608596, + "txid": "030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0", + "walletconflicts": [], + "time": 1708608291, + "timereceived": 1708608291, + "details": [ + { + "account": "", + "address": "bc1qm24wp577nk8aacckv8np465z3dvmu7ry45el6y", + "amount": -0.00002148, + "category": "send", + "involveswatchonly": true, + "fee": -0.00027213, + "vout": 0 + }, + { + "account": "", + "address": "bc1qpsdlklfcmlcfgm77c43x65ddtrt7n0z57hsyjp", + "amount": -0.00012, + "category": "send", + "involveswatchonly": true, + "fee": -0.00027213, + "vout": 1 + }, + { + "account": "", + "address": "bc1qm24wp577nk8aacckv8np465z3dvmu7ry45el6y", + "amount": -0.39041489, + "category": "send", + "involveswatchonly": true, + "fee": -0.00027213, + "vout": 2 + }, + { + "account": "", + "address": "bc1qm24wp577nk8aacckv8np465z3dvmu7ry45el6y", + "amount": 0.00002148, + "category": "receive", + "involveswatchonly": true, + "vout": 0 + }, + { + "account": "", + "address": "bc1qm24wp577nk8aacckv8np465z3dvmu7ry45el6y", + "amount": 0.39041489, + "category": "receive", + "involveswatchonly": true, + "vout": 2 + } + ], + "hex": "0100000000010792fe0c144838ef41b22aa655771547acca8ee913efafb8b3eb8cbd182a30caef0000000000ffffffffefceb531b2f8db989f7f7cfba458a434e313aa84ea7070713e391d0ceb05c03d0000000000ffffffffd7e5f6f569fa1b52bdab189ca836e8c1d4409c934d34ee821bb2f325a8acc3740000000000ffffffffbfc5c2e4988acfa851d68880b4869621c3be2132c5993cab4a1f580eef4c26870000000000ffffffff7400bc3a0f71a60a4241c8ffc2dbe860a8791d34e14a62963df03d973349f25a0000000000ffffffff92fe0c144838ef41b22aa655771547acca8ee913efafb8b3eb8cbd182a30caef0200000000ffffffff30a5ad76ee984c7e2da0b44e2b7153f4885201cfba5f3ed1b226c08a935557b80000000000ffffffff036408000000000000160014daaae0d3de9d8fdee31661e61aea828b59be7864e02e0000000000001600140c1bfb7d38dff0946fdec5626d51ad58d7e9bc54d1b9530200000000160014daaae0d3de9d8fdee31661e61aea828b59be786402483045022100febd70794057e61b83e2aac09302d89910e3b79a031b084b4f148c5529a189c602207c60a3834eeb7dd989c108c735bc17e0d53ce77a016e8c0890dbc1e5dc573b6c012102fad3348b7c7d73e85a7cfe53af42b535b750419efc55c36c807f38dd0ba06edc02483045022100e2a7db33480b4623c191be348525f48d2c6c7fd8c7a976ab87d556a3d9f1bf0c022060c51f38542f13c332c114aaaa2093728343a95506930d5c810bddde4a312153012102fad3348b7c7d73e85a7cfe53af42b535b750419efc55c36c807f38dd0ba06edc0247304402203cebf1856c0af2c5c2d8ece435418e999e700507ea647bb5cf109949b337d57302206753f01b7c81b731046b18aff7589e8471524aefe6349a346870bc09ca1de8d7012102fad3348b7c7d73e85a7cfe53af42b535b750419efc55c36c807f38dd0ba06edc02473044022053358333d728affa4a95e23f66599f66c9401e4f802bbb13f014acfae28217680220793c58356267a7c0052889a7da83d37eb01306277496b3e044e4a5b4abdc4461012102fad3348b7c7d73e85a7cfe53af42b535b750419efc55c36c807f38dd0ba06edc02473044022021552ae9009183979b534dedb251d555aacaa68f140c016ebfb1e6787a35c5e00220309df76bd511124352c3955f92d0db1cbab6504fe5649b0585b0c7400403570b012102fad3348b7c7d73e85a7cfe53af42b535b750419efc55c36c807f38dd0ba06edc02483045022100fabcbdbccc0d2468af60743fa817159fdd09a3f97193f119780ee95ab4014c9202205f728cb2dea3a9272651d09c23822649b38c160d0cc222613cb75db1bcde5e18012102fad3348b7c7d73e85a7cfe53af42b535b750419efc55c36c807f38dd0ba06edc024830450221008535da95650bcd786495f3fdbf19103fc0d6408d002d60212bb61fbc7bf6894502202ba5e5e32989ac3ef293f556196c2633e6e73750e3253ed474e366c5649ac1dd012102fad3348b7c7d73e85a7cfe53af42b535b750419efc55c36c807f38dd0ba06edc00000000" +} diff --git a/zetaclient/testutils/mocks/bitcoin_client.go b/zetaclient/testutils/mocks/bitcoin_client.go index c9219f6aab..e921e1a54a 100644 --- a/zetaclient/testutils/mocks/bitcoin_client.go +++ b/zetaclient/testutils/mocks/bitcoin_client.go @@ -357,6 +357,34 @@ func (_m *BitcoinClient) GetBlockVerboseByStr(ctx context.Context, blockHash str return r0, r1 } +// GetEstimatedFeeRate provides a mock function with given fields: ctx, confTarget, regnet +func (_m *BitcoinClient) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) { + ret := _m.Called(ctx, confTarget, regnet) + + if len(ret) == 0 { + panic("no return value specified for GetEstimatedFeeRate") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64, bool) (int64, error)); ok { + return rf(ctx, confTarget, regnet) + } + if rf, ok := ret.Get(0).(func(context.Context, int64, bool) int64); ok { + r0 = rf(ctx, confTarget, regnet) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, int64, bool) error); ok { + r1 = rf(ctx, confTarget, regnet) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetNetworkInfo provides a mock function with given fields: ctx func (_m *BitcoinClient) GetNetworkInfo(ctx context.Context) (*btcjson.GetNetworkInfoResult, error) { ret := _m.Called(ctx) diff --git a/zetaclient/testutils/mocks/zetacore_client_opts.go b/zetaclient/testutils/mocks/zetacore_client_opts.go index 503264d867..723daff490 100644 --- a/zetaclient/testutils/mocks/zetacore_client_opts.go +++ b/zetaclient/testutils/mocks/zetacore_client_opts.go @@ -34,6 +34,17 @@ func (_m *ZetacoreClient) WithPostVoteOutbound(zetaTxHash string, ballotIndex st return _m } +func (_m *ZetacoreClient) WithPostOutboundTracker(zetaTxHash string) *ZetacoreClient { + on := _m.On("PostOutboundTracker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe() + if zetaTxHash != "" { + on.Return(zetaTxHash, nil) + } else { + on.Return("", errSomethingIsWrong) + } + + return _m +} + func (_m *ZetacoreClient) WithPostVoteInbound(zetaTxHash string, ballotIndex string) *ZetacoreClient { _m.On("PostVoteInbound", mock.Anything, mock.Anything, mock.Anything, mock.Anything). Maybe(). diff --git a/zetaclient/testutils/testdata.go b/zetaclient/testutils/testdata.go index f79c53e5c8..9dbc0e20e3 100644 --- a/zetaclient/testutils/testdata.go +++ b/zetaclient/testutils/testdata.go @@ -117,6 +117,14 @@ func LoadBTCMsgTx(t *testing.T, dir string, chainID int64, txHash string) *wire. return msgTx } +// LoadBTCTransaction loads archived Bitcoin transaction from file +func LoadBTCTransaction(t *testing.T, dir string, chainID int64, txHash string) *btcjson.GetTransactionResult { + name := path.Join(dir, TestDataPathBTC, FileNameBTCTransaction(chainID, txHash)) + tx := &btcjson.GetTransactionResult{} + LoadObjectFromJSONFile(t, tx, name) + return tx +} + // LoadBTCTxRawResult loads archived Bitcoin tx raw result from file func LoadBTCTxRawResult(t *testing.T, dir string, chainID int64, txType string, txHash string) *btcjson.TxRawResult { name := path.Join(dir, TestDataPathBTC, FileNameBTCTxByType(chainID, txType, txHash)) diff --git a/zetaclient/testutils/testdata_naming.go b/zetaclient/testutils/testdata_naming.go index be09b0b0fa..03751c7722 100644 --- a/zetaclient/testutils/testdata_naming.go +++ b/zetaclient/testutils/testdata_naming.go @@ -64,6 +64,11 @@ func FileNameBTCMsgTx(chainID int64, txHash string) string { return fmt.Sprintf("chain_%d_msgtx_%s.json", chainID, txHash) } +// FileNameBTCTransaction returns unified archive file name for btc transaction +func FileNameBTCTransaction(chainID int64, txHash string) string { + return fmt.Sprintf("chain_%d_tx_%s.json", chainID, txHash) +} + // FileNameEVMOutbound returns unified archive file name for outbound tx func FileNameEVMOutbound(chainID int64, txHash string, coinType coin.CoinType) string { return fmt.Sprintf("chain_%d_outbound_%s_%s.json", chainID, coinType, txHash) diff --git a/zetaclient/zetacore/broadcast.go b/zetaclient/zetacore/broadcast.go index 26480df638..498308622a 100644 --- a/zetaclient/zetacore/broadcast.go +++ b/zetaclient/zetacore/broadcast.go @@ -19,6 +19,7 @@ import ( "github.com/zeta-chain/node/app/ante" "github.com/zeta-chain/node/cmd/zetacored/config" "github.com/zeta-chain/node/zetaclient/authz" + "github.com/zeta-chain/node/zetaclient/logs" ) // paying 50% more than the current base gas price to buffer for potential block-by-block @@ -158,16 +159,17 @@ func (c *Client) QueryTxResult(hash string) (*sdktypes.TxResponse, error) { // HandleBroadcastError returns whether to retry in a few seconds, and whether to report via AddOutboundTracker // returns (bool retry, bool report) -func HandleBroadcastError(err error, nonce, toChain, outboundHash string) (bool, bool) { +func HandleBroadcastError(err error, nonce uint64, toChain int64, outboundHash string) (bool, bool) { if err == nil { return false, false } msg := err.Error() evt := log.Warn().Err(err). - Str("broadcast.nonce", nonce). - Str("broadcast.to_chain", toChain). - Str("broadcast.outbound_hash", outboundHash) + Str(logs.FieldMethod, "HandleBroadcastError"). + Int64(logs.FieldChain, toChain). + Uint64(logs.FieldNonce, nonce). + Str(logs.FieldTx, outboundHash) switch { case strings.Contains(msg, "nonce too low"): diff --git a/zetaclient/zetacore/broadcast_test.go b/zetaclient/zetacore/broadcast_test.go index 3fb5093963..56607ee585 100644 --- a/zetaclient/zetacore/broadcast_test.go +++ b/zetaclient/zetacore/broadcast_test.go @@ -31,7 +31,7 @@ func TestHandleBroadcastError(t *testing.T) { errors.New(""): {retry: true, report: false}, } for input, output := range testCases { - retry, report := HandleBroadcastError(input, "", "", "") + retry, report := HandleBroadcastError(input, 100, 1, "") require.Equal(t, output.report, report) require.Equal(t, output.retry, retry) } From 24816546c0352bfe175b0bce796a8e31434c6e2f Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Mon, 20 Jan 2025 22:10:16 -0600 Subject: [PATCH 02/74] add changelog entry --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 9e1585f319..b097e4e560 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,7 @@ * [3332](https://github.com/zeta-chain/node/pull/3332) - implement orchestrator V2. Move BTC observer-signer to V2 * [3360](https://github.com/zeta-chain/node/pull/3360) - update protocol contract imports using consolidated path * [3349](https://github.com/zeta-chain/node/pull/3349) - implement new bitcoin rpc in zetaclient with improved performance and observability +* [3381](https://github.com/zeta-chain/node/pull/3381) - some code refactoring in zetaclient for Bitcoin RBF adoption ### Fixes From 111d703f8875da01c05c2dccebd643ae8d80f0ff Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Mon, 20 Jan 2025 23:18:18 -0600 Subject: [PATCH 03/74] add unit test for FeeRateToSatPerByte --- zetaclient/chains/bitcoin/client/helpers.go | 15 ++------ zetaclient/chains/bitcoin/common/fee.go | 10 ++++++ zetaclient/chains/bitcoin/common/fee_test.go | 36 +++++++++++++++++++ .../chains/bitcoin/observer/observer_test.go | 1 + .../chains/bitcoin/signer/outbound_data.go | 6 ++-- 5 files changed, 53 insertions(+), 15 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 2c4590e7ff..abc05f9af7 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -3,13 +3,14 @@ package client import ( "context" "fmt" - "math/big" "time" types "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/pkg/errors" + + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" ) const ( @@ -18,9 +19,6 @@ const ( // maxBTCSupply is the maximum supply of Bitcoin maxBTCSupply = 21000000.0 - - // bytesPerKB is the number of vB in a KB - bytesPerKB = 1000 ) // GetBlockVerboseByStr alias for GetBlockVerbose @@ -116,13 +114,6 @@ func (c *Client) GetRawTransactionResult(ctx context.Context, } } -// FeeRateToSatPerByte converts a fee rate from BTC/KB to sat/vB. -func FeeRateToSatPerByte(rate float64) *big.Int { - // #nosec G115 always in range - satPerKB := new(big.Int).SetInt64(int64(rate * btcutil.SatoshiPerBitcoin)) - return new(big.Int).Div(satPerKB, big.NewInt(bytesPerKB)) -} - // GetEstimatedFeeRate gets estimated smart fee rate (BTC/Kb) targeting given block confirmation func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) { // RPC 'EstimateSmartFee' is not available in regnet @@ -144,7 +135,7 @@ func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regn return 0, fmt.Errorf("invalid fee rate: %f", *feeResult.FeeRate) } - return FeeRateToSatPerByte(*feeResult.FeeRate).Int64(), nil + return common.FeeRateToSatPerByte(*feeResult.FeeRate), nil } // GetTransactionFeeAndRate gets the transaction fee and rate for a given tx result diff --git a/zetaclient/chains/bitcoin/common/fee.go b/zetaclient/chains/bitcoin/common/fee.go index f7ae0559fd..ffa457f784 100644 --- a/zetaclient/chains/bitcoin/common/fee.go +++ b/zetaclient/chains/bitcoin/common/fee.go @@ -31,6 +31,9 @@ const ( OutboundBytesMin = int64(239) // 239vB == EstimateOutboundSize(2, 2, toP2WPKH) OutboundBytesMax = int64(1543) // 1543v == EstimateOutboundSize(21, 2, toP2TR) + // bytesPerKB is the number of vB in a KB + bytesPerKB = 1000 + // defaultDepositorFeeRate is the default fee rate for depositor fee, 20 sat/vB defaultDepositorFeeRate = 20 @@ -65,6 +68,13 @@ type RPC interface { // DepositorFeeCalculator is a function type to calculate the Bitcoin depositor fee type DepositorFeeCalculator func(context.Context, RPC, *btcjson.TxRawResult, *chaincfg.Params) (float64, error) +// FeeRateToSatPerByte converts a fee rate from BTC/KB to sat/vB. +func FeeRateToSatPerByte(rate float64) int64 { + satPerKB := rate * btcutil.SatoshiPerBitcoin + // #nosec G115 always in range + return int64(satPerKB / bytesPerKB) +} + // WiredTxSize calculates the wired tx size in bytes func WiredTxSize(numInputs uint64, numOutputs uint64) int64 { // Version 4 bytes + LockTime 4 bytes + Serialized varint size for the diff --git a/zetaclient/chains/bitcoin/common/fee_test.go b/zetaclient/chains/bitcoin/common/fee_test.go index e73e4150b9..4465ed6645 100644 --- a/zetaclient/chains/bitcoin/common/fee_test.go +++ b/zetaclient/chains/bitcoin/common/fee_test.go @@ -178,6 +178,42 @@ func signTx(t *testing.T, tx *wire.MsgTx, payerScript []byte, privateKey *btcec. } } +func Test_FeeRateToSatPerByte(t *testing.T) { + tests := []struct { + name string + rate float64 + expected int64 + }{ + { + name: "0 sat/vByte", + rate: 0.00000999, + expected: 0, + }, + { + name: "1 sat/vByte", + rate: 0.00001, + expected: 1, + }, + { + name: "5 sat/vByte", + rate: 0.00005999, + expected: 5, + }, + { + name: "10 sat/vByte", + rate: 0.0001, + expected: 10, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rate := FeeRateToSatPerByte(tt.rate) + require.Equal(t, tt.expected, rate) + }) + } +} + func TestOutboundSize2In3Out(t *testing.T) { // Generate payer/payee private keys and P2WPKH addresss privateKey, _, payerScript := generateKeyPair(t, &chaincfg.TestNet3Params) diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index 2e664a3734..c5d5a5be2f 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -292,6 +292,7 @@ func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { database, err = db.NewFromSqliteInMemory(true) } else { database, err = db.NewFromSqlite(dbPath, "test.db", true) + t.Cleanup(func() { os.RemoveAll(dbPath) }) } require.NoError(t, err) diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index fdffadc602..7caecef55c 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -13,7 +13,7 @@ import ( "github.com/zeta-chain/node/pkg/coin" "github.com/zeta-chain/node/pkg/constant" "github.com/zeta-chain/node/x/crosschain/types" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/zetaclient/compliance" ) @@ -99,8 +99,8 @@ func NewOutboundData( // add minimum relay fee (1000 satoshis/KB by default) to gasPrice to avoid minRelayTxFee error // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/policy.h#L35 - satPerByte := client.FeeRateToSatPerByte(minRelayFee) - feeRate += satPerByte.Int64() + satPerByte := common.FeeRateToSatPerByte(minRelayFee) + feeRate += satPerByte // compliance check restrictedCCTX := compliance.IsCctxRestricted(cctx) From 37f4ebbd5ed1c86d7ff7be5912c7c0cf7212a0d6 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 21 Jan 2025 10:57:52 -0600 Subject: [PATCH 04/74] make changelog descriptive; rename specialHandleFeeRate as GetFeeRateForRegnetAndTestnet; code simplification --- changelog.md | 4 +++- zetaclient/chains/bitcoin/client/helpers.go | 8 ++++---- zetaclient/chains/bitcoin/observer/gas_price.go | 13 ++++++------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/changelog.md b/changelog.md index b097e4e560..f25b7e2009 100644 --- a/changelog.md +++ b/changelog.md @@ -7,7 +7,9 @@ * [3332](https://github.com/zeta-chain/node/pull/3332) - implement orchestrator V2. Move BTC observer-signer to V2 * [3360](https://github.com/zeta-chain/node/pull/3360) - update protocol contract imports using consolidated path * [3349](https://github.com/zeta-chain/node/pull/3349) - implement new bitcoin rpc in zetaclient with improved performance and observability -* [3381](https://github.com/zeta-chain/node/pull/3381) - some code refactoring in zetaclient for Bitcoin RBF adoption +* [3381](https://github.com/zeta-chain/node/pull/3381) - split Bitcoin observer and signer into small files and organize outbound logic into reusable/testable functions; renaming, type unification, etc. + +* some code refactoring in zetaclient for Bitcoin RBF adoption ### Fixes diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index abc05f9af7..6797070c40 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -131,11 +131,11 @@ func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regn if feeResult.FeeRate == nil { return 0, fmt.Errorf("nil fee rate") } - if *feeResult.FeeRate <= 0 || *feeResult.FeeRate >= maxBTCSupply { - return 0, fmt.Errorf("invalid fee rate: %f", *feeResult.FeeRate) + feeRate := *feeResult.FeeRate + if feeRate <= 0 || feeRate >= maxBTCSupply { + return 0, fmt.Errorf("invalid fee rate: %f", feeRate) } - - return common.FeeRateToSatPerByte(*feeResult.FeeRate), nil + return common.FeeRateToSatPerByte(feeRate), nil } // GetTransactionFeeAndRate gets the transaction fee and rate for a given tx result diff --git a/zetaclient/chains/bitcoin/observer/gas_price.go b/zetaclient/chains/bitcoin/observer/gas_price.go index 1263b3a2ef..54786fc67d 100644 --- a/zetaclient/chains/bitcoin/observer/gas_price.go +++ b/zetaclient/chains/bitcoin/observer/gas_price.go @@ -19,16 +19,13 @@ func (ob *Observer) PostGasPrice(ctx context.Context) error { ) // special handle regnet and testnet gas rate - // regnet: RPC 'EstimateSmartFee' is not available - // testnet: RPC 'EstimateSmartFee' returns unreasonable high gas rate if ob.Chain().NetworkType != chains.NetworkType_mainnet { - feeRateEstimated, err = ob.specialHandleFeeRate(ctx) + feeRateEstimated, err = ob.GetFeeRateForRegnetAndTestnet(ctx) if err != nil { return errors.Wrap(err, "unable to execute specialHandleFeeRate") } } else { - isRegnet := chains.IsBitcoinRegnet(ob.Chain().ChainId) - feeRateEstimated, err = ob.rpc.GetEstimatedFeeRate(ctx, 1, isRegnet) + feeRateEstimated, err = ob.rpc.GetEstimatedFeeRate(ctx, 1, false) if err != nil { return errors.Wrap(err, "unable to get estimated fee rate") } @@ -53,8 +50,10 @@ func (ob *Observer) PostGasPrice(ctx context.Context) error { return nil } -// specialHandleFeeRate handles the fee rate for regnet and testnet -func (ob *Observer) specialHandleFeeRate(ctx context.Context) (int64, error) { +// GetFeeRateForRegnetAndTestnet handles the fee rate for regnet and testnet +// regnet: RPC 'EstimateSmartFee' is not available +// testnet: RPC 'EstimateSmartFee' can return unreasonable high fee rate +func (ob *Observer) GetFeeRateForRegnetAndTestnet(ctx context.Context) (int64, error) { switch ob.Chain().NetworkType { case chains.NetworkType_privnet: return client.FeeRateRegnet, nil From e3f2bc678ce664fe05d9c3394795e8503789a7a5 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 16:18:34 -0600 Subject: [PATCH 05/74] renaming sample function; make switch case in PostGasPrice to estimate fee rate according to Bitcoin network type --- testutil/sample/crypto.go | 10 +++--- x/crosschain/types/cctx_test.go | 2 +- x/crosschain/types/revert_options_test.go | 2 +- zetaclient/chains/bitcoin/client/helpers.go | 2 +- .../chains/bitcoin/observer/event_test.go | 6 ++-- .../chains/bitcoin/observer/gas_price.go | 35 +++++++------------ .../chains/bitcoin/observer/inbound_test.go | 2 +- zetaclient/chains/bitcoin/signer/sign_test.go | 6 ++-- 8 files changed, 27 insertions(+), 38 deletions(-) diff --git a/testutil/sample/crypto.go b/testutil/sample/crypto.go index 6ebf295010..393be2e2c3 100644 --- a/testutil/sample/crypto.go +++ b/testutil/sample/crypto.go @@ -91,8 +91,8 @@ func EthAddressFromRand(r *rand.Rand) ethcommon.Address { return ethcommon.BytesToAddress(sdk.AccAddress(PubKey(r).Address()).Bytes()) } -// BtcAddressP2WPKH returns a sample btc P2WPKH address -func BtcAddressP2WPKH(t *testing.T, net *chaincfg.Params) *btcutil.AddressWitnessPubKeyHash { +// BTCAddressP2WPKH returns a sample Bitcoin P2WPKH address +func BTCAddressP2WPKH(t *testing.T, net *chaincfg.Params) *btcutil.AddressWitnessPubKeyHash { privateKey, err := btcec.NewPrivateKey() require.NoError(t, err) @@ -103,9 +103,9 @@ func BtcAddressP2WPKH(t *testing.T, net *chaincfg.Params) *btcutil.AddressWitnes return addr } -// BtcAddressP2WPKH returns a pkscript for a sample btc P2WPKH address -func BtcAddressP2WPKHScript(t *testing.T, net *chaincfg.Params) []byte { - addr := BtcAddressP2WPKH(t, net) +// BtcAddressP2WPKH returns a pkscript for a sample Bitcoin P2WPKH address +func BTCAddressP2WPKHScript(t *testing.T, net *chaincfg.Params) []byte { + addr := BTCAddressP2WPKH(t, net) script, err := txscript.PayToAddrScript(addr) require.NoError(t, err) return script diff --git a/x/crosschain/types/cctx_test.go b/x/crosschain/types/cctx_test.go index 1e2d6c830d..6ef409e296 100644 --- a/x/crosschain/types/cctx_test.go +++ b/x/crosschain/types/cctx_test.go @@ -140,7 +140,7 @@ func Test_SetRevertOutboundValues(t *testing.T) { cctx := sample.CrossChainTx(t, "test") cctx.InboundParams.SenderChainId = chains.BitcoinTestnet.ChainId cctx.OutboundParams = cctx.OutboundParams[:1] - cctx.RevertOptions.RevertAddress = sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params).String() + cctx.RevertOptions.RevertAddress = sample.BTCAddressP2WPKH(t, &chaincfg.TestNet3Params).String() err := cctx.AddRevertOutbound(100) require.NoError(t, err) diff --git a/x/crosschain/types/revert_options_test.go b/x/crosschain/types/revert_options_test.go index 3fa6a6e8ba..f77fb83363 100644 --- a/x/crosschain/types/revert_options_test.go +++ b/x/crosschain/types/revert_options_test.go @@ -49,7 +49,7 @@ func TestRevertOptions_GetEVMRevertAddress(t *testing.T) { func TestRevertOptions_GetBTCRevertAddress(t *testing.T) { t.Run("valid Bitcoin revert address", func(t *testing.T) { - addr := sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params).String() + addr := sample.BTCAddressP2WPKH(t, &chaincfg.TestNet3Params).String() actualAddr, valid := types.RevertOptions{ RevertAddress: addr, }.GetBTCRevertAddress(chains.BitcoinTestnet.ChainId) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 6797070c40..395c8ccd4f 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -129,7 +129,7 @@ func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regn return 0, fmt.Errorf("fee result contains errors: %s", feeResult.Errors) } if feeResult.FeeRate == nil { - return 0, fmt.Errorf("nil fee rate") + return 0, errors.New("nil fee rate") } feeRate := *feeResult.FeeRate if feeRate <= 0 || feeRate >= maxBTCSupply { diff --git a/zetaclient/chains/bitcoin/observer/event_test.go b/zetaclient/chains/bitcoin/observer/event_test.go index ca8d79e155..a616f39d37 100644 --- a/zetaclient/chains/bitcoin/observer/event_test.go +++ b/zetaclient/chains/bitcoin/observer/event_test.go @@ -32,7 +32,7 @@ func createTestBtcEvent( memoStd *memo.InboundMemo, ) observer.BTCInboundEvent { return observer.BTCInboundEvent{ - FromAddress: sample.BtcAddressP2WPKH(t, net).String(), + FromAddress: sample.BTCAddressP2WPKH(t, net).String(), ToAddress: sample.EthAddress().Hex(), MemoBytes: memo, MemoStd: memoStd, @@ -249,7 +249,7 @@ func Test_ValidateStandardMemo(t *testing.T) { }, FieldsV0: memo.FieldsV0{ RevertOptions: crosschaintypes.RevertOptions{ - RevertAddress: sample.BtcAddressP2WPKH(t, &chaincfg.TestNet3Params).String(), + RevertAddress: sample.BTCAddressP2WPKH(t, &chaincfg.TestNet3Params).String(), }, }, }, @@ -400,7 +400,7 @@ func Test_NewInboundVoteFromStdMemo(t *testing.T) { t.Run("should create new inbound vote msg with standard memo", func(t *testing.T) { // create revert options revertOptions := crosschaintypes.NewEmptyRevertOptions() - revertOptions.RevertAddress = sample.BtcAddressP2WPKH(t, &chaincfg.MainNetParams).String() + revertOptions.RevertAddress = sample.BTCAddressP2WPKH(t, &chaincfg.MainNetParams).String() // create test event receiver := sample.EthAddress() diff --git a/zetaclient/chains/bitcoin/observer/gas_price.go b/zetaclient/chains/bitcoin/observer/gas_price.go index 54786fc67d..ea6d85e6bd 100644 --- a/zetaclient/chains/bitcoin/observer/gas_price.go +++ b/zetaclient/chains/bitcoin/observer/gas_price.go @@ -18,17 +18,24 @@ func (ob *Observer) PostGasPrice(ctx context.Context) error { feeRateEstimated int64 ) - // special handle regnet and testnet gas rate - if ob.Chain().NetworkType != chains.NetworkType_mainnet { - feeRateEstimated, err = ob.GetFeeRateForRegnetAndTestnet(ctx) + // estimate fee rate according to network type + switch ob.Chain().NetworkType { + case chains.NetworkType_privnet: + // regnet RPC 'EstimateSmartFee' is not available + feeRateEstimated = client.FeeRateRegnet + case chains.NetworkType_testnet: + // testnet RPC 'EstimateSmartFee' can return unreasonable high fee rate + feeRateEstimated, err = common.GetRecentFeeRate(ctx, ob.rpc, ob.netParams) if err != nil { - return errors.Wrap(err, "unable to execute specialHandleFeeRate") + return errors.Wrapf(err, "unable to get recent fee rate") } - } else { + case chains.NetworkType_mainnet: feeRateEstimated, err = ob.rpc.GetEstimatedFeeRate(ctx, 1, false) if err != nil { return errors.Wrap(err, "unable to get estimated fee rate") } + default: + return fmt.Errorf("unsupported bitcoin network type %d", ob.Chain().NetworkType) } // query the current block number @@ -49,21 +56,3 @@ func (ob *Observer) PostGasPrice(ctx context.Context) error { return nil } - -// GetFeeRateForRegnetAndTestnet handles the fee rate for regnet and testnet -// regnet: RPC 'EstimateSmartFee' is not available -// testnet: RPC 'EstimateSmartFee' can return unreasonable high fee rate -func (ob *Observer) GetFeeRateForRegnetAndTestnet(ctx context.Context) (int64, error) { - switch ob.Chain().NetworkType { - case chains.NetworkType_privnet: - return client.FeeRateRegnet, nil - case chains.NetworkType_testnet: - feeRateEstimated, err := common.GetRecentFeeRate(ctx, ob.rpc, ob.netParams) - if err != nil { - return 0, errors.Wrapf(err, "error GetRecentFeeRate") - } - return feeRateEstimated, nil - default: - return 0, fmt.Errorf(" unsupported bitcoin network type %d", ob.Chain().NetworkType) - } -} diff --git a/zetaclient/chains/bitcoin/observer/inbound_test.go b/zetaclient/chains/bitcoin/observer/inbound_test.go index d2775d9d28..e57b5a6d56 100644 --- a/zetaclient/chains/bitcoin/observer/inbound_test.go +++ b/zetaclient/chains/bitcoin/observer/inbound_test.go @@ -167,7 +167,7 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { { name: "should return vote for standard memo", event: &observer.BTCInboundEvent{ - FromAddress: sample.BtcAddressP2WPKH(t, &chaincfg.MainNetParams).String(), + FromAddress: sample.BTCAddressP2WPKH(t, &chaincfg.MainNetParams).String(), // a deposit and call MemoBytes: testutil.HexToBytes( t, diff --git a/zetaclient/chains/bitcoin/signer/sign_test.go b/zetaclient/chains/bitcoin/signer/sign_test.go index f550204b0f..95e90fe580 100644 --- a/zetaclient/chains/bitcoin/signer/sign_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_test.go @@ -36,13 +36,13 @@ func Test_AddTxInputs(t *testing.T) { { TxID: sample.BtcHash().String(), Vout: 0, - Address: sample.BtcAddressP2WPKH(t, net).String(), + Address: sample.BTCAddressP2WPKH(t, net).String(), Amount: 0.1, }, { TxID: sample.BtcHash().String(), Vout: 1, - Address: sample.BtcAddressP2WPKH(t, net).String(), + Address: sample.BTCAddressP2WPKH(t, net).String(), Amount: 0.2, }, }, @@ -308,7 +308,7 @@ func Test_SignTx(t *testing.T) { // add outputs for _, amount := range tt.outputs { - pkScript := sample.BtcAddressP2WPKHScript(t, tt.net) + pkScript := sample.BTCAddressP2WPKHScript(t, tt.net) tx.AddTxOut(wire.NewTxOut(amount, pkScript)) } From 009b55b221107178068988eaf952f1eb4e28c2df Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 17:59:48 -0600 Subject: [PATCH 06/74] remove redundant variable description --- zetaclient/chains/bitcoin/observer/observer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 72e78a0c6b..3ea17950e8 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -104,7 +104,6 @@ type Observer struct { // utxos contains the UTXOs owned by the TSS address utxos []btcjson.ListUnspentResult - // tssOutboundHashes indexes included tx with tx hash // tssOutboundHashes keeps track of outbound hashes sent from TSS address tssOutboundHashes map[string]bool From 09fb0ac824d2b44baf2523bcc378336e94e99568 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 18:28:21 -0600 Subject: [PATCH 07/74] make AddWithdrawTxOutputs a one-line call --- zetaclient/chains/bitcoin/signer/sign.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index d20ca9fb0e..97f04b3c9b 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -92,15 +92,8 @@ func (signer *Signer) SignWithdrawTx( txData.nonce, txData.feeRate, txSize, fees, selected.ConsolidatedUTXOs, selected.ConsolidatedValue) // add tx outputs - err = signer.AddWithdrawTxOutputs( - tx, - txData.to, - selected.Value, - txData.amountSats, - nonceMark, - fees, - txData.cancelTx, - ) + inputValue := selected.Value + err = signer.AddWithdrawTxOutputs(tx, txData.to, inputValue, txData.amountSats, nonceMark, fees, txData.cancelTx) if err != nil { return nil, err } From aeceb26307d933042e3d2c4bb42b280fe1158447 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 20:42:56 -0600 Subject: [PATCH 08/74] implementation of Bitcoin RBF transaction --- pkg/math/integer.go | 40 ++ pkg/math/integer_test.go | 31 + zetaclient/chains/bitcoin/bitcoin.go | 4 + .../chains/bitcoin/client/client_rbf_test.go | 622 ++++++++++++++++++ zetaclient/chains/bitcoin/client/commands.go | 35 + zetaclient/chains/bitcoin/client/helpers.go | 71 ++ zetaclient/chains/bitcoin/client/mockgen.go | 7 + zetaclient/chains/bitcoin/observer/mempool.go | 277 ++++++++ .../chains/bitcoin/observer/mempool_test.go | 441 +++++++++++++ .../chains/bitcoin/observer/observer.go | 42 +- .../chains/bitcoin/observer/observer_test.go | 35 + .../chains/bitcoin/observer/outbound.go | 11 +- .../chains/bitcoin/signer/fee_bumper.go | 208 ++++++ .../chains/bitcoin/signer/fee_bumper_test.go | 383 +++++++++++ zetaclient/chains/bitcoin/signer/sign.go | 45 +- zetaclient/chains/bitcoin/signer/sign_rbf.go | 94 +++ .../chains/bitcoin/signer/sign_rbf_test.go | 242 +++++++ zetaclient/chains/bitcoin/signer/signer.go | 56 +- .../chains/bitcoin/signer/signer_test.go | 31 +- zetaclient/common/constant.go | 3 + zetaclient/testutils/mocks/bitcoin_client.go | 109 +++ 21 files changed, 2749 insertions(+), 38 deletions(-) create mode 100644 pkg/math/integer.go create mode 100644 pkg/math/integer_test.go create mode 100644 zetaclient/chains/bitcoin/client/client_rbf_test.go create mode 100644 zetaclient/chains/bitcoin/observer/mempool.go create mode 100644 zetaclient/chains/bitcoin/observer/mempool_test.go create mode 100644 zetaclient/chains/bitcoin/signer/fee_bumper.go create mode 100644 zetaclient/chains/bitcoin/signer/fee_bumper_test.go create mode 100644 zetaclient/chains/bitcoin/signer/sign_rbf.go create mode 100644 zetaclient/chains/bitcoin/signer/sign_rbf_test.go diff --git a/pkg/math/integer.go b/pkg/math/integer.go new file mode 100644 index 0000000000..8d74b5dc27 --- /dev/null +++ b/pkg/math/integer.go @@ -0,0 +1,40 @@ +// Package implements helper functions for integer math operations. +package math + +import ( + "math" + "math/big" +) + +// IncreaseIntByPercent is a function that increases integer by a percentage. +// Example1: IncreaseIntByPercent(10, 15) = 10 * 1.15 = 11 +// Example2: IncreaseIntByPercent(-10, 15) = -10 * 1.15 = -11 +// +// Note: use with caution if passing negative values. +func IncreaseIntByPercent(value int64, percent uint32) int64 { + if percent == 0 { + return value + } + + if value < 0 { + return -IncreaseIntByPercent(-value, percent) + } + + bigValue := big.NewInt(value) + bigPercent := big.NewInt(int64(percent)) + + // product = value * percent + product := new(big.Int).Mul(bigValue, bigPercent) + + // dividing product by 100 + product.Div(product, big.NewInt(100)) + + // result = original value + product + result := new(big.Int).Add(bigValue, product) + + // be mindful if result > MaxInt64 + if result.Cmp(big.NewInt(math.MaxInt64)) > 0 { + return math.MaxInt64 + } + return result.Int64() +} diff --git a/pkg/math/integer_test.go b/pkg/math/integer_test.go new file mode 100644 index 0000000000..daf6f966fc --- /dev/null +++ b/pkg/math/integer_test.go @@ -0,0 +1,31 @@ +package math + +import ( + "fmt" + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_IncreaseIntByPercent(t *testing.T) { + for i, tt := range []struct { + value int64 + percent uint32 + expected int64 + }{ + {value: 10, percent: 0, expected: 10}, + {value: 10, percent: 15, expected: 11}, + {value: 10, percent: 225, expected: 32}, + {value: math.MaxInt64 / 2, percent: 101, expected: math.MaxInt64}, + {value: -10, percent: 0, expected: -10}, + {value: -10, percent: 15, expected: -11}, + {value: -10, percent: 225, expected: -32}, + {value: -math.MaxInt64 / 2, percent: 101, expected: -math.MaxInt64}, + } { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + result := IncreaseIntByPercent(tt.value, tt.percent) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/zetaclient/chains/bitcoin/bitcoin.go b/zetaclient/chains/bitcoin/bitcoin.go index 0cfcbe1bad..a3ebe16f38 100644 --- a/zetaclient/chains/bitcoin/bitcoin.go +++ b/zetaclient/chains/bitcoin/bitcoin.go @@ -13,6 +13,7 @@ import ( "github.com/zeta-chain/node/pkg/ticker" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" + "github.com/zeta-chain/node/zetaclient/common" zctx "github.com/zeta-chain/node/zetaclient/context" "github.com/zeta-chain/node/zetaclient/outboundprocessor" ) @@ -72,6 +73,8 @@ func (b *Bitcoin) Start(ctx context.Context) error { return ticker.DurationFromUint64Seconds(b.observer.ChainParams().WatchUtxoTicker) }) + optMempoolInterval := scheduler.Interval(common.BTCMempoolStuckTxCheckInterval) + optOutboundInterval := scheduler.IntervalUpdater(func() time.Duration { return ticker.DurationFromUint64Seconds(b.observer.ChainParams().OutboundTicker) }) @@ -101,6 +104,7 @@ func (b *Bitcoin) Start(ctx context.Context) error { register(b.observer.ObserveInbound, "observe_inbound", optInboundInterval, optInboundSkipper) register(b.observer.ObserveInboundTrackers, "observe_inbound_trackers", optInboundInterval, optInboundSkipper) register(b.observer.FetchUTXOs, "fetch_utxos", optUTXOInterval, optGenericSkipper) + register(b.observer.WatchMempoolTxs, "watch_mempool_txs", optMempoolInterval, optGenericSkipper) register(b.observer.PostGasPrice, "post_gas_price", optGasInterval, optGenericSkipper) register(b.observer.CheckRPCStatus, "check_rpc_status") register(b.observer.ObserveOutbound, "observe_outbound", optOutboundInterval, optOutboundSkipper) diff --git a/zetaclient/chains/bitcoin/client/client_rbf_test.go b/zetaclient/chains/bitcoin/client/client_rbf_test.go new file mode 100644 index 0000000000..91dd2dc49f --- /dev/null +++ b/zetaclient/chains/bitcoin/client/client_rbf_test.go @@ -0,0 +1,622 @@ +package client_test + +import ( + "context" + "encoding/hex" + "fmt" + "os" + "sort" + "testing" + "time" + + "github.com/btcsuite/btcd/btcec/v2/ecdsa" + "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/mempool" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" + btccommon "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" + "github.com/zeta-chain/node/zetaclient/common" + "github.com/zeta-chain/node/zetaclient/config" +) + +// Test_BitcoinLive runs RBF tests on a live Bitcoin network. +func Test_BitcoinRBFLive(t *testing.T) { + if !common.LiveTestEnabled() { + t.Skip("skipping live test") + } + + LiveTest_RBFTransaction(t) + LiveTest_RBFTransaction_Chained_CPFP(t) + LiveTest_PendingMempoolTx(t) +} + +// setupRBFTest initializes the test suite, privateKey, sender, receiver +func setupRBFTest(t *testing.T) (*testSuite, *secp256k1.PrivateKey, btcutil.Address, btcutil.Address) { + // network to use + chain := chains.BitcoinTestnet4 + net, err := chains.GetBTCChainParams(chain.ChainId) + require.NoError(t, err) + config := config.BTCConfig{ + RPCHost: os.Getenv(common.EnvBtcRPCMainnet), + RPCParams: "testnet3", + } + + // load test private key + privKeyHex := os.Getenv("TEST_PK_BTC") + privKeyBytes, err := hex.DecodeString(privKeyHex) + require.NoError(t, err) + + // construct a secp256k1 private key object + privKey := secp256k1.PrivKeyFromBytes(privKeyBytes) + pubKeyHash := btcutil.Hash160(privKey.PubKey().SerializeCompressed()) + sender, err := btcutil.NewAddressWitnessPubKeyHash(pubKeyHash, net) + require.NoError(t, err) + fmt.Printf("sender : %s\n", sender.EncodeAddress()) + + // receiver address + to, err := btcutil.DecodeAddress("tb1qxr8zcffrkmqwvtkzjz8nxs05p2vs6pt9rzq27a", net) + require.NoError(t, err) + fmt.Printf("receiver: %s\n", to.EncodeAddress()) + + // create a new test suite + ts := newTestSuite(t, config) + + return ts, privKey, sender, to +} + +func LiveTest_RBFTransaction(t *testing.T) { + // setup test + ts, privKey, sender, to := setupRBFTest(t) + + // define amount, fee rate and bump fee reserved + amount := 0.00001 + nonceMark := chains.NonceMarkAmount(1) + feeRate := int64(2) + bumpFeeReserved := int64(10000) + + // STEP 1 + // build and send tx1 + nonceMark += 1 + txHash1 := buildAndSendRBFTx( + t, + ts.ctx, + ts.Client, + privKey, + nil, + sender, + to, + amount, + nonceMark, + feeRate, + bumpFeeReserved, + ) + fmt.Printf("sent tx1: %s\n", txHash1) + + // STEP 2 + // build and send tx2 (child of tx1) + nonceMark += 1 + txHash2 := buildAndSendRBFTx( + t, + ts.ctx, + ts.Client, + privKey, + txHash1, + sender, + to, + amount, + nonceMark, + feeRate, + bumpFeeReserved, + ) + fmt.Printf("sent tx2: %s\n", txHash2) + + // STEP 3 + // wait for a short time before bumping fee + rawTx1, confirmed := waitForTxConfirmation(ts.ctx, ts.Client, sender, txHash1, 10*time.Second) + if confirmed { + fmt.Println("Opps: tx1 confirmed, no chance to bump fee; please start over") + return + } + + // STEP 4 + // bump gas fee for tx1 (the parent of tx2) + // we assume that tx1, tx2 and tx3 have same vBytes for simplicity + // two rules to satisfy: + // - feeTx3 >= feeTx1 + feeTx2 + // - additionalFees >= vSizeTx3 * minRelayFeeRate + // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/rbf.cpp#L166-L183 + minRelayFeeRate := int64(1) + feeRateIncrease := minRelayFeeRate + sizeTx3 := mempool.GetTxVirtualSize(rawTx1) + additionalFees := (sizeTx3 + 1) * (feeRate + feeRateIncrease) // +1 in case Bitcoin Core rounds up the vSize + fmt.Printf("additional fee: %d sats\n", additionalFees) + tx3, err := bumpRBFTxFee(rawTx1.MsgTx(), additionalFees) + require.NoError(t, err) + + // STEP 5 + // sign and send tx3, which replaces tx1 + signTx(t, ts.ctx, ts.Client, privKey, tx3) + txHash3, err := ts.Client.SendRawTransaction(ts.ctx, tx3, true) + require.NoError(t, err) + fmt.Printf("sent tx3: %s\n", txHash3) + + // STEP 6 + // wait for tx3 confirmation + rawTx3, confirmed := waitForTxConfirmation(ts.ctx, ts.Client, sender, txHash3, 30*time.Minute) + require.True(t, confirmed) + printTx(rawTx3.MsgTx()) + fmt.Println("tx3 confirmed") + + // STEP 7 + // tx1 and tx2 must be dropped + ensureTxDropped(t, ts.ctx, ts.Client, txHash1) + fmt.Println("tx1 dropped") + ensureTxDropped(t, ts.ctx, ts.Client, txHash2) + fmt.Println("tx2 dropped") +} + +// Test_RBFTransactionChained_CPFP tests Child-Pays-For-Parent (CPFP) fee bumping strategy for chained RBF transactions +func LiveTest_RBFTransaction_Chained_CPFP(t *testing.T) { + // setup test + ts, privKey, sender, to := setupRBFTest(t) + + // define amount, fee rate and bump fee reserved + amount := 0.00001 + nonceMark := chains.NonceMarkAmount(0) + feeRate := int64(2) + bumpFeeReserved := int64(10000) + + // STEP 1 + // build and send tx1 + nonceMark += 1 + txHash1 := buildAndSendRBFTx( + t, + ts.ctx, + ts.Client, + privKey, + nil, + sender, + to, + amount, + nonceMark, + feeRate, + bumpFeeReserved, + ) + fmt.Printf("sent tx1: %s\n", txHash1) + + // STEP 2 + // build and send tx2 (child of tx1) + nonceMark += 1 + txHash2 := buildAndSendRBFTx( + t, + ts.ctx, + ts.Client, + privKey, + txHash1, + sender, + to, + amount, + nonceMark, + feeRate, + bumpFeeReserved, + ) + fmt.Printf("sent tx2: %s\n", txHash2) + + // STEP 3 + // build and send tx3 (child of tx2) + nonceMark += 1 + txHash3 := buildAndSendRBFTx( + t, + ts.ctx, + ts.Client, + privKey, + txHash2, + sender, + to, + amount, + nonceMark, + feeRate, + bumpFeeReserved, + ) + fmt.Printf("sent tx3: %s\n", txHash3) + + // STEP 4 + // wait for a short time before bumping fee + rawTx3, confirmed := waitForTxConfirmation(ts.ctx, ts.Client, sender, txHash3, 10*time.Second) + if confirmed { + fmt.Println("Opps: tx3 confirmed, no chance to bump fee; please start over") + return + } + + // STEP 5 + // bump gas fee for tx3 (the child/grandchild of tx1/tx2) + // we assume that tx3 has same vBytes as the fee-bump tx (tx4) for simplicity + // two rules to satisfy: + // - feeTx4 >= feeTx3 + // - additionalFees >= vSizeTx4 * minRelayFeeRate + // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/rbf.cpp#L166-L183 + minRelayFeeRate := int64(1) + feeRateIncrease := minRelayFeeRate + additionalFees := (mempool.GetTxVirtualSize(rawTx3) + 1) * feeRateIncrease + fmt.Printf("additional fee: %d sats\n", additionalFees) + tx4, err := bumpRBFTxFee(rawTx3.MsgTx(), additionalFees) + require.NoError(t, err) + + // STEP 6 + // sign and send tx4, which replaces tx3 + signTx(t, ts.ctx, ts.Client, privKey, tx4) + txHash4, err := ts.Client.SendRawTransaction(ts.ctx, tx4, true) + require.NoError(t, err) + fmt.Printf("sent tx4: %s\n", txHash4) + + // STEP 7 + // wait for tx4 confirmation + rawTx4, confirmed := waitForTxConfirmation(ts.ctx, ts.Client, sender, txHash4, 30*time.Minute) + require.True(t, confirmed) + printTx(rawTx4.MsgTx()) + fmt.Println("tx4 confirmed") + + // STEP 8 + // tx3 must be dropped + ensureTxDropped(t, ts.ctx, ts.Client, txHash3) + fmt.Println("tx1 dropped") +} + +func LiveTest_PendingMempoolTx(t *testing.T) { + // network to use + config := config.BTCConfig{ + RPCHost: os.Getenv(common.EnvBtcRPCMainnet), + RPCParams: "mainnet", + } + + // create a new test suite + ts := newTestSuite(t, config) + + // get mempool transactions + mempoolTxs, err := ts.Client.GetRawMempool(ts.ctx) + require.NoError(t, err) + fmt.Printf("mempool txs: %d\n", len(mempoolTxs)) + + // get last block height + lastHeight, err := ts.Client.GetBlockCount(ts.ctx) + require.NoError(t, err) + fmt.Printf("block height: %d\n", lastHeight) + + const ( + // average minutes per block is about 10 minutes + minutesPerBlockAverage = 10.0 + + // maxBlockTimeDiffPercentage is the maximum error percentage between the estimated and actual block time + // note: 25% is a percentage to make sure the test is not too strict + maxBlockTimeDiffPercentage = 0.25 + ) + + // the goal of the test is to ensure the 'Time' and 'Height' provided by the mempool are correct. + // otherwise, zetaclient should not rely on these information to schedule RBF/CPFP transactions. + // loop through the mempool to sample N pending txs that are pending for more than 2 hours + N := 10 + for i := len(mempoolTxs) - 1; i >= 0; i-- { + txHash := mempoolTxs[i] + entry, err := ts.Client.GetMempoolEntry(ts.ctx, txHash.String()) + if err == nil { + require.Positive(t, entry.Fee) + txTime := time.Unix(entry.Time, 0) + txTimeStr := txTime.Format(time.DateTime) + elapsed := time.Since(txTime) + if elapsed > 30*time.Minute { + // calculate average block time + elapsedBlocks := lastHeight - entry.Height + minutesPerBlockCalculated := elapsed.Minutes() / float64(elapsedBlocks) + blockTimeDiff := minutesPerBlockAverage - minutesPerBlockCalculated + if blockTimeDiff < 0 { + blockTimeDiff = -blockTimeDiff + } + + // the block time difference should fall within 25% of the average block time + require.Less(t, blockTimeDiff, minutesPerBlockAverage*maxBlockTimeDiffPercentage) + fmt.Printf( + "txid: %s, height: %d, time: %s, pending: %f minutes, block time: %f minutes, diff: %f%%\n", + txHash, + entry.Height, + txTimeStr, + elapsed.Minutes(), + minutesPerBlockCalculated, + blockTimeDiff/minutesPerBlockAverage*100, + ) + + // break if we have enough samples + if N -= 1; N == 0 { + break + } + } + } + } +} + +// buildAndSendRBFTx builds, signs and sends an RBF transaction +func buildAndSendRBFTx( + t *testing.T, + ctx context.Context, + client *client.Client, + privKey *secp256k1.PrivateKey, + parent *chainhash.Hash, + sender, to btcutil.Address, + amount float64, + nonceMark int64, + feeRate int64, + bumpFeeReserved int64, +) *chainhash.Hash { + // list outputs + utxos := listUTXOs(ctx, client, sender) + require.NotEmpty(t, utxos) + + // ensure all inputs are from the parent tx + if parent != nil { + for _, out := range utxos { + require.Equal(t, parent.String(), out.TxID) + } + } + + // build tx opt-in RBF + tx := buildRBFTx(t, utxos, sender, to, amount, nonceMark, feeRate, bumpFeeReserved) + + // sign tx + signTx(t, ctx, client, privKey, tx) + + // broadcast tx + txHash, err := client.SendRawTransaction(ctx, tx, true) + require.NoError(t, err) + + return txHash +} + +func listUTXOs(ctx context.Context, client *client.Client, address btcutil.Address) []btcjson.ListUnspentResult { + utxos, err := client.ListUnspentMinMaxAddresses(ctx, 0, 9999999, []btcutil.Address{address}) + if err != nil { + fmt.Printf("ListUnspent failed: %s\n", err) + return nil + } + + // sort utxos by amount, txid, vout + sort.SliceStable(utxos, func(i, j int) bool { + if utxos[i].Amount == utxos[j].Amount { + if utxos[i].TxID == utxos[j].TxID { + return utxos[i].Vout < utxos[j].Vout + } + return utxos[i].TxID < utxos[j].TxID + } + return utxos[i].Amount < utxos[j].Amount + }) + + // print utxos + fmt.Println("utxos:") + for _, out := range utxos { + fmt.Printf( + " txid: %s, vout: %d, amount: %f, confirmation: %d\n", + out.TxID, + out.Vout, + out.Amount, + out.Confirmations, + ) + } + + return utxos +} + +func buildRBFTx( + t *testing.T, + utxos []btcjson.ListUnspentResult, + sender, to btcutil.Address, + amount float64, + nonceMark int64, + feeRate int64, + bumpFeeReserved int64, +) *wire.MsgTx { + // build tx with all unspents + total := 0.0 + tx := wire.NewMsgTx(wire.TxVersion) + for _, output := range utxos { + hash, err := chainhash.NewHashFromStr(output.TxID) + require.NoError(t, err) + + // add input + outpoint := wire.NewOutPoint(hash, output.Vout) + txIn := wire.NewTxIn(outpoint, nil, nil) + txIn.Sequence = 1 // opt-in for RBF + tx.AddTxIn(txIn) + total += output.Amount + } + totalSats, err := btccommon.GetSatoshis(total) + require.NoError(t, err) + + // amount to send in satoshis + amountSats, err := btccommon.GetSatoshis(amount) + require.NoError(t, err) + + // calculate tx fee + txSize, err := btccommon.EstimateOutboundSize(int64(len(utxos)), []btcutil.Address{to}) + require.NoError(t, err) + fees := int64(txSize) * feeRate + + // make sure total is greater than amount + fees + require.GreaterOrEqual(t, totalSats, nonceMark+amountSats+fees+bumpFeeReserved) + + // 1st output: simulated nonce-mark amount to self + pkScriptSender, err := txscript.PayToAddrScript(sender) + require.NoError(t, err) + txOut0 := wire.NewTxOut(nonceMark, pkScriptSender) + tx.AddTxOut(txOut0) + + // 2nd output: payment to receiver + pkScriptReceiver, err := txscript.PayToAddrScript(to) + require.NoError(t, err) + txOut1 := wire.NewTxOut(amountSats, pkScriptReceiver) + tx.AddTxOut(txOut1) + + // 3rd output: change to self + changeSats := totalSats - nonceMark - amountSats - fees + require.GreaterOrEqual(t, changeSats, bumpFeeReserved) + txOut2 := wire.NewTxOut(changeSats, pkScriptSender) + tx.AddTxOut(txOut2) + + return tx +} + +func signTx(t *testing.T, ctx context.Context, client *client.Client, privKey *secp256k1.PrivateKey, tx *wire.MsgTx) { + // we know that the first output is the nonce-mark amount, so it contains the sender pkScript + pkScriptSender := tx.TxOut[0].PkScript + + sigHashes := txscript.NewTxSigHashes(tx, txscript.NewCannedPrevOutputFetcher([]byte{}, 0)) + for idx, input := range tx.TxIn { + // get input amount from previous tx outpoint via RPC + rawTx, err := client.GetRawTransaction(ctx, &input.PreviousOutPoint.Hash) + require.NoError(t, err) + amount := rawTx.MsgTx().TxOut[input.PreviousOutPoint.Index].Value + + // calculate witness signature hash for signing + witnessHash, err := txscript.CalcWitnessSigHash(pkScriptSender, sigHashes, txscript.SigHashAll, tx, idx, amount) + require.NoError(t, err) + + // sign the witness hash + sig := ecdsa.Sign(privKey, witnessHash) + tx.TxIn[idx].Witness = wire.TxWitness{ + append(sig.Serialize(), byte(txscript.SigHashAll)), + privKey.PubKey().SerializeCompressed(), + } + } + + printTx(tx) +} + +func printTx(tx *wire.MsgTx) { + fmt.Printf("\n==============================================================\n") + fmt.Printf("tx version: %d\n", tx.Version) + fmt.Printf("tx locktime: %d\n", tx.LockTime) + fmt.Println("tx inputs:") + for i, vin := range tx.TxIn { + fmt.Printf(" input[%d]:\n", i) + fmt.Printf(" prevout hash: %s\n", vin.PreviousOutPoint.Hash) + fmt.Printf(" prevout index: %d\n", vin.PreviousOutPoint.Index) + fmt.Printf(" sig script: %s\n", hex.EncodeToString(vin.SignatureScript)) + fmt.Printf(" sequence: %d\n", vin.Sequence) + fmt.Printf(" witness: \n") + for j, w := range vin.Witness { + fmt.Printf(" witness[%d]: %s\n", j, hex.EncodeToString(w)) + } + } + fmt.Println("tx outputs:") + for i, vout := range tx.TxOut { + fmt.Printf(" output[%d]:\n", i) + fmt.Printf(" value: %d\n", vout.Value) + fmt.Printf(" script: %s\n", hex.EncodeToString(vout.PkScript)) + } + fmt.Printf("==============================================================\n\n") +} + +func peekUnconfirmedTx(ctx context.Context, client *client.Client, txHash *chainhash.Hash) (*btcutil.Tx, bool) { + confirmed := false + + // try querying tx result + getTxResult, err := client.GetTransaction(ctx, txHash) + if err == nil { + confirmed = getTxResult.Confirmations > 0 + fmt.Printf("tx confirmations: %d\n", getTxResult.Confirmations) + } else { + fmt.Printf("GetTxResultByHash failed: %s\n", err) + } + + // query tx from mempool + entry, err := client.GetMempoolEntry(ctx, txHash.String()) + switch { + case err != nil: + fmt.Println("tx in mempool: NO") + default: + txTime := time.Unix(entry.Time, 0) + txTimeStr := txTime.Format(time.DateTime) + elapsed := int64(time.Since(txTime).Seconds()) + fmt.Printf( + "tx in mempool: YES, VSize: %d, height: %d, time: %s, elapsed: %d\n", + entry.VSize, + entry.Height, + txTimeStr, + elapsed, + ) + } + + // query the raw tx + rawTx, err := client.GetRawTransaction(ctx, txHash) + if err != nil { + fmt.Printf("GetRawTransaction failed: %s\n", err) + } + + return rawTx, confirmed +} + +func waitForTxConfirmation( + ctx context.Context, + client *client.Client, + sender btcutil.Address, + txHash *chainhash.Hash, + timeOut time.Duration, +) (*btcutil.Tx, bool) { + start := time.Now() + for { + rawTx, confirmed := peekUnconfirmedTx(ctx, client, txHash) + listUTXOs(ctx, client, sender) + fmt.Println() + + if confirmed { + return rawTx, true + } + if time.Since(start) > timeOut { + return rawTx, false + } + + time.Sleep(5 * time.Second) + } +} + +func bumpRBFTxFee(oldTx *wire.MsgTx, additionalFee int64) (*wire.MsgTx, error) { + // copy the old tx and reset + newTx := oldTx.Copy() + for idx := range newTx.TxIn { + newTx.TxIn[idx].Witness = wire.TxWitness{} + newTx.TxIn[idx].Sequence = 1 + } + + // original change needs to be enough to cover the additional fee + if newTx.TxOut[2].Value <= additionalFee { + return nil, errors.New("change amount is not enough to cover the additional fee") + } + + // bump fee by reducing the change amount + newTx.TxOut[2].Value = newTx.TxOut[2].Value - additionalFee + + return newTx, nil +} + +func ensureTxDropped(t *testing.T, ctx context.Context, client *client.Client, txHash *chainhash.Hash) { + // dropped tx must has negative confirmations (if returned) + getTxResult, err := client.GetTransaction(ctx, txHash) + if err == nil { + require.Negative(t, getTxResult.Confirmations) + } + + // dropped tx should be removed from mempool + entry, err := client.GetMempoolEntry(ctx, txHash.String()) + require.Error(t, err) + require.Nil(t, entry) + + // dropped tx should not be found + // -5: No such mempool or blockchain transaction + rawTx, err := client.GetRawTransaction(ctx, txHash) + require.Error(t, err) + require.Nil(t, rawTx) +} diff --git a/zetaclient/chains/bitcoin/client/commands.go b/zetaclient/chains/bitcoin/client/commands.go index 1a7954d6f1..393f6b8933 100644 --- a/zetaclient/chains/bitcoin/client/commands.go +++ b/zetaclient/chains/bitcoin/client/commands.go @@ -77,6 +77,41 @@ func (c *Client) GetBlockHeader(ctx context.Context, hash *chainhash.Hash) (*wir return &bh, nil } +func (c *Client) GetRawMempool(ctx context.Context) ([]*chainhash.Hash, error) { + cmd := types.NewGetRawMempoolCmd(types.Bool(false)) + + out, err := c.sendCommand(ctx, cmd) + if err != nil { + return nil, errors.Wrap(err, "unable to list unspent") + } + + txHashStrs, err := unmarshal[[]string](out) + if err != nil { + return nil, errors.Wrap(err, "unable to unmarshal to strings") + } + + txHashes := make([]*chainhash.Hash, len(txHashStrs)) + for i, hashString := range txHashStrs { + txHashes[i], err = chainhash.NewHashFromStr(hashString) + if err != nil { + return nil, err + } + } + + return txHashes, nil +} + +func (c *Client) GetMempoolEntry(ctx context.Context, txHash string) (*types.GetMempoolEntryResult, error) { + cmd := types.NewGetMempoolEntryCmd(txHash) + + out, err := c.sendCommand(ctx, cmd) + if err != nil { + return nil, errors.Wrapf(err, "unable to get mempool entry for %s", txHash) + } + + return unmarshalPtr[types.GetMempoolEntryResult](out) +} + func (c *Client) GetBlockVerbose(ctx context.Context, hash *chainhash.Hash) (*types.GetBlockVerboseTxResult, error) { cmd := types.NewGetBlockCmd(hash.String(), types.Int(2)) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 395c8ccd4f..872c834118 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -3,6 +3,8 @@ package client import ( "context" "fmt" + "math" + "strings" "time" types "github.com/btcsuite/btcd/btcjson" @@ -17,6 +19,9 @@ const ( // FeeRateRegnet is the hardcoded fee rate for regnet FeeRateRegnet = 1 + // FeeRateRegnetRBF is the hardcoded fee rate for regnet RBF + FeeRateRegnetRBF = 5 + // maxBTCSupply is the maximum supply of Bitcoin maxBTCSupply = 21000000.0 ) @@ -219,6 +224,72 @@ func (c *Client) Healthcheck(ctx context.Context, tssAddress btcutil.Address) (t return header.Timestamp, nil } +// GetTotalMempoolParentsSizeNFees returns the information of all pending parent txs of a given tx (inclusive) +// +// A parent tx is defined as: +// - a tx that is also pending in the mempool +// - a tx that has its first output spent by the child as first input +// +// Returns: (totalTxs, totalFees, totalVSize, error) +func (c *Client) GetTotalMempoolParentsSizeNFees( + ctx context.Context, + childHash string, + timeout time.Duration, +) (int64, float64, int64, int64, error) { + var ( + totalTxs int64 + totalFees float64 + totalVSize int64 + avgFeeRate int64 + ) + + // loop through all parents + startTime := time.Now() + parentHash := childHash + for { + memplEntry, err := c.GetMempoolEntry(ctx, parentHash) + if err != nil { + if strings.Contains(err.Error(), "Transaction not in mempool") { + // not a mempool tx, stop looking for parents + break + } + return 0, 0, 0, 0, errors.Wrapf(err, "unable to get mempool entry for tx %s", parentHash) + } + + // accumulate fees and vsize + totalTxs++ + totalFees += memplEntry.Fee + totalVSize += int64(memplEntry.VSize) + + // find the parent tx + tx, err := c.GetRawTransactionByStr(ctx, parentHash) + if err != nil { + return 0, 0, 0, 0, errors.Wrapf(err, "unable to get tx %s", parentHash) + } + parentHash = tx.MsgTx().TxIn[0].PreviousOutPoint.Hash.String() + + // check timeout to avoid infinite loop + if time.Since(startTime) > timeout { + return 0, 0, 0, 0, errors.Errorf("timeout reached on %dth tx: %s", totalTxs, parentHash) + } + } + + // no pending tx found + if totalTxs == 0 { + return 0, 0, 0, 0, errors.Errorf("given tx is not pending: %s", childHash) + } + + // sanity check, should never happen + if totalFees < 0 || totalVSize <= 0 { + return 0, 0, 0, 0, errors.Errorf("invalid result: totalFees %f, totalVSize %d", totalFees, totalVSize) + } + + // calculate the average fee rate + avgFeeRate = int64(math.Ceil(totalFees / float64(totalVSize))) + + return totalTxs, totalFees, totalVSize, avgFeeRate, nil +} + func strToHash(s string) (*chainhash.Hash, error) { hash, err := chainhash.NewHashFromStr(s) if err != nil { diff --git a/zetaclient/chains/bitcoin/client/mockgen.go b/zetaclient/chains/bitcoin/client/mockgen.go index 69cfe1cd6b..229e9e2b75 100644 --- a/zetaclient/chains/bitcoin/client/mockgen.go +++ b/zetaclient/chains/bitcoin/client/mockgen.go @@ -29,6 +29,13 @@ type client interface { GetTransaction(ctx context.Context, hash *hash.Hash) (*types.GetTransactionResult, error) GetRawTransaction(ctx context.Context, hash *hash.Hash) (*btcutil.Tx, error) GetRawTransactionVerbose(ctx context.Context, hash *hash.Hash) (*types.TxRawResult, error) + GetMempoolEntry(ctx context.Context, txHash string) (*types.GetMempoolEntryResult, error) + GetRawMempool(ctx context.Context) ([]*hash.Hash, error) + GetTotalMempoolParentsSizeNFees( + ctx context.Context, + childHash string, + timeout time.Duration, + ) (int64, float64, int64, int64, error) GetRawTransactionResult( ctx context.Context, diff --git a/zetaclient/chains/bitcoin/observer/mempool.go b/zetaclient/chains/bitcoin/observer/mempool.go new file mode 100644 index 0000000000..82ca6a4fb4 --- /dev/null +++ b/zetaclient/chains/bitcoin/observer/mempool.go @@ -0,0 +1,277 @@ +package observer + +import ( + "context" + "strings" + "time" + + "github.com/btcsuite/btcd/btcutil" + "github.com/pkg/errors" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/zetaclient/logs" +) + +const ( + // PendingTxFeeBumpWaitBlocks is the number of blocks to await before considering a tx stuck in mempool + PendingTxFeeBumpWaitBlocks = 3 + + // PendingTxFeeBumpWaitBlocksRegnet is the number of blocks to await before considering a tx stuck in mempool in regnet + // Note: this is used for E2E test only + PendingTxFeeBumpWaitBlocksRegnet = 30 + + // blockTimeBTC represents the average time to mine a block in Bitcoin + blockTimeBTC = 10 * time.Minute +) + +// LastStuckOutbound contains the last stuck outbound tx information. +type LastStuckOutbound struct { + // Nonce is the nonce of the outbound. + Nonce uint64 + + // Tx is the original transaction. + Tx *btcutil.Tx + + // StuckFor is the duration for which the tx has been stuck. + StuckFor time.Duration +} + +// NewLastStuckOutbound creates a new LastStuckOutbound struct. +func NewLastStuckOutbound(nonce uint64, tx *btcutil.Tx, stuckFor time.Duration) *LastStuckOutbound { + return &LastStuckOutbound{ + Nonce: nonce, + Tx: tx, + StuckFor: stuckFor, + } +} + +// PendingTxFinder is a function type for finding the last Bitcoin pending tx. +type PendingTxFinder func(ctx context.Context, ob *Observer) (*btcutil.Tx, uint64, error) + +// StuckTxChecker is a function type for checking if a tx is stuck in the mempool. +type StuckTxChecker func(ctx context.Context, rpc RPC, txHash string, maxWaitBlocks int64) (bool, time.Duration, error) + +// WatchMempoolTxs monitors pending outbound txs in the Bitcoin mempool. +func (ob *Observer) WatchMempoolTxs(ctx context.Context) error { + txChecker := GetStuckTxChecker(ob.Chain().ChainId) + + if err := ob.RefreshLastStuckOutbound(ctx, GetLastPendingOutbound, txChecker); err != nil { + ob.Logger().Chain.Err(err).Msg("RefreshLastStuckOutbound failed") + } + return nil +} + +// RefreshLastStuckOutbound refreshes the information about the last stuck tx in the Bitcoin mempool. +// Once 2/3+ of the observers reach consensus on last stuck outbound, RBF will start. +func (ob *Observer) RefreshLastStuckOutbound( + ctx context.Context, + txFinder PendingTxFinder, + txChecker StuckTxChecker, +) error { + lf := map[string]any{ + logs.FieldMethod: "RefreshLastStuckOutbound", + } + + // step 1: get last TSS transaction + lastTx, lastNonce, err := txFinder(ctx, ob) + if err != nil { + ob.logger.Outbound.Info().Fields(lf).Msgf("last pending outbound not found: %s", err.Error()) + return nil + } + + // log fields + txHash := lastTx.MsgTx().TxID() + lf[logs.FieldNonce] = lastNonce + lf[logs.FieldTx] = txHash + ob.logger.Outbound.Info().Fields(lf).Msg("checking last TSS outbound") + + // step 2: is last tx stuck in mempool? + feeBumpWaitBlocks := GetFeeBumpWaitBlocks(ob.Chain().ChainId) + stuck, stuckFor, err := txChecker(ctx, ob.rpc, txHash, feeBumpWaitBlocks) + if err != nil { + return errors.Wrapf(err, "cannot determine if tx %s nonce %d is stuck", txHash, lastNonce) + } + + // step 3: update last outbound stuck tx information + // + // the key ideas to determine if Bitcoin outbound is stuck/unstuck: + // 1. outbound txs are a sequence of txs chained by nonce-mark UTXOs. + // 2. outbound tx with nonce N+1 MUST spend the nonce-mark UTXO produced by parent tx with nonce N. + // 3. when the last descendant tx is stuck, none of its ancestor txs can go through, so the stuck flag is set. + // 4. then RBF kicks in, it bumps the fee of the last descendant tx and aims to increase the average fee + // rate of the whole tx chain (as a package) to make it attractive to miners. + // 5. after RBF replacement, zetaclient clears the stuck flag immediately, hoping the new tx will be included + // within next 'PendingTxFeeBumpWaitBlocks' blocks. + // 6. the new tx may get stuck again (e.g. surging traffic) after 'PendingTxFeeBumpWaitBlocks' blocks, and + // the stuck flag will be set again to trigger another RBF, and so on. + // 7. all pending txs will be eventually cleared by fee bumping, and the stuck flag will be cleared. + // + // Note: reserved RBF bumping fee might be not enough to clear the stuck txs during extreme traffic surges, two options: + // 1. wait for the gas rate to drop. + // 2. manually clear the stuck txs by using offline accelerator services. + if stuck { + ob.SetLastStuckOutbound(NewLastStuckOutbound(lastNonce, lastTx, stuckFor)) + } else { + ob.SetLastStuckOutbound(nil) + } + + return nil +} + +// GetLastPendingOutbound gets the last pending outbound (with highest nonce) that sits in the Bitcoin mempool. +// Bitcoin outbound txs can be found from two sources: +// 1. txs that had been reported to tracker and then checked and included by this observer self. +// 2. txs that had been broadcasted by this observer self. +// +// Returns error if last pending outbound is not found +func GetLastPendingOutbound(ctx context.Context, ob *Observer) (*btcutil.Tx, uint64, error) { + var ( + lastNonce uint64 + lastHash string + ) + + // wait for pending nonce to refresh + pendingNonce := ob.GetPendingNonce() + if ob.GetPendingNonce() == 0 { + return nil, 0, errors.New("pending nonce is zero") + } + + // source 1: + // pick highest nonce tx from included txs + txResult := ob.GetIncludedTx(pendingNonce - 1) + if txResult != nil { + lastNonce = pendingNonce - 1 + lastHash = txResult.TxID + } + + // source 2: + // pick highest nonce tx from broadcasted txs + p, err := ob.ZetacoreClient().GetPendingNoncesByChain(ctx, ob.Chain().ChainId) + if err != nil { + return nil, 0, errors.Wrap(err, "GetPendingNoncesByChain failed") + } + // #nosec G115 always in range + for nonce := uint64(p.NonceLow); nonce < uint64(p.NonceHigh); nonce++ { + if nonce > lastNonce { + txID, found := ob.GetBroadcastedTx(nonce) + if found { + lastNonce = nonce + lastHash = txID + } + } + } + + // stop if last tx not found, and it is okay + // this individual zetaclient lost track of the last tx for some reason (offline, db reset, etc.) + if lastNonce == 0 { + return nil, 0, errors.New("last tx not found") + } + + // is tx in the mempool? + if _, err = ob.rpc.GetMempoolEntry(ctx, lastHash); err != nil { + return nil, 0, errors.New("last tx is not in mempool") + } + + // ensure this tx is the REAL last transaction + // cross-check the latest UTXO list, the nonce-mark utxo exists ONLY for last nonce + if ob.FetchUTXOs(ctx) != nil { + return nil, 0, errors.New("FetchUTXOs failed") + } + if _, err = ob.findNonceMarkUTXO(lastNonce, lastHash); err != nil { + return nil, 0, errors.Wrapf(err, "findNonceMarkUTXO failed for last tx %s nonce %d", lastHash, lastNonce) + } + + // query last transaction + // 'GetRawTransaction' is preferred over 'GetTransaction' here for three reasons: + // 1. it can fetch both stuck tx and non-stuck tx as far as they are valid txs. + // 2. it never fetch invalid tx (e.g., old tx replaced by RBF), so we can exclude invalid ones. + // 3. zetaclient needs the original tx body of a stuck tx to bump its fee and sign again. + lastTx, err := ob.rpc.GetRawTransactionByStr(ctx, lastHash) + if err != nil { + return nil, 0, errors.Wrapf(err, "GetRawTxByHash failed for last tx %s nonce %d", lastHash, lastNonce) + } + + return lastTx, lastNonce, nil +} + +// IsTxStuckInMempool checks if the transaction is stuck in the mempool. +// +// A pending tx with 'confirmations == 0' will be considered stuck due to excessive pending time. +func IsTxStuckInMempool( + ctx context.Context, + rpc RPC, + txHash string, + maxWaitBlocks int64, +) (bool, time.Duration, error) { + lastBlock, err := rpc.GetBlockCount(ctx) + if err != nil { + return false, 0, errors.Wrap(err, "GetBlockCount failed") + } + + memplEntry, err := rpc.GetMempoolEntry(ctx, txHash) + if err != nil { + if strings.Contains(err.Error(), "Transaction not in mempool") { + return false, 0, nil // not a mempool tx, of course not stuck + } + return false, 0, errors.Wrap(err, "GetMempoolEntry failed") + } + + // is the tx pending for too long? + pendingTime := time.Since(time.Unix(memplEntry.Time, 0)) + pendingTimeAllowed := blockTimeBTC * time.Duration(maxWaitBlocks) + pendingDeadline := memplEntry.Height + maxWaitBlocks + if pendingTime > pendingTimeAllowed && lastBlock > pendingDeadline { + return true, pendingTime, nil + } + + return false, pendingTime, nil +} + +// IsTxStuckInMempoolRegnet checks if the transaction is stuck in the mempool in regnet. +// Note: this function is a simplified version used in regnet for E2E test. +func IsTxStuckInMempoolRegnet( + ctx context.Context, + rpc RPC, + txHash string, + maxWaitBlocks int64, +) (bool, time.Duration, error) { + lastBlock, err := rpc.GetBlockCount(ctx) + if err != nil { + return false, 0, errors.Wrap(err, "GetBlockCount failed") + } + + memplEntry, err := rpc.GetMempoolEntry(ctx, txHash) + if err != nil { + if strings.Contains(err.Error(), "Transaction not in mempool") { + return false, 0, nil // not a mempool tx, of course not stuck + } + return false, 0, errors.Wrap(err, "GetMempoolEntry failed") + } + + // is the tx pending for too long? + pendingTime := time.Since(time.Unix(memplEntry.Time, 0)) + pendingTimeAllowed := time.Second * time.Duration(maxWaitBlocks) + + // the block mining is frozen in Regnet for E2E test + if pendingTime > pendingTimeAllowed && memplEntry.Height == lastBlock { + return true, pendingTime, nil + } + + return false, pendingTime, nil +} + +// GetStuckTxChecker returns the stuck tx checker function based on the chain ID. +func GetStuckTxChecker(chainID int64) StuckTxChecker { + if chains.IsBitcoinRegnet(chainID) { + return IsTxStuckInMempoolRegnet + } + return IsTxStuckInMempool +} + +// GetFeeBumpWaitBlocks returns the number of blocks to await before bumping tx fees +func GetFeeBumpWaitBlocks(chainID int64) int64 { + if chains.IsBitcoinRegnet(chainID) { + return PendingTxFeeBumpWaitBlocksRegnet + } + return PendingTxFeeBumpWaitBlocks +} diff --git a/zetaclient/chains/bitcoin/observer/mempool_test.go b/zetaclient/chains/bitcoin/observer/mempool_test.go new file mode 100644 index 0000000000..e2db6ac3a1 --- /dev/null +++ b/zetaclient/chains/bitcoin/observer/mempool_test.go @@ -0,0 +1,441 @@ +package observer_test + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/wire" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/pkg/chains" + crosschaintypes "github.com/zeta-chain/node/x/observer/types" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" + "github.com/zeta-chain/node/zetaclient/testutils" +) + +func Test_NewLastStuckOutbound(t *testing.T) { + nonce := uint64(1) + tx := btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)) + stuckFor := 30 * time.Minute + stuckOutbound := observer.NewLastStuckOutbound(nonce, tx, stuckFor) + + require.Equal(t, nonce, stuckOutbound.Nonce) + require.Equal(t, tx, stuckOutbound.Tx) + require.Equal(t, stuckFor, stuckOutbound.StuckFor) +} + +func Test_FefreshLastStuckOutbound(t *testing.T) { + sampleTx1 := btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)) + sampleTx2 := btcutil.NewTx(wire.NewMsgTx(2)) + + tests := []struct { + name string + txFinder observer.PendingTxFinder + txChecker observer.StuckTxChecker + oldStuckTx *observer.LastStuckOutbound + expectedTx *observer.LastStuckOutbound + errMsg string + }{ + { + name: "should set last stuck tx successfully", + txFinder: makePendingTxFinder(sampleTx1, 1, ""), + txChecker: makeStuckTxChecker(true, 30*time.Minute, ""), + oldStuckTx: nil, + expectedTx: observer.NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), + }, + { + name: "should update last stuck tx successfully", + txFinder: makePendingTxFinder(sampleTx2, 2, ""), + txChecker: makeStuckTxChecker(true, 40*time.Minute, ""), + oldStuckTx: observer.NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), + expectedTx: observer.NewLastStuckOutbound(2, sampleTx2, 40*time.Minute), + }, + { + name: "should clear last stuck tx successfully", + txFinder: makePendingTxFinder(sampleTx1, 1, ""), + txChecker: makeStuckTxChecker(false, 1*time.Minute, ""), + oldStuckTx: observer.NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), + expectedTx: nil, + }, + { + name: "do nothing if unable to find last pending tx", + txFinder: makePendingTxFinder(nil, 0, "txFinder failed"), + expectedTx: nil, + }, + { + name: "should return error if txChecker failed", + txFinder: makePendingTxFinder(sampleTx1, 1, ""), + txChecker: makeStuckTxChecker(false, 0, "txChecker failed"), + expectedTx: nil, + errMsg: "cannot determine", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // create observer + ob := newTestSuite(t, chains.BitcoinMainnet, "") + + // setup old stuck tx + if tt.oldStuckTx != nil { + ob.SetLastStuckOutbound(tt.oldStuckTx) + } + + // refresh + ctx := context.Background() + err := ob.RefreshLastStuckOutbound(ctx, tt.txFinder, tt.txChecker) + + if tt.errMsg == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.errMsg) + } + + // check + stuckTx := ob.GetLastStuckOutbound() + require.Equal(t, tt.expectedTx, stuckTx) + }) + } +} + +func Test_GetLastPendingOutbound(t *testing.T) { + sampleTx := btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)) + + tests := []struct { + name string + chain chains.Chain + pendingNonce uint64 + pendingNonces *crosschaintypes.PendingNonces + utxos []btcjson.ListUnspentResult + tx *btcutil.Tx + saveTx bool + includeTx bool + failMempool bool + failGetTx bool + expectedTx *btcutil.Tx + expectedNonce uint64 + errMsg string + }{ + { + name: "should return last included outbound", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + pendingNonces: &crosschaintypes.PendingNonces{ + NonceLow: 9, + NonceHigh: 10, + }, + utxos: []btcjson.ListUnspentResult{ + { + TxID: sampleTx.MsgTx().TxID(), + Vout: 0, + Address: testutils.TSSAddressBTCMainnet, + Amount: float64(chains.NonceMarkAmount(9)) / btcutil.SatoshiPerBitcoin, + }, + }, + tx: sampleTx, + saveTx: false, + includeTx: true, + expectedTx: sampleTx, + expectedNonce: 9, + }, + { + name: "should return last broadcasted outbound", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + pendingNonces: &crosschaintypes.PendingNonces{ + NonceLow: 9, + NonceHigh: 10, + }, + utxos: []btcjson.ListUnspentResult{ + { + TxID: sampleTx.MsgTx().TxID(), + Vout: 0, + Address: testutils.TSSAddressBTCMainnet, + Amount: float64(chains.NonceMarkAmount(9)) / btcutil.SatoshiPerBitcoin, + }, + }, + tx: sampleTx, + saveTx: true, + includeTx: false, + expectedTx: sampleTx, + expectedNonce: 9, + }, + { + name: "return error if pending nonce is zero", + chain: chains.BitcoinMainnet, + pendingNonce: 0, + expectedTx: nil, + expectedNonce: 0, + errMsg: "pending nonce is zero", + }, + { + name: "return error if GetPendingNoncesByChain failed", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + saveTx: true, + includeTx: false, + expectedTx: nil, + expectedNonce: 0, + errMsg: "GetPendingNoncesByChain failed", + }, + { + name: "return error if no last tx found", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + pendingNonces: &crosschaintypes.PendingNonces{ + NonceLow: 9, + NonceHigh: 10, + }, + saveTx: false, + includeTx: false, + expectedTx: nil, + expectedNonce: 0, + errMsg: "last tx not found", + }, + { + name: "return error if GetMempoolEntry failed", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + pendingNonces: &crosschaintypes.PendingNonces{ + NonceLow: 9, + NonceHigh: 10, + }, + tx: sampleTx, + saveTx: true, + includeTx: false, + failMempool: true, + expectedTx: nil, + expectedNonce: 0, + errMsg: "last tx is not in mempool", + }, + { + name: "return error if FetchUTXOs failed", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + pendingNonces: &crosschaintypes.PendingNonces{ + NonceLow: 9, + NonceHigh: 10, + }, + tx: sampleTx, + saveTx: true, + includeTx: false, + expectedTx: nil, + expectedNonce: 0, + errMsg: "FetchUTXOs failed", + }, + { + name: "return error if unable to find nonce-mark UTXO", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + pendingNonces: &crosschaintypes.PendingNonces{ + NonceLow: 9, + NonceHigh: 10, + }, + utxos: []btcjson.ListUnspentResult{ + { + TxID: sampleTx.MsgTx().TxID(), + Vout: 1, // wrong output index + Address: testutils.TSSAddressBTCMainnet, + Amount: float64(chains.NonceMarkAmount(9)) / btcutil.SatoshiPerBitcoin, + }, + }, + tx: sampleTx, + saveTx: true, + includeTx: false, + expectedTx: nil, + expectedNonce: 0, + errMsg: "findNonceMarkUTXO failed", + }, + { + name: "return error if GetRawTxByHash failed", + chain: chains.BitcoinMainnet, + pendingNonce: 10, + pendingNonces: &crosschaintypes.PendingNonces{ + NonceLow: 9, + NonceHigh: 10, + }, + utxos: []btcjson.ListUnspentResult{ + { + TxID: sampleTx.MsgTx().TxID(), + Vout: 0, + Address: testutils.TSSAddressBTCMainnet, + Amount: float64(chains.NonceMarkAmount(9)) / btcutil.SatoshiPerBitcoin, + }, + }, + tx: sampleTx, + saveTx: false, + includeTx: true, + failGetTx: true, + expectedTx: nil, + expectedNonce: 0, + errMsg: "GetRawTxByHash failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // create observer + ob := newTestSuite(t, chains.BitcoinMainnet, "") + + // set pending nonce + ob.SetPendingNonce(tt.pendingNonce) + + if tt.tx != nil { + // save tx to simulate broadcasted tx + txNonce := tt.pendingNonce - 1 + if tt.saveTx { + ob.SaveBroadcastedTx(tt.tx.MsgTx().TxID(), txNonce) + } + + // include tx to simulate included tx + if tt.includeTx { + ob.SetIncludedTx(txNonce, &btcjson.GetTransactionResult{ + TxID: tt.tx.MsgTx().TxID(), + }) + } + } + + // mock zetacore client response + if tt.pendingNonces != nil { + ob.zetacore.On("GetPendingNoncesByChain", mock.Anything, mock.Anything). + Maybe(). + Return(*tt.pendingNonces, nil) + } else { + res := crosschaintypes.PendingNonces{} + ob.zetacore.On("GetPendingNoncesByChain", mock.Anything, mock.Anything).Maybe().Return(res, errors.New("failed")) + } + + // mock btc client response + if tt.utxos != nil { + ob.client.On("ListUnspentMinMaxAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Maybe(). + Return(tt.utxos, nil) + } else { + ob.client.On("ListUnspentMinMaxAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil, errors.New("failed")) + } + if !tt.failMempool { + ob.client.On("GetMempoolEntry", mock.Anything, mock.Anything).Maybe().Return(nil, nil) + } else { + ob.client.On("GetMempoolEntry", mock.Anything, mock.Anything).Maybe().Return(nil, errors.New("failed")) + } + if tt.tx != nil && !tt.failGetTx { + ob.client.On("GetRawTransactionByStr", mock.Anything, mock.Anything).Maybe().Return(tt.tx, nil) + } else { + ob.client.On("GetRawTransactionByStr", mock.Anything, mock.Anything).Maybe().Return(nil, errors.New("failed")) + } + + ctx := context.Background() + lastTx, lastNonce, err := observer.GetLastPendingOutbound(ctx, ob.Observer) + + if tt.errMsg != "" { + require.ErrorContains(t, err, tt.errMsg) + require.Nil(t, lastTx) + require.Zero(t, lastNonce) + return + } + + require.NoError(t, err) + require.Equal(t, tt.expectedTx, lastTx) + require.Equal(t, tt.expectedNonce, lastNonce) + }) + } +} + +func Test_GetStuckTxCheck(t *testing.T) { + tests := []struct { + name string + chainID int64 + txChecker observer.StuckTxChecker + }{ + { + name: "should return 3 blocks for Bitcoin mainnet", + chainID: chains.BitcoinMainnet.ChainId, + txChecker: observer.IsTxStuckInMempool, + }, + { + name: "should return 3 blocks for Bitcoin testnet4", + chainID: chains.BitcoinTestnet.ChainId, + txChecker: observer.IsTxStuckInMempool, + }, + { + name: "should return 3 blocks for Bitcoin Signet", + chainID: chains.BitcoinSignetTestnet.ChainId, + txChecker: observer.IsTxStuckInMempool, + }, + { + name: "should return 10 blocks for Bitcoin regtest", + chainID: chains.BitcoinRegtest.ChainId, + txChecker: observer.IsTxStuckInMempoolRegnet, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + txChecker := observer.GetStuckTxChecker(tt.chainID) + require.Equal(t, reflect.ValueOf(tt.txChecker).Pointer(), reflect.ValueOf(txChecker).Pointer()) + }) + } +} + +func Test_GetFeeBumpWaitBlocks(t *testing.T) { + tests := []struct { + name string + chainID int64 + expectedWaitBlocks int64 + }{ + { + name: "should return wait blocks for Bitcoin mainnet", + chainID: chains.BitcoinMainnet.ChainId, + expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocks, + }, + { + name: "should return wait blocks for Bitcoin testnet4", + chainID: chains.BitcoinTestnet.ChainId, + expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocks, + }, + { + name: "should return wait blocks for Bitcoin signet", + chainID: chains.BitcoinSignetTestnet.ChainId, + expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocks, + }, + { + name: "should return wait blocks for Bitcoin regtest", + chainID: chains.BitcoinRegtest.ChainId, + expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocksRegnet, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + blocks := observer.GetFeeBumpWaitBlocks(tt.chainID) + require.Equal(t, tt.expectedWaitBlocks, blocks) + }) + } +} + +// makePendingTxFinder is a helper function to create a mock pending tx finder +func makePendingTxFinder(tx *btcutil.Tx, nonce uint64, errMsg string) observer.PendingTxFinder { + var err error + if errMsg != "" { + err = errors.New(errMsg) + } + return func(_ context.Context, _ *observer.Observer) (*btcutil.Tx, uint64, error) { + return tx, nonce, err + } +} + +// makeStuckTxChecker is a helper function to create a mock stuck tx checker +func makeStuckTxChecker(stuck bool, stuckFor time.Duration, errMsg string) observer.StuckTxChecker { + var err error + if errMsg != "" { + err = errors.New(errMsg) + } + return func(_ context.Context, _ observer.RPC, _ string, _ int64) (bool, time.Duration, error) { + return stuck, stuckFor, err + } +} diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 72e78a0c6b..35ac81b7e2 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -20,6 +20,7 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/base" "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/db" + "github.com/zeta-chain/node/zetaclient/logs" "github.com/zeta-chain/node/zetaclient/metrics" ) @@ -38,6 +39,7 @@ type RPC interface { hash *hash.Hash, res *btcjson.GetTransactionResult, ) (btcjson.TxRawResult, error) + GetMempoolEntry(ctx context.Context, txHash string) (*btcjson.GetMempoolEntryResult, error) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) GetTransactionFeeAndRate(ctx context.Context, tx *btcjson.TxRawResult) (int64, int64, error) @@ -56,6 +58,7 @@ type RPC interface { GetBlockHeightByStr(ctx context.Context, blockHash string) (int64, error) GetTransactionByStr(ctx context.Context, hash string) (*hash.Hash, *btcjson.GetTransactionResult, error) + GetRawTransactionByStr(ctx context.Context, hash string) (*btcutil.Tx, error) } const ( @@ -101,10 +104,13 @@ type Observer struct { // pendingNonce is the outbound artificial pending nonce pendingNonce uint64 + // lastStuckTx contains the last stuck outbound tx information + // Note: nil if outbound is not stuck + lastStuckTx *LastStuckOutbound + // utxos contains the UTXOs owned by the TSS address utxos []btcjson.ListUnspentResult - // tssOutboundHashes indexes included tx with tx hash // tssOutboundHashes keeps track of outbound hashes sent from TSS address tssOutboundHashes map[string]bool @@ -248,9 +254,41 @@ func (ob *Observer) GetBlockByNumberCached(ctx context.Context, blockNumber int6 return blockNheader, nil } +// GetLastStuckOutbound returns the last stuck outbound tx information +func (ob *Observer) GetLastStuckOutbound() *LastStuckOutbound { + ob.Mu().Lock() + defer ob.Mu().Unlock() + return ob.lastStuckTx +} + +// SetLastStuckOutbound sets the information of last stuck outbound +func (ob *Observer) SetLastStuckOutbound(stuckTx *LastStuckOutbound) { + ob.Mu().Lock() + defer ob.Mu().Unlock() + + lf := map[string]any{ + logs.FieldMethod: "SetLastStuckOutbound", + } + + if stuckTx != nil { + lf[logs.FieldNonce] = stuckTx.Nonce + lf[logs.FieldTx] = stuckTx.Tx.MsgTx().TxID() + ob.logger.Outbound.Warn(). + Fields(lf). + Msgf("Bitcoin outbound is stuck for %f minutes", stuckTx.StuckFor.Minutes()) + } else if ob.lastStuckTx != nil { + lf[logs.FieldNonce] = ob.lastStuckTx.Nonce + lf[logs.FieldTx] = ob.lastStuckTx.Tx.MsgTx().TxID() + ob.logger.Outbound.Info().Fields(lf).Msgf("Bitcoin outbound is no longer stuck") + } + ob.lastStuckTx = stuckTx +} + // IsTSSTransaction checks if a given transaction was sent by TSS itself. -// An unconfirmed transaction is safe to spend only if it was sent by TSS and verified by ourselves. +// An unconfirmed transaction is safe to spend only if it was sent by TSS self. func (ob *Observer) IsTSSTransaction(txid string) bool { + ob.Mu().Lock() + defer ob.Mu().Unlock() _, found := ob.tssOutboundHashes[txid] return found } diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index c5d5a5be2f..4f78abfbc1 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -6,8 +6,10 @@ import ( "os" "strconv" "testing" + "time" "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/wire" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" @@ -239,6 +241,39 @@ func TestConfirmationThreshold(t *testing.T) { }) } +func Test_SetLastStuckOutbound(t *testing.T) { + // create observer and example stuck tx + ob := newTestSuite(t, chains.BitcoinMainnet, "") + tx := btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)) + + // STEP 1 + // initial stuck outbound is nil + require.Nil(t, ob.GetLastStuckOutbound()) + + // STEP 2 + // set stuck outbound + stuckTx := observer.NewLastStuckOutbound(100, tx, 30*time.Minute) + ob.SetLastStuckOutbound(stuckTx) + + // retrieve stuck outbound + require.Equal(t, stuckTx, ob.GetLastStuckOutbound()) + + // STEP 3 + // update stuck outbound + stuckTxUpdate := observer.NewLastStuckOutbound(101, tx, 40*time.Minute) + ob.SetLastStuckOutbound(stuckTxUpdate) + + // retrieve updated stuck outbound + require.Equal(t, stuckTxUpdate, ob.GetLastStuckOutbound()) + + // STEP 4 + // clear stuck outbound + ob.SetLastStuckOutbound(nil) + + // stuck outbound should be nil + require.Nil(t, ob.GetLastStuckOutbound()) +} + func TestSubmittedTx(t *testing.T) { // setup db db, submittedTx := setupDBTxResults(t) diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index 7192b754b0..d4e01125c6 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -353,11 +353,16 @@ func (ob *Observer) SetIncludedTx(nonce uint64, getTxResult *btcjson.GetTransact } } else { // for other hash: - // be alert for duplicate payment!!! As we got a new hash paying same cctx (for whatever reason). - // we can't tell which txHash is true, so we remove all to be safe + // got multiple hashes for same nonce. RBF happened. + ob.logger.Outbound.Info().Fields(lf).Msgf("replaced bitcoin outbound %s", res.TxID) + + // remove prior txHash and txResult delete(ob.tssOutboundHashes, res.TxID) delete(ob.includedTxResults, outboundID) - ob.logger.Outbound.Error().Msgf("setIncludedTx: duplicate payment by bitcoin outbound %s outboundID %s, prior outbound %s", txHash, outboundID, res.TxID) + + // add new txHash and txResult + ob.tssOutboundHashes[txHash] = true + ob.includedTxResults[outboundID] = getTxResult } } diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go new file mode 100644 index 0000000000..66e76b3e85 --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -0,0 +1,208 @@ +package signer + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/mempool" + "github.com/btcsuite/btcd/wire" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/constant" + mathpkg "github.com/zeta-chain/node/pkg/math" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" +) + +const ( + // feeRateCap is the maximum average fee rate for CPFP fee bumping + // 100 sat/vB is a heuristic based on Bitcoin mempool statistics to avoid excessive fees + // see: https://mempool.space/graphs/mempool#3y + feeRateCap = 100 + + // minCPFPFeeBumpPercent is the minimum percentage by which the CPFP average fee rate should be bumped. + // This value 20% is a heuristic, not mandated by the Bitcoin protocol, designed to balance effectiveness + // in replacing stuck transactions while avoiding excessive sensitivity to fee market fluctuations. + minCPFPFeeBumpPercent = 20 +) + +// CPFPFeeBumper is a helper struct to contain CPFP (child-pays-for-parent) fee bumping logic +type CPFPFeeBumper struct { + Ctx context.Context + + Chain chains.Chain + + // RPC is the interface to interact with the Bitcoin chain + RPC RPC + + // Tx is the stuck transaction to bump + Tx *btcutil.Tx + + // MinRelayFee is the minimum relay fee in BTC + MinRelayFee float64 + + // CCTXRate is the most recent fee rate of the CCTX + CCTXRate int64 + + // LiveRate is the most recent market fee rate + LiveRate int64 + + // TotalTxs is the total number of stuck TSS txs + TotalTxs int64 + + // TotalFees is the total fees of all stuck TSS txs + TotalFees int64 + + // TotalVSize is the total vsize of all stuck TSS txs + TotalVSize int64 + + // AvgFeeRate is the average fee rate of all stuck TSS txs + AvgFeeRate int64 +} + +// NewCPFPFeeBumper creates a new CPFPFeeBumper +func NewCPFPFeeBumper( + ctx context.Context, + rpc RPC, + chain chains.Chain, + tx *btcutil.Tx, + cctxRate int64, + minRelayFee float64, + logger zerolog.Logger, +) (*CPFPFeeBumper, error) { + fb := &CPFPFeeBumper{ + Ctx: ctx, + Chain: chain, + RPC: rpc, + Tx: tx, + MinRelayFee: minRelayFee, + CCTXRate: cctxRate, + } + + err := fb.FetchFeeBumpInfo(logger) + if err != nil { + return nil, err + } + return fb, nil +} + +// BumpTxFee bumps the fee of the stuck transaction using reserved bump fees +func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, int64, error) { + // reuse old tx body + newTx := CopyMsgTxNoWitness(b.Tx.MsgTx()) + if len(newTx.TxOut) < 3 { + return nil, 0, 0, errors.New("original tx has no reserved bump fees") + } + + // tx replacement is triggered only when market fee rate goes 20% higher than current paid rate. + // zetacore updates the cctx fee rate evey 10 minutes, we could hold on and retry later. + minBumpRate := mathpkg.IncreaseIntByPercent(b.AvgFeeRate, minCPFPFeeBumpPercent) + if b.CCTXRate < minBumpRate { + return nil, 0, 0, fmt.Errorf( + "hold on RBF: cctx rate %d is lower than the min bumped rate %d", + b.CCTXRate, + minBumpRate, + ) + } + + // the live rate may continue increasing during network congestion, we should wait until it stabilizes a bit. + // this is to ensure the live rate is not 20%+ higher than the cctx rate, otherwise, the replacement tx may + // also get stuck and need another replacement. + bumpedRate := mathpkg.IncreaseIntByPercent(b.CCTXRate, minCPFPFeeBumpPercent) + if b.LiveRate > bumpedRate { + return nil, 0, 0, fmt.Errorf( + "hold on RBF: live rate %d is much higher than the cctx rate %d", + b.LiveRate, + b.CCTXRate, + ) + } + + // cap the fee rate to avoid excessive fees + feeRateNew := b.CCTXRate + if b.CCTXRate > feeRateCap { + feeRateNew = feeRateCap + } + + // calculate minmimum relay fees of the new replacement tx + // the new tx will have almost same size as the old one because the tx body stays the same + txVSize := mempool.GetTxVirtualSize(b.Tx) + minRelayFeeRate := common.FeeRateToSatPerByte(b.MinRelayFee) + minRelayTxFees := txVSize * minRelayFeeRate + + // calculate the RBF additional fees required by Bitcoin protocol + // two conditions to satisfy: + // 1. new txFees >= old txFees (already handled above) + // 2. additionalFees >= minRelayTxFees + // + // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/rbf.cpp#L166-L183 + additionalFees := b.TotalVSize*feeRateNew - b.TotalFees + if additionalFees < minRelayTxFees { + return nil, 0, 0, fmt.Errorf( + "hold on RBF: additional fees %d is lower than min relay fees %d", + additionalFees, + minRelayTxFees, + ) + } + + // bump fees in two ways: + // 1. deduct additional fees from the change amount + // 2. give up the whole change amount if it's not enough + if newTx.TxOut[2].Value >= additionalFees+constant.BTCWithdrawalDustAmount { + newTx.TxOut[2].Value -= additionalFees + } else { + additionalFees = newTx.TxOut[2].Value + newTx.TxOut = newTx.TxOut[:2] + } + + // effective fee rate + feeRateNew = int64(math.Ceil(float64(b.TotalFees+additionalFees) / float64(b.TotalVSize))) + + return newTx, additionalFees, feeRateNew, nil +} + +// fetchFeeBumpInfo fetches all necessary information needed to bump the stuck tx +func (b *CPFPFeeBumper) FetchFeeBumpInfo(logger zerolog.Logger) error { + // query live fee rate + isRegnet := chains.IsBitcoinRegnet(b.Chain.ChainId) + liveRate, err := b.RPC.GetEstimatedFeeRate(b.Ctx, 1, isRegnet) + if err != nil { + return errors.Wrap(err, "GetEstimatedFeeRate failed") + } + b.LiveRate = liveRate + + // query total fees and sizes of all pending parent TSS txs + totalTxs, totalFees, totalVSize, avgFeeRate, err := b.RPC.GetTotalMempoolParentsSizeNFees( + b.Ctx, + b.Tx.MsgTx().TxID(), + time.Minute, + ) + if err != nil { + return errors.Wrap(err, "unable to fetch mempool txs info") + } + totalFeesSats, err := common.GetSatoshis(totalFees) + if err != nil { + return errors.Wrapf(err, "cannot convert total fees %f", totalFees) + } + + b.TotalTxs = totalTxs + b.TotalFees = totalFeesSats + b.TotalVSize = totalVSize + b.AvgFeeRate = avgFeeRate + logger.Info(). + Msgf("totalTxs %d, totalFees %f, totalVSize %d, avgFeeRate %d", totalTxs, totalFees, totalVSize, avgFeeRate) + + return nil +} + +// CopyMsgTxNoWitness creates a deep copy of the given MsgTx and clears the witness data +func CopyMsgTxNoWitness(tx *wire.MsgTx) *wire.MsgTx { + copyTx := tx.Copy() + for idx := range copyTx.TxIn { + copyTx.TxIn[idx].Witness = wire.TxWitness{} + } + return copyTx +} diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go new file mode 100644 index 0000000000..c45b60a78b --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -0,0 +1,383 @@ +package signer_test + +import ( + "context" + "testing" + + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/wire" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/constant" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" + "github.com/zeta-chain/node/zetaclient/testutils" + "github.com/zeta-chain/node/zetaclient/testutils/mocks" +) + +// mempoolTxsInfo is a helper struct to contain mempool txs information +type mempoolTxsInfo struct { + totalTxs int64 + totalFees float64 + totalVSize int64 + avgFeeRate int64 +} + +func newMempoolTxsInfo(totalTxs int64, totalFees float64, totalVSize int64, avgFeeRate int64) *mempoolTxsInfo { + return &mempoolTxsInfo{ + totalTxs: totalTxs, + totalFees: totalFees, + totalVSize: totalVSize, + avgFeeRate: avgFeeRate, + } +} + +func Test_NewCPFPFeeBumper(t *testing.T) { + tests := []struct { + name string + chain chains.Chain + client *mocks.BitcoinClient + tx *btcutil.Tx + cctxRate int64 + liveRate int64 + minRelayFee float64 + memplTxsInfo *mempoolTxsInfo + errMsg string + expected *signer.CPFPFeeBumper + }{ + { + chain: chains.BitcoinMainnet, + name: "should create new CPFPFeeBumper successfully", + client: mocks.NewBitcoinClient(t), + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + cctxRate: 10, + liveRate: 12, + minRelayFee: 0.00001, + memplTxsInfo: newMempoolTxsInfo( + 2, // 2 stuck TSS txs + 0.0001, // total fees 0.0001 BTC + 1000, // total vsize 1000 + 10, // average fee rate 10 sat/vB + ), + expected: &signer.CPFPFeeBumper{ + Ctx: context.Background(), + Chain: chains.BitcoinMainnet, + Tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + MinRelayFee: 0.00001, + CCTXRate: 10, + LiveRate: 12, + TotalTxs: 2, + TotalFees: 10000, + TotalVSize: 1000, + AvgFeeRate: 10, + }, + }, + { + chain: chains.BitcoinMainnet, + name: "should fail when mempool txs info fetcher returns error", + client: mocks.NewBitcoinClient(t), + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + liveRate: 12, + memplTxsInfo: nil, + errMsg: "unable to fetch mempool txs info", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // ARRANGE + // mock RPC fee rate + tt.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(tt.liveRate, nil) + + // mock mempool txs information + if tt.memplTxsInfo != nil { + tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything). + Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) + } else { + v := int64(0) + tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Return(v, 0.0, v, v, errors.New("rpc error")) + } + + // ACT + bumper, err := signer.NewCPFPFeeBumper( + context.Background(), + tt.client, + tt.chain, + tt.tx, + tt.cctxRate, + tt.minRelayFee, + log.Logger, + ) + + // ASSERT + if tt.errMsg != "" { + require.Nil(t, bumper) + require.ErrorContains(t, err, tt.errMsg) + } else { + bumper.RPC = nil // ignore the RPC + require.NoError(t, err) + require.Equal(t, tt.expected, bumper) + } + }) + } +} + +func Test_BumpTxFee(t *testing.T) { + // https://mempool.space/tx/030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0 + chain := chains.BitcoinMainnet + txid := "030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0" + msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chain.ChainId, txid) + + tests := []struct { + name string + feeBumper *signer.CPFPFeeBumper + additionalFees int64 + expectedNewRate int64 + expectedNewTx *wire.MsgTx + errMsg string + }{ + { + name: "should bump tx fee successfully", + feeBumper: &signer.CPFPFeeBumper{ + Tx: btcutil.NewTx(msgTx), + MinRelayFee: 0.00001, + CCTXRate: 57, + LiveRate: 60, + TotalFees: 27213, + TotalVSize: 579, + AvgFeeRate: 47, + }, + additionalFees: 5790, + expectedNewRate: 57, + expectedNewTx: func() *wire.MsgTx { + // deduct additional fees + newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx.TxOut[2].Value -= 5790 + return newTx + }(), + }, + { + name: "should give up all reserved bump fees", + feeBumper: &signer.CPFPFeeBumper{ + Tx: func() *btcutil.Tx { + // modify reserved bump fees to barely cover bump fees + newTx := msgTx.Copy() + newTx.TxOut[2].Value = 5790 + constant.BTCWithdrawalDustAmount - 1 + return btcutil.NewTx(newTx) + }(), + MinRelayFee: 0.00001, + CCTXRate: 57, + LiveRate: 60, + TotalFees: 27213, + TotalVSize: 579, + AvgFeeRate: 47, + }, + additionalFees: 5790 + constant.BTCWithdrawalDustAmount - 1, // 6789 + expectedNewRate: 59, // (27213 + 6789) / 579 ≈ 59 + expectedNewTx: func() *wire.MsgTx { + // give up all reserved bump fees + newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx.TxOut = newTx.TxOut[:2] + return newTx + }(), + }, + { + name: "should cap new gas rate to 'gasRateCap'", + feeBumper: &signer.CPFPFeeBumper{ + Tx: btcutil.NewTx(msgTx), + MinRelayFee: 0.00001, + CCTXRate: 101, // > 100 + LiveRate: 120, + TotalFees: 27213, + TotalVSize: 579, + AvgFeeRate: 47, + }, + additionalFees: 30687, // (100-47)*579 + expectedNewRate: 100, + expectedNewTx: func() *wire.MsgTx { + // deduct additional fees + newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx.TxOut[2].Value -= 30687 + return newTx + }(), + }, + { + name: "should fail if original tx has no reserved bump fees", + feeBumper: &signer.CPFPFeeBumper{ + Tx: func() *btcutil.Tx { + // remove the change output + newTx := msgTx.Copy() + newTx.TxOut = newTx.TxOut[:2] + return btcutil.NewTx(newTx) + }(), + }, + errMsg: "original tx has no reserved bump fees", + }, + { + name: "should hold on RBF if CCTX rate is lower than minimum bumpeed rate", + feeBumper: &signer.CPFPFeeBumper{ + Tx: btcutil.NewTx(msgTx), + CCTXRate: 55, // 56 < 47 * 120% + AvgFeeRate: 47, + }, + errMsg: "lower than the min bumped rate", + }, + { + name: "should hold on RBF if live rate is much higher than CCTX rate", + feeBumper: &signer.CPFPFeeBumper{ + Tx: btcutil.NewTx(msgTx), + CCTXRate: 57, + LiveRate: 70, // 70 > 57 * 120% + AvgFeeRate: 47, + }, + errMsg: "much higher than the cctx rate", + }, + { + name: "should hold on RBF if additional fees is lower than min relay fees", + feeBumper: &signer.CPFPFeeBumper{ + Tx: btcutil.NewTx(msgTx), + MinRelayFee: 0.00002, // min relay fee will be 579vB * 2 = 1158 sats + CCTXRate: 6, + LiveRate: 7, + TotalFees: 2895, + TotalVSize: 579, + AvgFeeRate: 5, + }, + errMsg: "lower than min relay fees", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newTx, additionalFees, newRate, err := tt.feeBumper.BumpTxFee() + if tt.errMsg != "" { + require.Nil(t, newTx) + require.Zero(t, additionalFees) + require.ErrorContains(t, err, tt.errMsg) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedNewTx, newTx) + require.Equal(t, tt.additionalFees, additionalFees) + require.Equal(t, tt.expectedNewRate, newRate) + } + }) + } +} + +func Test_FetchFeeBumpInfo(t *testing.T) { + liveRate := int64(12) + + tests := []struct { + name string + tx *btcutil.Tx + liveRate int64 + memplTxsInfo *mempoolTxsInfo + expected *signer.CPFPFeeBumper + errMsg string + }{ + { + name: "should fetch fee bump info successfully", + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + liveRate: 12, + memplTxsInfo: newMempoolTxsInfo( + 2, // 2 stuck TSS txs + 0.0001, // total fees 0.0001 BTC + 1000, // total vsize 1000 + 10, // average fee rate 10 sat/vB + ), + expected: &signer.CPFPFeeBumper{ + Tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + LiveRate: 12, + TotalTxs: 2, + TotalFees: 10000, + TotalVSize: 1000, + AvgFeeRate: 10, + }, + }, + { + name: "should fail if unable to estimate smart fee", + liveRate: 0, + errMsg: "GetEstimatedFeeRate failed", + }, + { + name: "should fail if unable to fetch mempool txs info", + liveRate: 12, + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + memplTxsInfo: nil, + errMsg: "unable to fetch mempool txs info", + }, + { + name: "should fail on invalid total fees", + liveRate: 12, + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + memplTxsInfo: newMempoolTxsInfo(2, 21000000.1, 1000, 10), // fee exceeds max BTC supply + errMsg: "cannot convert total fees", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // ARRANGE + // mock RPC fee rate + client := mocks.NewBitcoinClient(t) + if tt.liveRate > 0 { + client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(liveRate, nil) + } else { + client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), errors.New("rpc error")) + } + + // mock mempool txs information + if tt.memplTxsInfo != nil { + client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything). + Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) + } else { + v := int64(0) + client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(v, 0.0, v, v, errors.New("rpc error")) + } + + // ACT + bumper := &signer.CPFPFeeBumper{ + RPC: client, + Tx: tt.tx, + } + err := bumper.FetchFeeBumpInfo(log.Logger) + + // ASSERT + if tt.errMsg != "" { + require.ErrorContains(t, err, tt.errMsg) + } else { + bumper.RPC = nil // ignore the RPC client + require.NoError(t, err) + require.Equal(t, tt.expected, bumper) + } + }) + } +} + +func Test_CopyMsgTxNoWitness(t *testing.T) { + t.Run("should copy tx msg without witness", func(t *testing.T) { + chain := chains.BitcoinMainnet + txid := "030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0" + msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chain.ChainId, txid) + + // make a non-witness copy + copyTx := signer.CopyMsgTxNoWitness(msgTx) + + // make another copy and clear witness data manually + newTx := msgTx.Copy() + for idx := range newTx.TxIn { + newTx.TxIn[idx].Witness = wire.TxWitness{} + } + + // check + require.Equal(t, newTx, copyTx) + }) + + t.Run("should handle nil input", func(t *testing.T) { + require.Panics(t, func() { + signer.CopyMsgTxNoWitness(nil) + }, "should panic on nil input") + }) +} diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index d20ca9fb0e..176a74fb37 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -25,6 +25,17 @@ const ( // the rank below (or equal to) which we consolidate UTXOs consolidationRank = 10 + + // reservedRBFFees is the amount of BTC reserved for RBF fee bumping. + // the TSS keysign stops automatically when transactions get stuck in the mempool + // 0.01 BTC can bump 10 transactions (1KB each) by 100 sat/vB + reservedRBFFees = 0.01 + + // rbfTxInSequenceNum is the sequence number used to signal an opt-in full-RBF (Replace-By-Fee) transaction + // Setting sequenceNum to "1" effectively makes the transaction timelocks irrelevant. + // See: https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki + // See: https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/05_2_Resending_a_Transaction_with_RBF.md + rbfTxInSequenceNum uint32 = 1 ) // SignWithdrawTx signs a BTC withdrawal tx and returns the signed tx @@ -35,11 +46,18 @@ func (signer *Signer) SignWithdrawTx( ) (*wire.MsgTx, error) { nonceMark := chains.NonceMarkAmount(txData.nonce) estimateFee := float64(txData.feeRate*common.OutboundBytesMax) / 1e8 - totalAmount := txData.amount + estimateFee + float64(nonceMark)*1e-8 + totalAmount := txData.amount + estimateFee + reservedRBFFees + float64(nonceMark)*1e-8 - // refresh unspent UTXOs and continue with keysign regardless of error - if err := ob.FetchUTXOs(ctx); err != nil { - signer.Logger().Std.Error().Err(err).Uint64("nonce", txData.nonce).Msg("SignWithdrawTx: FetchUTXOs failed") + // refreshing UTXO list before TSS keysign is important: + // 1. all TSS outbounds have opted-in for RBF to be replaceable + // 2. using old UTXOs may lead to accidental double-spending, which may trigger unwanted RBF + // + // Note: unwanted RBF is very unlikely to happen for two reasons: + // 1. it requires 2/3 TSS signers to accidentally sign the same tx using same outdated UTXOs. + // 2. RBF requires a higher fee rate than the original tx, otherwise it will fail. + err := ob.FetchUTXOs(ctx) + if err != nil { + return nil, errors.Wrap(err, "FetchUTXOs failed") } // select N UTXOs to cover the total expense @@ -92,15 +110,11 @@ func (signer *Signer) SignWithdrawTx( txData.nonce, txData.feeRate, txSize, fees, selected.ConsolidatedUTXOs, selected.ConsolidatedValue) // add tx outputs - err = signer.AddWithdrawTxOutputs( - tx, - txData.to, - selected.Value, - txData.amountSats, - nonceMark, - fees, - txData.cancelTx, - ) + inputValue := selected.Value + err = signer.AddWithdrawTxOutputs(tx, txData.to, inputValue, txData.amountSats, nonceMark, fees, txData.cancelTx) + if err != nil { + return nil, err + } if err != nil { return nil, err } @@ -123,8 +137,13 @@ func (signer *Signer) AddTxInputs(tx *wire.MsgTx, utxos []btcjson.ListUnspentRes return nil, err } + // add input and set 'nSequence' to opt-in for RBF + // it doesn't matter on which input we set the RBF sequence outpoint := wire.NewOutPoint(hash, utxo.Vout) txIn := wire.NewTxIn(outpoint, nil, nil) + if i == 0 { + txIn.Sequence = rbfTxInSequenceNum + } tx.AddTxIn(txIn) // store the amount for later signing use diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go new file mode 100644 index 0000000000..c7c0154219 --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -0,0 +1,94 @@ +package signer + +import ( + "context" + "fmt" + "strconv" + + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/wire" + "github.com/pkg/errors" + + "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/x/crosschain/types" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" + "github.com/zeta-chain/node/zetaclient/logs" +) + +// SignRBFTx signs a RBF (Replace-By-Fee) to unblock last stuck outbound transaction. +// +// The key points: +// - It reuses the stuck tx's inputs and outputs but gives a higher fee to miners. +// - Funding the last stuck outbound will be considered as CPFP (child-pays-for-parent) by miners. +func (signer *Signer) SignRBFTx( + ctx context.Context, + cctx *types.CrossChainTx, + height uint64, + lastTx *btcutil.Tx, + minRelayFee float64, +) (*wire.MsgTx, error) { + var ( + params = cctx.GetCurrentOutboundParam() + lf = map[string]any{ + logs.FieldMethod: "SignRBFTx", + logs.FieldNonce: params.TssNonce, + logs.FieldTx: lastTx.MsgTx().TxID(), + } + logger = signer.Logger().Std.With().Fields(lf).Logger() + ) + + var cctxRate int64 + switch signer.Chain().ChainId { + case chains.BitcoinRegtest.ChainId: + // hardcode for regnet E2E test, zetacore won't feed it to CCTX + cctxRate = client.FeeRateRegnetRBF + default: + // parse recent fee rate from CCTX + recentRate, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) + if err != nil || recentRate <= 0 { + return nil, fmt.Errorf("invalid fee rate %s", params.GasPriorityFee) + } + cctxRate = recentRate + } + + // create fee bumper + fb, err := NewCPFPFeeBumper( + ctx, + signer.rpc, + signer.Chain(), + lastTx, + cctxRate, + minRelayFee, + logger, + ) + if err != nil { + return nil, errors.Wrap(err, "NewCPFPFeeBumper failed") + } + + // bump tx fees + newTx, additionalFees, newRate, err := fb.BumpTxFee() + if err != nil { + return nil, errors.Wrap(err, "BumpTxFee failed") + } + logger.Info(). + Msgf("BumpTxFee succeed, additional fees: %d sats, rate: %d => %d sat/vB", additionalFees, fb.AvgFeeRate, newRate) + + // collect input amounts for signing + inAmounts := make([]int64, len(newTx.TxIn)) + for i, input := range newTx.TxIn { + preOut := input.PreviousOutPoint + preTx, err := signer.rpc.GetRawTransaction(ctx, &preOut.Hash) + if err != nil { + return nil, errors.Wrapf(err, "unable to get previous tx %s", preOut.Hash) + } + inAmounts[i] = preTx.MsgTx().TxOut[preOut.Index].Value + } + + // sign the RBF tx + err = signer.SignTx(ctx, newTx, inAmounts, height, params.TssNonce) + if err != nil { + return nil, errors.Wrap(err, "SignTx failed") + } + + return newTx, nil +} diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go new file mode 100644 index 0000000000..71126c24a9 --- /dev/null +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -0,0 +1,242 @@ +package signer_test + +import ( + "context" + "testing" + + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/pkg/errors" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" + "github.com/zeta-chain/node/zetaclient/testutils" + + "github.com/zeta-chain/node/pkg/chains" +) + +func Test_SignRBFTx(t *testing.T) { + // https://mempool.space/tx/030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0 + chain := chains.BitcoinMainnet + nonce := uint64(148) + cctx := testutils.LoadCctxByNonce(t, chain.ChainId, nonce) + txid := "030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0" + msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chain.ChainId, txid) + + // inputs + type prevTx struct { + hash *chainhash.Hash + vout uint32 + amount int64 + } + preTxs := []prevTx{ + { + hash: hashFromTXID( + t, + "efca302a18bd8cebb3b8afef13e98ecaac47157755a62ab241ef3848140cfe92", + ), vout: 0, amount: 2147, + }, + { + hash: hashFromTXID( + t, + "efca302a18bd8cebb3b8afef13e98ecaac47157755a62ab241ef3848140cfe92", + ), vout: 2, amount: 28240703, + }, + { + hash: hashFromTXID( + t, + "3dc005eb0c1d393e717070ea84aa13e334a458a4fb7c7f9f98dbf8b231b5ceef", + ), vout: 0, amount: 10000, + }, + { + hash: hashFromTXID( + t, + "74c3aca825f3b21b82ee344d939c40d4c1e836a89c18abbd521bfa69f5f6e5d7", + ), vout: 0, amount: 10000, + }, + { + hash: hashFromTXID( + t, + "87264cef0e581f4aab3c99c53221bec3219686b48088d651a8cf8a98e4c2c5bf", + ), vout: 0, amount: 10000, + }, + { + hash: hashFromTXID( + t, + "5af24933973df03d96624ae1341d79a860e8dbc2ffc841420aa6710f3abc0074", + ), vout: 0, amount: 1200000, + }, + { + hash: hashFromTXID( + t, + "b85755938ac026b2d13e5fbacf015288f453712b4eb4a02d7e4c98ee76ada530", + ), vout: 0, amount: 9610000, + }, + } + + // test cases + tests := []struct { + name string + chain chains.Chain + cctx *crosschaintypes.CrossChainTx + lastTx *btcutil.Tx + preTxs []prevTx + minRelayFee float64 + cctxRate string + liveRate int64 + memplTxsInfo *mempoolTxsInfo + errMsg string + expectedTx *wire.MsgTx + }{ + { + name: "should sign RBF tx successfully", + chain: chains.BitcoinMainnet, + cctx: cctx, + lastTx: btcutil.NewTx(msgTx.Copy()), + preTxs: preTxs, + minRelayFee: 0.00001, + cctxRate: "57", + liveRate: 59, // 59 sat/vB + memplTxsInfo: newMempoolTxsInfo( + 1, // 1 stuck tx + 0.00027213, // fees: 0.00027213 BTC + 579, // size: 579 vByte + 47, // rate: 47 sat/vB + ), + expectedTx: func() *wire.MsgTx { + // deduct additional fees + newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx.TxOut[2].Value -= 5790 + return newTx + }(), + }, + { + name: "should return error if latest fee rate is not available", + chain: chains.BitcoinMainnet, + cctx: cctx, + lastTx: btcutil.NewTx(msgTx.Copy()), + minRelayFee: 0.00001, + cctxRate: "", + errMsg: "invalid fee rate", + }, + { + name: "should return error if unable to create fee bumper", + chain: chains.BitcoinMainnet, + cctx: cctx, + lastTx: btcutil.NewTx(msgTx.Copy()), + minRelayFee: 0.00001, + cctxRate: "57", + memplTxsInfo: nil, + errMsg: "NewCPFPFeeBumper failed", + }, + { + name: "should return error if live rate is too high", + chain: chains.BitcoinMainnet, + cctx: cctx, + lastTx: btcutil.NewTx(msgTx.Copy()), + minRelayFee: 0.00001, + cctxRate: "57", + liveRate: 99, // 99 sat/vB is much higher than ccxt rate + memplTxsInfo: newMempoolTxsInfo( + 1, // 1 stuck tx + 0.00027213, // fees: 0.00027213 BTC + 579, // size: 579 vByte + 47, // rate: 47 sat/vB + ), + errMsg: "BumpTxFee failed", + }, + { + name: "should return error if live rate is too high", + chain: chains.BitcoinMainnet, + cctx: cctx, + lastTx: btcutil.NewTx(msgTx.Copy()), + minRelayFee: 0.00001, + cctxRate: "57", + liveRate: 59, // 59 sat/vB + memplTxsInfo: newMempoolTxsInfo( + 1, // 1 stuck tx + 0.00027213, // fees: 0.00027213 BTC + 579, // size: 579 vByte + 47, // rate: 47 sat/vB + ), + errMsg: "unable to get previous tx", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // ARRANGE + // setup signer + s := newTestSuite(t, tt.chain) + + // mock cctx rate + tt.cctx.GetCurrentOutboundParam().GasPriorityFee = tt.cctxRate + + // mock RPC live fee rate + if tt.liveRate > 0 { + s.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(tt.liveRate, nil) + } else { + s.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(int64(0), errors.New("rpc error")) + } + + // mock mempool txs information + if tt.memplTxsInfo != nil { + s.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything). + Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) + } else { + s.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(0, 0.0, 0, 0, "rpc error") + } + + // mock RPC transactions + if tt.preTxs != nil { + // mock first two inputs they belong to same tx + mockMsg := wire.NewMsgTx(wire.TxVersion) + mockMsg.TxOut = make([]*wire.TxOut, 3) + for _, preTx := range tt.preTxs[:2] { + mockMsg.TxOut[preTx.vout] = wire.NewTxOut(preTx.amount, []byte{}) + } + s.client.On("GetRawTransaction", mock.Anything, tt.preTxs[0].hash). + Maybe(). + Return(btcutil.NewTx(mockMsg), nil) + + // mock other inputs + for _, preTx := range tt.preTxs[2:] { + mockMsg := wire.NewMsgTx(wire.TxVersion) + mockMsg.TxOut = make([]*wire.TxOut, 3) + mockMsg.TxOut[preTx.vout] = wire.NewTxOut(preTx.amount, []byte{}) + + s.client.On("GetRawTransaction", mock.Anything, preTx.hash). + Maybe(). + Return(btcutil.NewTx(mockMsg), nil) + } + } else { + s.client.On("GetRawTransaction", mock.Anything, mock.Anything).Maybe().Return(nil, errors.New("rpc error")) + } + + // ACT + // sign tx + ctx := context.Background() + newTx, err := s.SignRBFTx(ctx, tt.cctx, 1, tt.lastTx, tt.minRelayFee) + if tt.errMsg != "" { + require.ErrorContains(t, err, tt.errMsg) + return + } + + // ASSERT + require.NoError(t, err) + + // check tx signature + for i := range newTx.TxIn { + require.Len(t, newTx.TxIn[i].Witness, 2) + } + }) + } +} + +func hashFromTXID(t *testing.T, txid string) *chainhash.Hash { + h, err := chainhash.NewHashFromStr(txid) + require.NoError(t, err) + return h +} diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 99289acc03..8d363d981d 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -36,6 +36,11 @@ type RPC interface { GetRawTransaction(ctx context.Context, hash *chainhash.Hash) (*btcutil.Tx, error) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*chainhash.Hash, error) + GetTotalMempoolParentsSizeNFees( + ctx context.Context, + childHash string, + timeout time.Duration, + ) (int64, float64, int64, int64, error) } // Signer deals with signing & broadcasting BTC transactions. @@ -125,23 +130,41 @@ func (signer *Signer) TryProcessOutbound( return } - // setup outbound data - txData, err := NewOutboundData(cctx, chain.ChainId, height, minRelayFee, logger, signer.Logger().Compliance) - if err != nil { - logger.Error().Err(err).Msg("failed to setup Bitcoin outbound data") - return - } + var ( + rbfTx = false + signedTx *wire.MsgTx + stuckTx = observer.GetLastStuckOutbound() + ) + + // sign outbound + if stuckTx != nil && params.TssNonce == stuckTx.Nonce { + // sign RBF tx + rbfTx = true + signedTx, err = signer.SignRBFTx(ctx, cctx, height, stuckTx.Tx, minRelayFee) + if err != nil { + logger.Error().Err(err).Msg("SignRBFTx failed") + return + } + logger.Info().Str(logs.FieldTx, signedTx.TxID()).Msg("SignRBFTx succeed") + } else { + // setup outbound data + txData, err := NewOutboundData(cctx, chain.ChainId, height, minRelayFee, logger, signer.Logger().Compliance) + if err != nil { + logger.Error().Err(err).Msg("failed to setup Bitcoin outbound data") + return + } - // sign withdraw tx - signedTx, err := signer.SignWithdrawTx(ctx, txData, observer) - if err != nil { - logger.Error().Err(err).Msg("SignWithdrawTx failed") - return + // sign withdraw tx + signedTx, err = signer.SignWithdrawTx(ctx, txData, observer) + if err != nil { + logger.Error().Err(err).Msg("SignWithdrawTx failed") + return + } + logger.Info().Str(logs.FieldTx, signedTx.TxID()).Msg("SignWithdrawTx succeed") } - logger.Info().Str(logs.FieldTx, signedTx.TxID()).Msg("SignWithdrawTx succeed") // broadcast signed outbound - signer.BroadcastOutbound(ctx, signedTx, params.TssNonce, cctx, observer, zetacoreClient) + signer.BroadcastOutbound(ctx, signedTx, params.TssNonce, rbfTx, cctx, observer, zetacoreClient) } // BroadcastOutbound sends the signed transaction to the Bitcoin network @@ -149,6 +172,7 @@ func (signer *Signer) BroadcastOutbound( ctx context.Context, tx *wire.MsgTx, nonce uint64, + rbfTx bool, cctx *types.CrossChainTx, ob *observer.Observer, zetacoreClient interfaces.ZetacoreClient, @@ -164,6 +188,12 @@ func (signer *Signer) BroadcastOutbound( } logger := signer.Logger().Std + // double check to ensure the tx being replaced is still the last outbound + if rbfTx && ob.GetPendingNonce() > nonce+1 { + logger.Warn().Fields(lf).Msgf("RBF tx nonce is outdated, skipping broadcasting") + return + } + // try broacasting tx with increasing backoff (1s, 2s, 4s, 8s, 16s) in case of RPC error backOff := broadcastBackoff for i := 0; i < broadcastRetries; i++ { diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index 1d9202b7b8..7c7a659eba 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -100,6 +100,8 @@ func Test_BroadcastOutbound(t *testing.T) { name string chain chains.Chain nonce uint64 + rbfTx bool + skipRBFTx bool failTracker bool }{ { @@ -111,6 +113,7 @@ func Test_BroadcastOutbound(t *testing.T) { name: "should successfully broadcast and include RBF outbound", chain: chains.BitcoinMainnet, nonce: uint64(148), + rbfTx: true, }, { name: "should successfully broadcast and include outbound, but fail to post outbound tracker", @@ -118,10 +121,18 @@ func Test_BroadcastOutbound(t *testing.T) { nonce: uint64(148), failTracker: true, }, + { + name: "should skip broadcasting RBF tx", + chain: chains.BitcoinMainnet, + nonce: uint64(148), + rbfTx: true, + skipRBFTx: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // ARRANGE // setup signer and observer s := newTestSuite(t, tt.chain) observer := s.getNewObserver(t) @@ -153,19 +164,31 @@ func Test_BroadcastOutbound(t *testing.T) { TxID: rawResult.Vin[0].Txid, }) + // set a higher pending nonce so the RBF tx is not the last tx + if tt.rbfTx && tt.skipRBFTx { + observer.SetPendingNonce(tt.nonce + 2) + } + + // ACT ctx := makeCtx(t) s.BroadcastOutbound( ctx, msgTx, tt.nonce, + tt.rbfTx, cctx, observer, s.zetacoreClient, ) + // ASSERT // check if outbound is included gotResult := observer.GetIncludedTx(tt.nonce) - require.Equal(t, txResult, gotResult) + if tt.skipRBFTx { + require.Nil(t, gotResult) + } else { + require.Equal(t, txResult, gotResult) + } }) } } @@ -391,9 +414,3 @@ func (s *testSuite) getNewObserver(t *testing.T) *observer.Observer { require.NoError(t, err) return ob } - -func hashFromTXID(t *testing.T, txid string) *chainhash.Hash { - h, err := chainhash.NewHashFromStr(txid) - require.NoError(t, err) - return h -} diff --git a/zetaclient/common/constant.go b/zetaclient/common/constant.go index c54acade92..0c5a4bf1b5 100644 --- a/zetaclient/common/constant.go +++ b/zetaclient/common/constant.go @@ -14,4 +14,7 @@ const ( // RPCStatusCheckInterval is the interval to check RPC status, 1 minute RPCStatusCheckInterval = time.Minute + + // BTCMempoolStuckTxCheckInterval is the interval to check for Bitcoin stuck transactions in the mempool + BTCMempoolStuckTxCheckInterval = 30 * time.Second ) diff --git a/zetaclient/testutils/mocks/bitcoin_client.go b/zetaclient/testutils/mocks/bitcoin_client.go index e921e1a54a..eca2f21bcd 100644 --- a/zetaclient/testutils/mocks/bitcoin_client.go +++ b/zetaclient/testutils/mocks/bitcoin_client.go @@ -385,6 +385,36 @@ func (_m *BitcoinClient) GetEstimatedFeeRate(ctx context.Context, confTarget int return r0, r1 } +// GetMempoolEntry provides a mock function with given fields: ctx, txHash +func (_m *BitcoinClient) GetMempoolEntry(ctx context.Context, txHash string) (*btcjson.GetMempoolEntryResult, error) { + ret := _m.Called(ctx, txHash) + + if len(ret) == 0 { + panic("no return value specified for GetMempoolEntry") + } + + var r0 *btcjson.GetMempoolEntryResult + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*btcjson.GetMempoolEntryResult, error)); ok { + return rf(ctx, txHash) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *btcjson.GetMempoolEntryResult); ok { + r0 = rf(ctx, txHash) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*btcjson.GetMempoolEntryResult) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, txHash) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetNetworkInfo provides a mock function with given fields: ctx func (_m *BitcoinClient) GetNetworkInfo(ctx context.Context) (*btcjson.GetNetworkInfoResult, error) { ret := _m.Called(ctx) @@ -415,6 +445,36 @@ func (_m *BitcoinClient) GetNetworkInfo(ctx context.Context) (*btcjson.GetNetwor return r0, r1 } +// GetRawMempool provides a mock function with given fields: ctx +func (_m *BitcoinClient) GetRawMempool(ctx context.Context) ([]*chainhash.Hash, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetRawMempool") + } + + var r0 []*chainhash.Hash + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*chainhash.Hash, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []*chainhash.Hash); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*chainhash.Hash) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetNewAddress provides a mock function with given fields: ctx, account func (_m *BitcoinClient) GetNewAddress(ctx context.Context, account string) (btcutil.Address, error) { ret := _m.Called(ctx, account) @@ -563,6 +623,55 @@ func (_m *BitcoinClient) GetRawTransactionVerbose(ctx context.Context, hash *cha return r0, r1 } +// GetTotalMempoolParentsSizeNFees provides a mock function with given fields: ctx, childHash, timeout +func (_m *BitcoinClient) GetTotalMempoolParentsSizeNFees(ctx context.Context, childHash string, timeout time.Duration) (int64, float64, int64, int64, error) { + ret := _m.Called(ctx, childHash, timeout) + + if len(ret) == 0 { + panic("no return value specified for GetTotalMempoolParentsSizeNFees") + } + + var r0 int64 + var r1 float64 + var r2 int64 + var r3 int64 + var r4 error + if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) (int64, float64, int64, int64, error)); ok { + return rf(ctx, childHash, timeout) + } + if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) int64); ok { + r0 = rf(ctx, childHash, timeout) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, time.Duration) float64); ok { + r1 = rf(ctx, childHash, timeout) + } else { + r1 = ret.Get(1).(float64) + } + + if rf, ok := ret.Get(2).(func(context.Context, string, time.Duration) int64); ok { + r2 = rf(ctx, childHash, timeout) + } else { + r2 = ret.Get(2).(int64) + } + + if rf, ok := ret.Get(3).(func(context.Context, string, time.Duration) int64); ok { + r3 = rf(ctx, childHash, timeout) + } else { + r3 = ret.Get(3).(int64) + } + + if rf, ok := ret.Get(4).(func(context.Context, string, time.Duration) error); ok { + r4 = rf(ctx, childHash, timeout) + } else { + r4 = ret.Error(4) + } + + return r0, r1, r2, r3, r4 +} + // GetTransaction provides a mock function with given fields: ctx, hash func (_m *BitcoinClient) GetTransaction(ctx context.Context, hash *chainhash.Hash) (*btcjson.GetTransactionResult, error) { ret := _m.Called(ctx, hash) From 6c8f9f42a9960a684bc19c7fe2914831b7139ff4 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 21:46:52 -0600 Subject: [PATCH 09/74] switch to bitcoin-core image with mempool RPCs enabled --- contrib/localnet/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index daa5a16440..3f8ae51cf3 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -199,7 +199,7 @@ services: ipv4_address: 172.20.0.102 bitcoin: - image: ghcr.io/zeta-chain/bitcoin-core-docker:a94b52f + image: ghcr.io/zeta-chain/bitcoin-core-docker:28.0-zeta6 container_name: bitcoin hostname: bitcoin networks: From e193580ab2b8d3ce812aa40d00eda964329ffd1a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 21:48:33 -0600 Subject: [PATCH 10/74] bitcoin RBF E2E test --- cmd/zetae2e/local/local.go | 1 + e2e/e2etests/e2etests.go | 10 +++ e2e/e2etests/helpers.go | 45 ++++++----- ..._bitcoin_deposit_and_withdraw_with_dust.go | 15 +++- e2e/e2etests/test_bitcoin_withdraw_rbf.go | 76 +++++++++++++++++++ e2e/utils/bitcoin.go | 49 ++++++++++++ e2e/utils/zetacore.go | 69 +++++++++++++++++ .../evm/signer/outbound_tracker_reporter.go | 11 +++ 8 files changed, 255 insertions(+), 21 deletions(-) create mode 100644 e2e/e2etests/test_bitcoin_withdraw_rbf.go create mode 100644 e2e/utils/bitcoin.go diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 45578355f3..b513b66d37 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -317,6 +317,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { e2etests.TestBitcoinWithdrawP2WSHName, e2etests.TestBitcoinWithdrawMultipleName, e2etests.TestBitcoinWithdrawRestrictedName, + //e2etests.TestBitcoinWithdrawRBFName, // leave RBF as the last BTC test } if !light { diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index aff701a25a..eb084761c7 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -91,6 +91,7 @@ const ( TestBitcoinWithdrawP2SHName = "bitcoin_withdraw_p2sh" TestBitcoinWithdrawInvalidAddressName = "bitcoin_withdraw_invalid" TestBitcoinWithdrawRestrictedName = "bitcoin_withdraw_restricted" + TestBitcoinWithdrawRBFName = "bitcoin_withdraw_rbf" /* Application tests @@ -726,6 +727,15 @@ var AllE2ETests = []runner.E2ETest{ }, TestBitcoinWithdrawRestricted, ), + runner.NewE2ETest( + TestBitcoinWithdrawRBFName, + "withdraw Bitcoin from ZEVM and replace the outbound using RBF", + []runner.ArgDefinition{ + {Description: "receiver address", DefaultValue: ""}, + {Description: "amount in btc", DefaultValue: "0.001"}, + }, + TestBitcoinWithdrawRBF, + ), /* Application tests */ diff --git a/e2e/e2etests/helpers.go b/e2e/e2etests/helpers.go index d1820491a8..4a9d47f107 100644 --- a/e2e/e2etests/helpers.go +++ b/e2e/e2etests/helpers.go @@ -8,6 +8,7 @@ import ( "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg/chainhash" + ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/e2e/runner" @@ -25,31 +26,13 @@ func randomPayload(r *runner.E2ERunner) string { } func withdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) *btcjson.TxRawResult { - tx, err := r.BTCZRC20.Approve( - r.ZEVMAuth, - r.BTCZRC20Addr, - big.NewInt(amount.Int64()*2), - ) // approve more to cover withdraw fee - require.NoError(r, err) - - receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt) + // call approve and withdraw on ZRC20 contract + receipt := approveAndWithdrawBTCZRC20(r, to, amount) // mine blocks if testing on regnet stop := r.MineBlocksIfLocalBitcoin() defer stop() - // withdraw 'amount' of BTC from ZRC20 to BTC address - tx, err = r.BTCZRC20.Withdraw(r.ZEVMAuth, []byte(to.EncodeAddress()), amount) - require.NoError(r, err) - - receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt) - - // mine 10 blocks to confirm the withdrawal tx - _, err = r.GenerateToAddressIfLocalBitcoin(10, to) - require.NoError(r, err) - // get cctx and check status cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, receipt.TxHash.Hex(), r.CctxClient, r.Logger, r.CctxTimeout) utils.RequireCCTXStatus(r, cctx, crosschaintypes.CctxStatus_OutboundMined) @@ -79,6 +62,28 @@ func withdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) return rawTx } +// approveAndWithdrawBTCZRC20 is a helper function to call 'approve' and 'withdraw' on BTCZRC20 contract +func approveAndWithdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) *ethtypes.Receipt { + tx, err := r.BTCZRC20.Approve( + r.ZEVMAuth, + r.BTCZRC20Addr, + big.NewInt(amount.Int64()*2), + ) // approve more to cover withdraw fee + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + // withdraw 'amount' of BTC from ZRC20 to BTC address + tx, err = r.BTCZRC20.Withdraw(r.ZEVMAuth, []byte(to.EncodeAddress()), amount) + require.NoError(r, err) + + receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + return receipt +} + // bigAdd is shorthand for new(big.Int).Add(x, y) func bigAdd(x *big.Int, y *big.Int) *big.Int { return new(big.Int).Add(x, y) diff --git a/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go b/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go index 8b108f2103..f478eed2f3 100644 --- a/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go +++ b/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go @@ -2,6 +2,7 @@ package e2etests import ( "math/big" + "sync" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/stretchr/testify/require" @@ -12,12 +13,24 @@ import ( crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" ) +// wgDeposit is a wait group for deposit runner to finish +var wgDepositRunner sync.WaitGroup + +func init() { + // there is one single deposit runner for Bitcoin E2E tests + wgDepositRunner.Add(1) +} + // TestBitcoinDepositAndWithdrawWithDust deposits Bitcoin and call a smart contract that withdraw dust amount // It tests the edge case where during a cross-chain call, a invaild withdraw is initiated (processLogs fails) func TestBitcoinDepositAndWithdrawWithDust(r *runner.E2ERunner, args []string) { // Given "Live" BTC network stop := r.MineBlocksIfLocalBitcoin() - defer stop() + defer func() { + stop() + // signal the deposit runner is done after this last test + wgDepositRunner.Done() + }() require.Len(r, args, 0) diff --git a/e2e/e2etests/test_bitcoin_withdraw_rbf.go b/e2e/e2etests/test_bitcoin_withdraw_rbf.go new file mode 100644 index 0000000000..bde4efd5c0 --- /dev/null +++ b/e2e/e2etests/test_bitcoin_withdraw_rbf.go @@ -0,0 +1,76 @@ +package e2etests + +import ( + "strconv" + "time" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/stretchr/testify/require" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" +) + +// TestBitcoinWithdrawRBF tests the RBF (Replace-By-Fee) feature in Zetaclient. +// It needs block mining to be stopped and runs as the last test in the suite. +// +// IMPORTANT: the test requires to simulate a stuck tx in the Bitcoin regnet. +// Changing the 'minTxConfirmations' to 1 to not include Bitcoin pending txs. +// https://github.com/zeta-chain/node/blob/feat-bitcoin-Replace-By-Fee/zetaclient/chains/bitcoin/observer/outbound.go#L30 +func TestBitcoinWithdrawRBF(r *runner.E2ERunner, args []string) { + require.Len(r, args, 2) + + // wait for block mining to stop + wgDepositRunner.Wait() + r.Logger.Print("Bitcoin mining stopped, starting RBF test") + + // parse arguments + defaultReceiver := r.BTCDeployerAddress.EncodeAddress() + to, amount := utils.ParseBitcoinWithdrawArgs(r, args, defaultReceiver, r.GetBitcoinChainID()) + + // initiate a withdraw CCTX + receipt := approveAndWithdrawBTCZRC20(r, to, amount) + cctx := utils.GetCCTXByInboundHash(r.Ctx, receipt.TxHash.Hex(), r.CctxClient) + + // wait for the 1st outbound tracker hash to come in + nonce := cctx.GetCurrentOutboundParam().TssNonce + hashes := utils.WaitOutboundTracker(r.Ctx, r.CctxClient, r.GetBitcoinChainID(), nonce, 1, r.Logger, 3*time.Minute) + txHash, err := chainhash.NewHashFromStr(hashes[0]) + r.Logger.Info("got 1st tracker hash: %s", txHash) + + // get original tx + require.NoError(r, err) + txResult, err := r.BtcRPCClient.GetTransaction(r.Ctx, txHash) + require.NoError(r, err) + require.Zero(r, txResult.Confirmations) + + // wait for RBF tx to kick in + hashes = utils.WaitOutboundTracker(r.Ctx, r.CctxClient, r.GetBitcoinChainID(), nonce, 2, r.Logger, 3*time.Minute) + txHashRBF, err := chainhash.NewHashFromStr(hashes[1]) + require.NoError(r, err) + r.Logger.Info("got 2nd tracker hash: %s", txHashRBF) + + // resume block mining + stop := r.MineBlocksIfLocalBitcoin() + defer stop() + + // waiting for CCTX to be mined + cctx = utils.WaitCctxMinedByInboundHash(r.Ctx, receipt.TxHash.Hex(), r.CctxClient, r.Logger, r.CctxTimeout) + utils.RequireCCTXStatus(r, cctx, crosschaintypes.CctxStatus_OutboundMined) + + // ensure the original tx is dropped + utils.MustHaveDroppedTx(r.Ctx, r.BtcRPCClient, txHash) + + // ensure the RBF tx is mined + rawResult := utils.MustHaveMinedTx(r.Ctx, r.BtcRPCClient, txHashRBF) + + // ensure RBF fee rate > old rate + params := cctx.GetCurrentOutboundParam() + oldRate, err := strconv.ParseInt(params.GasPrice, 10, 64) + require.NoError(r, err) + + _, newRate, err := r.BtcRPCClient.GetTransactionFeeAndRate(r.Ctx, rawResult) + require.NoError(r, err) + require.Greater(r, newRate, oldRate, "RBF fee rate should be higher than the original tx") +} diff --git a/e2e/utils/bitcoin.go b/e2e/utils/bitcoin.go new file mode 100644 index 0000000000..782e727b03 --- /dev/null +++ b/e2e/utils/bitcoin.go @@ -0,0 +1,49 @@ +package utils + +import ( + "context" + + "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/stretchr/testify/require" + + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" +) + +// MustHaveDroppedTx ensures the given tx has been dropped +func MustHaveDroppedTx(ctx context.Context, client *client.Client, txHash *chainhash.Hash) { + t := TestingFromContext(ctx) + + // dropped tx has negative confirmations + txResult, err := client.GetTransaction(ctx, txHash) + if err == nil { + require.Negative(t, txResult.Confirmations) + } + + // dropped tx should be removed from mempool + entry, err := client.GetMempoolEntry(ctx, txHash.String()) + require.Error(t, err) + require.Nil(t, entry) + + // dropped tx won't exist in blockchain + // -5: No such mempool or blockchain transaction + rawTx, err := client.GetRawTransaction(ctx, txHash) + require.Error(t, err) + require.Nil(t, rawTx) +} + +// MustHaveMinedTx ensures the given tx has been mined +func MustHaveMinedTx(ctx context.Context, client *client.Client, txHash *chainhash.Hash) *btcjson.TxRawResult { + t := TestingFromContext(ctx) + + // positive confirmations + txResult, err := client.GetTransaction(ctx, txHash) + require.NoError(t, err) + require.Positive(t, txResult.Confirmations) + + // tx exists in blockchain + rawResult, err := client.GetRawTransactionVerbose(ctx, txHash) + require.NoError(t, err) + + return rawResult +} diff --git a/e2e/utils/zetacore.go b/e2e/utils/zetacore.go index 681e3bca7d..2e64d90f7b 100644 --- a/e2e/utils/zetacore.go +++ b/e2e/utils/zetacore.go @@ -2,6 +2,7 @@ package utils import ( "context" + "fmt" "time" rpchttp "github.com/cometbft/cometbft/rpc/client/http" @@ -28,6 +29,24 @@ const ( DefaultCctxTimeout = 8 * time.Minute ) +// GetCCTXByInboundHash gets cctx by inbound hash +func GetCCTXByInboundHash( + ctx context.Context, + inboundHash string, + client crosschaintypes.QueryClient, +) *crosschaintypes.CrossChainTx { + t := TestingFromContext(ctx) + + // query cctx by inbound hash + in := &crosschaintypes.QueryInboundHashToCctxDataRequest{InboundHash: inboundHash} + res, err := client.InTxHashToCctxData(ctx, in) + + require.NoError(t, err) + require.Len(t, res.CrossChainTxs, 1) + + return &res.CrossChainTxs[0] +} + // WaitCctxMinedByInboundHash waits until cctx is mined; returns the cctxIndex (the last one) func WaitCctxMinedByInboundHash( ctx context.Context, @@ -187,6 +206,56 @@ func WaitCCTXMinedByIndex( } } +// WaitOutboundTracker wait for outbound tracker to be filled with 'hashCount' hashes +func WaitOutboundTracker( + ctx context.Context, + client crosschaintypes.QueryClient, + chainID int64, + nonce uint64, + hashCount int, + logger infoLogger, + timeout time.Duration, +) []string { + if timeout == 0 { + timeout = DefaultCctxTimeout + } + + t := TestingFromContext(ctx) + startTime := time.Now() + in := &crosschaintypes.QueryAllOutboundTrackerByChainRequest{Chain: chainID} + + for { + require.False( + t, + time.Since(startTime) > timeout, + fmt.Sprintf("waiting outbound tracker timeout, chainID: %d, nonce: %d", chainID, nonce), + ) + time.Sleep(5 * time.Second) + + outboundTracker, err := client.OutboundTrackerAllByChain(ctx, in) + require.NoError(t, err) + + // loop through all outbound trackers + for i, tracker := range outboundTracker.OutboundTracker { + if tracker.Nonce == nonce { + logger.Info("Tracker[%d]:\n", i) + logger.Info(" ChainId: %d\n", tracker.ChainId) + logger.Info(" Nonce: %d\n", tracker.Nonce) + logger.Info(" HashList:\n") + + hashes := []string{} + for j, hash := range tracker.HashList { + hashes = append(hashes, hash.TxHash) + logger.Info(" hash[%d]: %s\n", j, hash.TxHash) + } + if len(hashes) >= hashCount { + return hashes + } + } + } + } +} + type WaitOpts func(c *waitConfig) // MatchStatus is the WaitOpts that matches CCTX with the given status. diff --git a/zetaclient/chains/evm/signer/outbound_tracker_reporter.go b/zetaclient/chains/evm/signer/outbound_tracker_reporter.go index 8be76f66a4..78f216ca5f 100644 --- a/zetaclient/chains/evm/signer/outbound_tracker_reporter.go +++ b/zetaclient/chains/evm/signer/outbound_tracker_reporter.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog" "github.com/zeta-chain/node/pkg/bg" + crosschainkeeper "github.com/zeta-chain/node/x/crosschain/keeper" "github.com/zeta-chain/node/zetaclient/chains/evm" "github.com/zeta-chain/node/zetaclient/chains/evm/rpc" "github.com/zeta-chain/node/zetaclient/chains/interfaces" @@ -69,6 +70,16 @@ func (signer *Signer) reportToOutboundTracker( continue } + // stop if the cctx is already finalized + cctx, err := zetacoreClient.GetCctxByNonce(ctx, chainID, nonce) + if err != nil { + logger.Err(err).Msg("unable to get cctx for outbound") + continue + } else if !crosschainkeeper.IsPending(cctx) { + logger.Info().Msg("cctx is finalized") + return nil + } + // report outbound hash to tracker zetaHash, err := zetacoreClient.PostOutboundTracker(ctx, chainID, nonce, outboundHash) if err != nil { From a80bd917e23709876e9c6eaab1b7ee90d5be8f1a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 21:59:16 -0600 Subject: [PATCH 11/74] add changelog entry --- changelog.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index eadeb13687..7ff2d0295c 100644 --- a/changelog.md +++ b/changelog.md @@ -2,13 +2,16 @@ ## unreleased +### Features + +* [3396](https://github.com/zeta-chain/node/pull/3396) - add support for Bitcoin RBF (Replace-By-Fee) in zetaclient + ### Refactor * [3332](https://github.com/zeta-chain/node/pull/3332) - implement orchestrator V2. Move BTC observer-signer to V2 * [3360](https://github.com/zeta-chain/node/pull/3360) - update protocol contract imports using consolidated path * [3349](https://github.com/zeta-chain/node/pull/3349) - implement new bitcoin rpc in zetaclient with improved performance and observability * [3379](https://github.com/zeta-chain/node/pull/3379) - add Avalanche, Arbitrum and World Chain in chain info -* [3381](https://github.com/zeta-chain/node/pull/3381) - split Bitcoin observer and signer into small files and organize outbound logic into reusable/testable functions; renaming, type unification, etc. ### Fixes From 6803a7c31b1dcbae02b25f960313ad6714d28c8f Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 22:00:03 -0600 Subject: [PATCH 12/74] add changelog entry --- changelog.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index eadeb13687..7ff2d0295c 100644 --- a/changelog.md +++ b/changelog.md @@ -2,13 +2,16 @@ ## unreleased +### Features + +* [3396](https://github.com/zeta-chain/node/pull/3396) - add support for Bitcoin RBF (Replace-By-Fee) in zetaclient + ### Refactor * [3332](https://github.com/zeta-chain/node/pull/3332) - implement orchestrator V2. Move BTC observer-signer to V2 * [3360](https://github.com/zeta-chain/node/pull/3360) - update protocol contract imports using consolidated path * [3349](https://github.com/zeta-chain/node/pull/3349) - implement new bitcoin rpc in zetaclient with improved performance and observability * [3379](https://github.com/zeta-chain/node/pull/3379) - add Avalanche, Arbitrum and World Chain in chain info -* [3381](https://github.com/zeta-chain/node/pull/3381) - split Bitcoin observer and signer into small files and organize outbound logic into reusable/testable functions; renaming, type unification, etc. ### Fixes From fc6f9a414570aef91076d1bd404b5e3b006f5603 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 22 Jan 2025 22:01:14 -0600 Subject: [PATCH 13/74] update changelog --- changelog.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 7ff2d0295c..140bef8f6d 100644 --- a/changelog.md +++ b/changelog.md @@ -2,9 +2,7 @@ ## unreleased -### Features - -* [3396](https://github.com/zeta-chain/node/pull/3396) - add support for Bitcoin RBF (Replace-By-Fee) in zetaclient +### Tests ### Refactor From c2a52c0b5e8397000d3cfafb23bc1583902f1ee0 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 24 Jan 2025 22:58:45 -0600 Subject: [PATCH 14/74] improve RBF test waiting mechanism --- cmd/zetae2e/local/bitcoin.go | 29 +++++++++++++++---- cmd/zetae2e/local/local.go | 2 +- ..._bitcoin_deposit_and_withdraw_with_dust.go | 16 +--------- e2e/e2etests/test_bitcoin_withdraw_rbf.go | 6 +--- e2e/utils/zetacore.go | 2 +- 5 files changed, 27 insertions(+), 28 deletions(-) diff --git a/cmd/zetae2e/local/bitcoin.go b/cmd/zetae2e/local/bitcoin.go index c11b00494c..f3d68762ec 100644 --- a/cmd/zetae2e/local/bitcoin.go +++ b/cmd/zetae2e/local/bitcoin.go @@ -2,6 +2,7 @@ package local import ( "fmt" + "sync" "time" "github.com/fatih/color" @@ -64,8 +65,8 @@ func bitcoinTestRoutines( } // create test routines - routineDeposit := createBitcoinTestRoutine(runnerDeposit, depositTests) - routineWithdraw := createBitcoinTestRoutine(runnerWithdraw, withdrawTests) + routineDeposit, wgDeposit := createBitcoinTestRoutine(runnerDeposit, depositTests, nil) + routineWithdraw, _ := createBitcoinTestRoutine(runnerWithdraw, withdrawTests, wgDeposit) return routineDeposit, routineWithdraw } @@ -108,7 +109,15 @@ func initBitcoinRunner( } // createBitcoinTestRoutine creates a test routine for given test names -func createBitcoinTestRoutine(r *runner.E2ERunner, testNames []string) func() error { +// The 'wgDependency' argument is used to wait for dependent routine to complete +func createBitcoinTestRoutine( + r *runner.E2ERunner, + testNames []string, + wgDependency *sync.WaitGroup, +) (func() error, *sync.WaitGroup) { + var thisRoutine sync.WaitGroup + thisRoutine.Add(1) + return func() (err error) { r.Logger.Print("🏃 starting bitcoin tests") startTime := time.Now() @@ -122,12 +131,20 @@ func createBitcoinTestRoutine(r *runner.E2ERunner, testNames []string) func() er return fmt.Errorf("bitcoin tests failed: %v", err) } - if err := r.RunE2ETests(testsToRun); err != nil { - return fmt.Errorf("bitcoin tests failed: %v", err) + for _, test := range testsToRun { + // RBF test needs to wait for all deposit tests to complete + if test.Name == e2etests.TestBitcoinWithdrawRBFName && wgDependency != nil { + r.Logger.Print("⏳waiting - %s", test.Description) + wgDependency.Wait() + } + if err := r.RunE2ETest(test, true); err != nil { + return fmt.Errorf("bitcoin tests failed: %v", err) + } } + thisRoutine.Done() r.Logger.Print("🍾 bitcoin tests completed in %s", time.Since(startTime).String()) return err - } + }, &thisRoutine } diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index b513b66d37..820c1d0b7a 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -317,7 +317,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { e2etests.TestBitcoinWithdrawP2WSHName, e2etests.TestBitcoinWithdrawMultipleName, e2etests.TestBitcoinWithdrawRestrictedName, - //e2etests.TestBitcoinWithdrawRBFName, // leave RBF as the last BTC test + e2etests.TestBitcoinWithdrawRBFName, } if !light { diff --git a/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go b/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go index f478eed2f3..42adcb9ab1 100644 --- a/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go +++ b/e2e/e2etests/test_bitcoin_deposit_and_withdraw_with_dust.go @@ -2,7 +2,6 @@ package e2etests import ( "math/big" - "sync" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/stretchr/testify/require" @@ -13,29 +12,16 @@ import ( crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" ) -// wgDeposit is a wait group for deposit runner to finish -var wgDepositRunner sync.WaitGroup - -func init() { - // there is one single deposit runner for Bitcoin E2E tests - wgDepositRunner.Add(1) -} - // TestBitcoinDepositAndWithdrawWithDust deposits Bitcoin and call a smart contract that withdraw dust amount // It tests the edge case where during a cross-chain call, a invaild withdraw is initiated (processLogs fails) func TestBitcoinDepositAndWithdrawWithDust(r *runner.E2ERunner, args []string) { // Given "Live" BTC network stop := r.MineBlocksIfLocalBitcoin() - defer func() { - stop() - // signal the deposit runner is done after this last test - wgDepositRunner.Done() - }() + defer stop() require.Len(r, args, 0) // ARRANGE - // Deploy the withdrawer contract on ZetaChain with a withdraw amount of 100 satoshis (dust amount is 1000 satoshis) withdrawerAddr, tx, _, err := withdrawer.DeployWithdrawer(r.ZEVMAuth, r.ZEVMClient, big.NewInt(100)) require.NoError(r, err) diff --git a/e2e/e2etests/test_bitcoin_withdraw_rbf.go b/e2e/e2etests/test_bitcoin_withdraw_rbf.go index bde4efd5c0..c98f1d9a54 100644 --- a/e2e/e2etests/test_bitcoin_withdraw_rbf.go +++ b/e2e/e2etests/test_bitcoin_withdraw_rbf.go @@ -21,17 +21,13 @@ import ( func TestBitcoinWithdrawRBF(r *runner.E2ERunner, args []string) { require.Len(r, args, 2) - // wait for block mining to stop - wgDepositRunner.Wait() - r.Logger.Print("Bitcoin mining stopped, starting RBF test") - // parse arguments defaultReceiver := r.BTCDeployerAddress.EncodeAddress() to, amount := utils.ParseBitcoinWithdrawArgs(r, args, defaultReceiver, r.GetBitcoinChainID()) // initiate a withdraw CCTX receipt := approveAndWithdrawBTCZRC20(r, to, amount) - cctx := utils.GetCCTXByInboundHash(r.Ctx, receipt.TxHash.Hex(), r.CctxClient) + cctx := utils.GetCCTXByInboundHash(r.Ctx, r.CctxClient, receipt.TxHash.Hex()) // wait for the 1st outbound tracker hash to come in nonce := cctx.GetCurrentOutboundParam().TssNonce diff --git a/e2e/utils/zetacore.go b/e2e/utils/zetacore.go index 2e64d90f7b..9536eaecc3 100644 --- a/e2e/utils/zetacore.go +++ b/e2e/utils/zetacore.go @@ -32,8 +32,8 @@ const ( // GetCCTXByInboundHash gets cctx by inbound hash func GetCCTXByInboundHash( ctx context.Context, - inboundHash string, client crosschaintypes.QueryClient, + inboundHash string, ) *crosschaintypes.CrossChainTx { t := TestingFromContext(ctx) From 1d2afef8cb2bbb9bb043e99c2151ce789d48ce87 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 24 Jan 2025 23:51:25 -0600 Subject: [PATCH 15/74] rename sample function BTCAddressP2WPKH --- testutil/sample/crypto.go | 14 +++++++------- x/crosschain/types/cctx_test.go | 3 ++- x/crosschain/types/revert_options_test.go | 3 ++- zetaclient/chains/bitcoin/observer/event_test.go | 9 ++++++--- zetaclient/chains/bitcoin/observer/inbound_test.go | 4 +++- zetaclient/chains/bitcoin/signer/sign_test.go | 8 +++++--- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/testutil/sample/crypto.go b/testutil/sample/crypto.go index 5f81147ce8..2ff635678f 100644 --- a/testutil/sample/crypto.go +++ b/testutil/sample/crypto.go @@ -8,7 +8,6 @@ import ( "strconv" "testing" - "github.com/btcsuite/btcd/btcec/v2" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" @@ -17,6 +16,7 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keys/ed25519" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" sdk "github.com/cosmos/cosmos-sdk/types" + secp "github.com/decred/dcrd/dcrec/secp256k1/v4" ethcommon "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" ethcrypto "github.com/ethereum/go-ethereum/crypto" @@ -91,9 +91,9 @@ func EthAddressFromRand(r *rand.Rand) ethcommon.Address { return ethcommon.BytesToAddress(sdk.AccAddress(PubKey(r).Address()).Bytes()) } -// BTCAddressP2WPKH returns a sample Bitcoin P2WPKH address -func BTCAddressP2WPKH(t *testing.T, net *chaincfg.Params) *btcutil.AddressWitnessPubKeyHash { - privateKey, err := btcec.NewPrivateKey() +// BTCAddressP2WPKH returns a sample Bitcoin Pay-to-Witness-Public-Key-Hash (P2WPKH) address +func BTCAddressP2WPKH(t *testing.T, r *rand.Rand, net *chaincfg.Params) *btcutil.AddressWitnessPubKeyHash { + privateKey, err := secp.GeneratePrivateKeyFromRand(r) require.NoError(t, err) pubKeyHash := btcutil.Hash160(privateKey.PubKey().SerializeCompressed()) @@ -103,9 +103,9 @@ func BTCAddressP2WPKH(t *testing.T, net *chaincfg.Params) *btcutil.AddressWitnes return addr } -// BTCAddressP2WPKHScript returns a pkscript for a sample Bitcoin P2WPKH address -func BTCAddressP2WPKHScript(t *testing.T, net *chaincfg.Params) []byte { - addr := BTCAddressP2WPKH(t, net) +// BtcAddressP2WPKH returns a pkscript for a sample btc P2WPKH address +func BTCAddressP2WPKHScript(t *testing.T, r *rand.Rand, net *chaincfg.Params) []byte { + addr := BTCAddressP2WPKH(t, r, net) script, err := txscript.PayToAddrScript(addr) require.NoError(t, err) return script diff --git a/x/crosschain/types/cctx_test.go b/x/crosschain/types/cctx_test.go index 6ef409e296..7fed655d89 100644 --- a/x/crosschain/types/cctx_test.go +++ b/x/crosschain/types/cctx_test.go @@ -137,10 +137,11 @@ func Test_SetRevertOutboundValues(t *testing.T) { }) t.Run("successfully set BTC revert address V1", func(t *testing.T) { + r := sample.Rand() cctx := sample.CrossChainTx(t, "test") cctx.InboundParams.SenderChainId = chains.BitcoinTestnet.ChainId cctx.OutboundParams = cctx.OutboundParams[:1] - cctx.RevertOptions.RevertAddress = sample.BTCAddressP2WPKH(t, &chaincfg.TestNet3Params).String() + cctx.RevertOptions.RevertAddress = sample.BTCAddressP2WPKH(t, r, &chaincfg.TestNet3Params).String() err := cctx.AddRevertOutbound(100) require.NoError(t, err) diff --git a/x/crosschain/types/revert_options_test.go b/x/crosschain/types/revert_options_test.go index f77fb83363..5c4757892b 100644 --- a/x/crosschain/types/revert_options_test.go +++ b/x/crosschain/types/revert_options_test.go @@ -49,7 +49,8 @@ func TestRevertOptions_GetEVMRevertAddress(t *testing.T) { func TestRevertOptions_GetBTCRevertAddress(t *testing.T) { t.Run("valid Bitcoin revert address", func(t *testing.T) { - addr := sample.BTCAddressP2WPKH(t, &chaincfg.TestNet3Params).String() + r := sample.Rand() + addr := sample.BTCAddressP2WPKH(t, r, &chaincfg.TestNet3Params).String() actualAddr, valid := types.RevertOptions{ RevertAddress: addr, }.GetBTCRevertAddress(chains.BitcoinTestnet.ChainId) diff --git a/zetaclient/chains/bitcoin/observer/event_test.go b/zetaclient/chains/bitcoin/observer/event_test.go index a616f39d37..4d3492105f 100644 --- a/zetaclient/chains/bitcoin/observer/event_test.go +++ b/zetaclient/chains/bitcoin/observer/event_test.go @@ -32,7 +32,7 @@ func createTestBtcEvent( memoStd *memo.InboundMemo, ) observer.BTCInboundEvent { return observer.BTCInboundEvent{ - FromAddress: sample.BTCAddressP2WPKH(t, net).String(), + FromAddress: sample.BTCAddressP2WPKH(t, sample.Rand(), net).String(), ToAddress: sample.EthAddress().Hex(), MemoBytes: memo, MemoStd: memoStd, @@ -235,6 +235,8 @@ func Test_DecodeEventMemoBytes(t *testing.T) { } func Test_ValidateStandardMemo(t *testing.T) { + r := sample.Rand() + // test cases tests := []struct { name string @@ -249,7 +251,7 @@ func Test_ValidateStandardMemo(t *testing.T) { }, FieldsV0: memo.FieldsV0{ RevertOptions: crosschaintypes.RevertOptions{ - RevertAddress: sample.BTCAddressP2WPKH(t, &chaincfg.TestNet3Params).String(), + RevertAddress: sample.BTCAddressP2WPKH(t, r, &chaincfg.TestNet3Params).String(), }, }, }, @@ -399,8 +401,9 @@ func Test_NewInboundVoteFromStdMemo(t *testing.T) { t.Run("should create new inbound vote msg with standard memo", func(t *testing.T) { // create revert options + r := sample.Rand() revertOptions := crosschaintypes.NewEmptyRevertOptions() - revertOptions.RevertAddress = sample.BTCAddressP2WPKH(t, &chaincfg.MainNetParams).String() + revertOptions.RevertAddress = sample.BTCAddressP2WPKH(t, r, &chaincfg.MainNetParams).String() // create test event receiver := sample.EthAddress() diff --git a/zetaclient/chains/bitcoin/observer/inbound_test.go b/zetaclient/chains/bitcoin/observer/inbound_test.go index e57b5a6d56..7af9737463 100644 --- a/zetaclient/chains/bitcoin/observer/inbound_test.go +++ b/zetaclient/chains/bitcoin/observer/inbound_test.go @@ -151,6 +151,8 @@ func TestAvgFeeRateBlock828440Errors(t *testing.T) { } func Test_GetInboundVoteFromBtcEvent(t *testing.T) { + r := sample.Rand() + // can use any bitcoin chain for testing chain := chains.BitcoinMainnet @@ -167,7 +169,7 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { { name: "should return vote for standard memo", event: &observer.BTCInboundEvent{ - FromAddress: sample.BTCAddressP2WPKH(t, &chaincfg.MainNetParams).String(), + FromAddress: sample.BTCAddressP2WPKH(t, r, &chaincfg.MainNetParams).String(), // a deposit and call MemoBytes: testutil.HexToBytes( t, diff --git a/zetaclient/chains/bitcoin/signer/sign_test.go b/zetaclient/chains/bitcoin/signer/sign_test.go index 6d85db5f4d..ba0aa8631c 100644 --- a/zetaclient/chains/bitcoin/signer/sign_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_test.go @@ -22,6 +22,7 @@ import ( ) func Test_AddTxInputs(t *testing.T) { + r := sample.Rand() net := &chaincfg.MainNetParams tests := []struct { @@ -36,13 +37,13 @@ func Test_AddTxInputs(t *testing.T) { { TxID: sample.BtcHash().String(), Vout: 0, - Address: sample.BTCAddressP2WPKH(t, net).String(), + Address: sample.BTCAddressP2WPKH(t, r, net).String(), Amount: 0.1, }, { TxID: sample.BtcHash().String(), Vout: 1, - Address: sample.BTCAddressP2WPKH(t, net).String(), + Address: sample.BTCAddressP2WPKH(t, r, net).String(), Amount: 0.2, }, }, @@ -310,8 +311,9 @@ func Test_SignTx(t *testing.T) { require.Len(t, inAmounts, len(tt.inputs)) // add outputs + r := sample.Rand() for _, amount := range tt.outputs { - pkScript := sample.BTCAddressP2WPKHScript(t, tt.net) + pkScript := sample.BTCAddressP2WPKHScript(t, r, tt.net) tx.AddTxOut(wire.NewTxOut(amount, pkScript)) } From 1033a91024990dd3eb394bbe023d50eeb78eedd6 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Mon, 27 Jan 2025 10:42:33 -0600 Subject: [PATCH 16/74] remove unused test and comment --- zetaclient/chains/bitcoin/observer/outbound.go | 1 - zetaclient/chains/bitcoin/signer/signer_test.go | 5 ----- 2 files changed, 6 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index 7192b754b0..d7ef25e85e 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -369,7 +369,6 @@ func (ob *Observer) GetIncludedTx(nonce uint64) *btcjson.GetTransactionResult { } // Basic TSS outbound checks: -// - confirmations >= 0 // - should be able to query the raw tx // - check if all inputs are segwit && TSS inputs // diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index 22345da98d..72db8b38b9 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -105,11 +105,6 @@ func Test_BroadcastOutbound(t *testing.T) { chain: chains.BitcoinMainnet, nonce: uint64(148), }, - { - name: "should successfully broadcast and include RBF outbound", - chain: chains.BitcoinMainnet, - nonce: uint64(148), - }, { name: "should successfully broadcast and include outbound, but fail to post outbound tracker", chain: chains.BitcoinMainnet, From 199bf2e2b6bc53ddb54b2d3742bdfdbec48cd168 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 15:23:38 -0600 Subject: [PATCH 17/74] use 1.5 as multiplier to send outbound txs so that we slightly make profit --- .../chains/bitcoin/client/client_test.go | 2 +- zetaclient/chains/bitcoin/common/fee.go | 14 ++++++- zetaclient/chains/bitcoin/common/fee_test.go | 21 ++++++++++ .../chains/bitcoin/observer/inbound_test.go | 8 ++-- .../chains/bitcoin/observer/witness_test.go | 2 +- .../chains/bitcoin/signer/outbound_data.go | 16 +++++--- .../bitcoin/signer/outbound_data_test.go | 8 ++-- zetaclient/chains/bitcoin/signer/sign.go | 6 +-- zetaclient/chains/bitcoin/signer/sign_rbf.go | 22 +++++------ .../chains/bitcoin/signer/sign_rbf_test.go | 34 ++++++----------- zetaclient/chains/bitcoin/signer/signer.go | 2 +- .../chains/bitcoin/signer/signer_test.go | 38 +++++++++---------- zetaclient/common/constant.go | 15 +++++--- zetaclient/zetacore/client_vote.go | 2 +- zetaclient/zetacore/tx.go | 10 ++--- zetaclient/zetacore/tx_test.go | 4 +- 16 files changed, 115 insertions(+), 89 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/client_test.go b/zetaclient/chains/bitcoin/client/client_test.go index 55091ebbef..87e9ee003a 100644 --- a/zetaclient/chains/bitcoin/client/client_test.go +++ b/zetaclient/chains/bitcoin/client/client_test.go @@ -464,7 +464,7 @@ func TestClientLive(t *testing.T) { // the actual fee rate is 860 sat/vByte // #nosec G115 always in range - expectedRate := int64(float64(860) * common.BTCOutboundGasPriceMultiplier) + expectedRate := int64(float64(860) * common.BTCGasPriceMultiplierFeeCharge) expectedFee := btc.DepositorFee(expectedRate) require.Equal(t, expectedFee, depositorFee) }) diff --git a/zetaclient/chains/bitcoin/common/fee.go b/zetaclient/chains/bitcoin/common/fee.go index ffa457f784..92cc9d1083 100644 --- a/zetaclient/chains/bitcoin/common/fee.go +++ b/zetaclient/chains/bitcoin/common/fee.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/pkg/errors" - clientcommon "github.com/zeta-chain/node/zetaclient/common" + "github.com/zeta-chain/node/zetaclient/common" ) const ( @@ -167,6 +167,16 @@ func DepositorFee(satPerByte int64) float64 { return float64(satPerByte) * float64(BtcOutboundBytesDepositor) / btcutil.SatoshiPerBitcoin } +// OutboundFeeRateFromCCTXRate calculates the outbound fee rate from the median rate +// Example: OutboundFeeRateFromCCTXRate(10) => 10 / 2.0 * 1.5 = 7.5 ≈ 8 +// Example: OutboundFeeRateFromCCTXRate(20) => 20 / 2.0 * 1.5 = 15 +// +// TSS will make profit from the difference between fee charged from users and fee paid to Bitcoin network +func OutboundFeeRateFromCCTXRate(cctxRate int64) int64 { + marketRate := float64(cctxRate) / common.BTCGasPriceMultiplierFeeCharge + return int64(math.Round(marketRate * common.BTCGasPriceMultiplierSendTx)) +} + // CalcBlockAvgFeeRate calculates the average gas rate (in sat/vByte) for a given block func CalcBlockAvgFeeRate(blockVb *btcjson.GetBlockVerboseTxResult, netParams *chaincfg.Params) (int64, error) { // sanity check @@ -250,7 +260,7 @@ func CalcDepositorFee( // apply gas price multiplier // #nosec G115 always in range - feeRate = int64(float64(feeRate) * clientcommon.BTCOutboundGasPriceMultiplier) + feeRate = int64(float64(feeRate) * common.BTCGasPriceMultiplierFeeCharge) return DepositorFee(feeRate), nil } diff --git a/zetaclient/chains/bitcoin/common/fee_test.go b/zetaclient/chains/bitcoin/common/fee_test.go index 4465ed6645..0ab0d9f9ca 100644 --- a/zetaclient/chains/bitcoin/common/fee_test.go +++ b/zetaclient/chains/bitcoin/common/fee_test.go @@ -1,6 +1,7 @@ package common import ( + "fmt" "math/rand" "testing" @@ -475,6 +476,26 @@ func TestOutboundSizeBreakdown(t *testing.T) { require.Equal(t, depositFee, 0.00001360) } +func TestOutboundFeeRateFromCCTXRate(t *testing.T) { + tests := []struct { + inputRate int64 + outputRate int64 + }{ + {inputRate: 0, outputRate: 0}, + {inputRate: 1, outputRate: 1}, + {inputRate: 2, outputRate: 2}, + {inputRate: 10, outputRate: 8}, + {inputRate: 20, outputRate: 15}, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + output := OutboundFeeRateFromCCTXRate(test.inputRate) + require.Equal(t, test.outputRate, output) + }) + } +} + func TestOutboundSizeMinMaxError(t *testing.T) { // P2TR output is the largest in size; P2WPKH is the smallest toP2TR := getTestAddrScript(t, ScriptTypeP2TR) diff --git a/zetaclient/chains/bitcoin/observer/inbound_test.go b/zetaclient/chains/bitcoin/observer/inbound_test.go index 3423aab4ce..2e2ee7926b 100644 --- a/zetaclient/chains/bitcoin/observer/inbound_test.go +++ b/zetaclient/chains/bitcoin/observer/inbound_test.go @@ -289,7 +289,7 @@ func TestGetBtcEventWithoutWitness(t *testing.T) { net := &chaincfg.MainNetParams // fee rate of above tx is 28 sat/vB - depositorFee := common.DepositorFee(28 * clientcommon.BTCOutboundGasPriceMultiplier) + depositorFee := common.DepositorFee(28 * clientcommon.BTCGasPriceMultiplierFeeCharge) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) // expected result @@ -647,7 +647,7 @@ func TestGetBtcEventErrors(t *testing.T) { blockNumber := uint64(835640) // fee rate of above tx is 28 sat/vB - depositorFee := common.DepositorFee(28 * clientcommon.BTCOutboundGasPriceMultiplier) + depositorFee := common.DepositorFee(28 * clientcommon.BTCGasPriceMultiplierFeeCharge) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) t.Run("should return error on invalid Vout[0] script", func(t *testing.T) { @@ -727,7 +727,7 @@ func TestGetBtcEvent(t *testing.T) { blockNumber := uint64(835640) net := &chaincfg.MainNetParams // 2.992e-05, see avgFeeRate https://mempool.space/api/v1/blocks/835640 - depositorFee := common.DepositorFee(22 * clientcommon.BTCOutboundGasPriceMultiplier) + depositorFee := common.DepositorFee(22 * clientcommon.BTCGasPriceMultiplierFeeCharge) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) txHash2 := "37777defed8717c581b4c0509329550e344bdc14ac38f71fc050096887e535c8" @@ -761,7 +761,7 @@ func TestGetBtcEvent(t *testing.T) { net := &chaincfg.MainNetParams // fee rate of above tx is 28 sat/vB - depositorFee := common.DepositorFee(28 * clientcommon.BTCOutboundGasPriceMultiplier) + depositorFee := common.DepositorFee(28 * clientcommon.BTCGasPriceMultiplierFeeCharge) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) // expected result diff --git a/zetaclient/chains/bitcoin/observer/witness_test.go b/zetaclient/chains/bitcoin/observer/witness_test.go index 469c7aee22..4fbd463a3f 100644 --- a/zetaclient/chains/bitcoin/observer/witness_test.go +++ b/zetaclient/chains/bitcoin/observer/witness_test.go @@ -64,7 +64,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { net := &chaincfg.MainNetParams // fee rate of above tx is 28 sat/vB - depositorFee := common.DepositorFee(28 * clientcommon.BTCOutboundGasPriceMultiplier) + depositorFee := common.DepositorFee(28 * clientcommon.BTCGasPriceMultiplierFeeCharge) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) t.Run("decode OP_RETURN ok", func(t *testing.T) { diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index 7caecef55c..b2df223d9e 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -79,6 +79,17 @@ func NewOutboundData( feeRate = newRate } + // apply outbound fee rate multiplier + feeRate = common.OutboundFeeRateFromCCTXRate(feeRate) + + // to avoid minRelayTxFee error, please do not use the minimum rate (1 sat/vB by default). + // we simply add additional 1 sat/vB to 'minRate' to avoid tx rejection by Bitcoin core. + // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/policy.h#L35 + minRate := common.FeeRateToSatPerByte(minRelayFee) + if feeRate <= minRate { + feeRate = minRate + 1 + } + // check receiver address to, err := chains.DecodeBtcAddress(params.Receiver, params.ReceiverChainId) if err != nil { @@ -97,11 +108,6 @@ func NewOutboundData( return nil, fmt.Errorf("invalid gas limit %d", params.CallOptions.GasLimit) } - // add minimum relay fee (1000 satoshis/KB by default) to gasPrice to avoid minRelayTxFee error - // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/policy.h#L35 - satPerByte := common.FeeRateToSatPerByte(minRelayFee) - feeRate += satPerByte - // compliance check restrictedCCTX := compliance.IsCctxRestricted(cctx) if restrictedCCTX { diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index 9ec6991178..550280921a 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -58,7 +58,7 @@ func Test_NewOutboundData(t *testing.T) { to: receiver, amount: 0.1, amountSats: 10000000, - feeRate: 11, // 10 + 1 (minRelayFee) + feeRate: 8, // Round(7.5) txSize: 254, nonce: 1, height: 101, @@ -87,7 +87,7 @@ func Test_NewOutboundData(t *testing.T) { to: receiver, amount: 0.1, amountSats: 10000000, - feeRate: 16, // 15 + 1 (minRelayFee) + feeRate: 11, // Round(11.25) txSize: 254, nonce: 1, height: 101, @@ -185,7 +185,7 @@ func Test_NewOutboundData(t *testing.T) { to: receiver, amount: 0, // should cancel the tx amountSats: 0, - feeRate: 11, // 10 + 1 (minRelayFee) + feeRate: 8, // Round(7.5) txSize: 254, nonce: 1, height: 101, @@ -212,7 +212,7 @@ func Test_NewOutboundData(t *testing.T) { to: receiver, amount: 0, // should cancel the tx amountSats: 0, - feeRate: 11, // 10 + 1 (minRelayFee) + feeRate: 8, // Round(7.5) txSize: 254, nonce: 1, height: 101, diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index 176a74fb37..d1c83863eb 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -90,10 +90,9 @@ func (signer *Signer) SignWithdrawTx( signer.Logger().Std.Info(). Msgf("txSize %d is less than BtcOutboundBytesWithdrawer %d for nonce %d", txData.txSize, txSize, txData.nonce) } - if txSize < common.OutboundBytesMin { // outbound shouldn't be blocked by low sizeLimit + if txSize < common.OutboundBytesMin { signer.Logger().Std.Warn(). Msgf("txSize %d is less than outboundBytesMin %d; use outboundBytesMin", txSize, common.OutboundBytesMin) - txSize = common.OutboundBytesMin } if txSize > common.OutboundBytesMax { // in case of accident signer.Logger().Std.Warn(). @@ -115,9 +114,6 @@ func (signer *Signer) SignWithdrawTx( if err != nil { return nil, err } - if err != nil { - return nil, err - } // sign the tx err = signer.SignTx(ctx, tx, inAmounts, txData.height, txData.nonce) diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go index c7c0154219..efc022f269 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -10,8 +10,8 @@ import ( "github.com/pkg/errors" "github.com/zeta-chain/node/pkg/chains" - "github.com/zeta-chain/node/x/crosschain/types" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/zetaclient/logs" ) @@ -22,16 +22,16 @@ import ( // - Funding the last stuck outbound will be considered as CPFP (child-pays-for-parent) by miners. func (signer *Signer) SignRBFTx( ctx context.Context, - cctx *types.CrossChainTx, height uint64, + nonce uint64, lastTx *btcutil.Tx, + latestRateStr string, minRelayFee float64, ) (*wire.MsgTx, error) { var ( - params = cctx.GetCurrentOutboundParam() - lf = map[string]any{ + lf = map[string]any{ logs.FieldMethod: "SignRBFTx", - logs.FieldNonce: params.TssNonce, + logs.FieldNonce: nonce, logs.FieldTx: lastTx.MsgTx().TxID(), } logger = signer.Logger().Std.With().Fields(lf).Logger() @@ -43,12 +43,12 @@ func (signer *Signer) SignRBFTx( // hardcode for regnet E2E test, zetacore won't feed it to CCTX cctxRate = client.FeeRateRegnetRBF default: - // parse recent fee rate from CCTX - recentRate, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) - if err != nil || recentRate <= 0 { - return nil, fmt.Errorf("invalid fee rate %s", params.GasPriorityFee) + // parse latest fee rate from CCTX + latestRate, err := strconv.ParseInt(latestRateStr, 10, 64) + if err != nil || latestRate <= 0 { + return nil, fmt.Errorf("invalid fee rate %s", latestRateStr) } - cctxRate = recentRate + cctxRate = common.OutboundFeeRateFromCCTXRate(latestRate) } // create fee bumper @@ -85,7 +85,7 @@ func (signer *Signer) SignRBFTx( } // sign the RBF tx - err = signer.SignTx(ctx, newTx, inAmounts, height, params.TssNonce) + err = signer.SignTx(ctx, newTx, inAmounts, height, nonce) if err != nil { return nil, errors.Wrap(err, "SignTx failed") } diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index 71126c24a9..bf86916965 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -10,7 +10,6 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" "github.com/zeta-chain/node/zetaclient/testutils" @@ -21,7 +20,6 @@ func Test_SignRBFTx(t *testing.T) { // https://mempool.space/tx/030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0 chain := chains.BitcoinMainnet nonce := uint64(148) - cctx := testutils.LoadCctxByNonce(t, chain.ChainId, nonce) txid := "030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0" msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chain.ChainId, txid) @@ -80,11 +78,10 @@ func Test_SignRBFTx(t *testing.T) { tests := []struct { name string chain chains.Chain - cctx *crosschaintypes.CrossChainTx lastTx *btcutil.Tx preTxs []prevTx minRelayFee float64 - cctxRate string + cctxRateStr string liveRate int64 memplTxsInfo *mempoolTxsInfo errMsg string @@ -93,12 +90,11 @@ func Test_SignRBFTx(t *testing.T) { { name: "should sign RBF tx successfully", chain: chains.BitcoinMainnet, - cctx: cctx, lastTx: btcutil.NewTx(msgTx.Copy()), preTxs: preTxs, minRelayFee: 0.00001, - cctxRate: "57", - liveRate: 59, // 59 sat/vB + cctxRateStr: "76", // 57 sat/vB as tx rate + liveRate: 59, // 59 sat/vB memplTxsInfo: newMempoolTxsInfo( 1, // 1 stuck tx 0.00027213, // fees: 0.00027213 BTC @@ -115,30 +111,27 @@ func Test_SignRBFTx(t *testing.T) { { name: "should return error if latest fee rate is not available", chain: chains.BitcoinMainnet, - cctx: cctx, lastTx: btcutil.NewTx(msgTx.Copy()), minRelayFee: 0.00001, - cctxRate: "", + cctxRateStr: "", errMsg: "invalid fee rate", }, { name: "should return error if unable to create fee bumper", chain: chains.BitcoinMainnet, - cctx: cctx, lastTx: btcutil.NewTx(msgTx.Copy()), minRelayFee: 0.00001, - cctxRate: "57", + cctxRateStr: "76", memplTxsInfo: nil, errMsg: "NewCPFPFeeBumper failed", }, { name: "should return error if live rate is too high", chain: chains.BitcoinMainnet, - cctx: cctx, lastTx: btcutil.NewTx(msgTx.Copy()), minRelayFee: 0.00001, - cctxRate: "57", - liveRate: 99, // 99 sat/vB is much higher than ccxt rate + cctxRateStr: "76", // 57 sat/vB as tx rate + liveRate: 99, // 99 sat/vB is much higher than ccxt rate memplTxsInfo: newMempoolTxsInfo( 1, // 1 stuck tx 0.00027213, // fees: 0.00027213 BTC @@ -148,13 +141,13 @@ func Test_SignRBFTx(t *testing.T) { errMsg: "BumpTxFee failed", }, { - name: "should return error if live rate is too high", + name: "should return error if unable to get previous tx", chain: chains.BitcoinMainnet, - cctx: cctx, lastTx: btcutil.NewTx(msgTx.Copy()), + preTxs: nil, minRelayFee: 0.00001, - cctxRate: "57", - liveRate: 59, // 59 sat/vB + cctxRateStr: "76", // 57 sat/vB as tx rate + liveRate: 59, // 59 sat/vB memplTxsInfo: newMempoolTxsInfo( 1, // 1 stuck tx 0.00027213, // fees: 0.00027213 BTC @@ -171,9 +164,6 @@ func Test_SignRBFTx(t *testing.T) { // setup signer s := newTestSuite(t, tt.chain) - // mock cctx rate - tt.cctx.GetCurrentOutboundParam().GasPriorityFee = tt.cctxRate - // mock RPC live fee rate if tt.liveRate > 0 { s.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(tt.liveRate, nil) @@ -218,7 +208,7 @@ func Test_SignRBFTx(t *testing.T) { // ACT // sign tx ctx := context.Background() - newTx, err := s.SignRBFTx(ctx, tt.cctx, 1, tt.lastTx, tt.minRelayFee) + newTx, err := s.SignRBFTx(ctx, 1, nonce, tt.lastTx, tt.cctxRateStr, tt.minRelayFee) if tt.errMsg != "" { require.ErrorContains(t, err, tt.errMsg) return diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index b1aa18c532..9d28019a85 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -136,7 +136,7 @@ func (signer *Signer) TryProcessOutbound( if stuckTx != nil && params.TssNonce == stuckTx.Nonce { // sign RBF tx rbfTx = true - signedTx, err = signer.SignRBFTx(ctx, cctx, height, stuckTx.Tx, minRelayFee) + signedTx, err = signer.SignRBFTx(ctx, height, params.TssNonce, stuckTx.Tx, params.GasPriorityFee, minRelayFee) if err != nil { logger.Error().Err(err).Msg("SignRBFTx failed") return diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index a53504e2fb..834ca42b16 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -107,25 +107,25 @@ func Test_BroadcastOutbound(t *testing.T) { chain: chains.BitcoinMainnet, nonce: uint64(148), }, - { - name: "should successfully broadcast and include RBF outbound", - chain: chains.BitcoinMainnet, - nonce: uint64(148), - rbfTx: true, - }, - { - name: "should successfully broadcast and include outbound, but fail to post outbound tracker", - chain: chains.BitcoinMainnet, - nonce: uint64(148), - failTracker: true, - }, - { - name: "should skip broadcasting RBF tx", - chain: chains.BitcoinMainnet, - nonce: uint64(148), - rbfTx: true, - skipRBFTx: true, - }, + // { + // name: "should successfully broadcast and include RBF outbound", + // chain: chains.BitcoinMainnet, + // nonce: uint64(148), + // rbfTx: true, + // }, + // { + // name: "should successfully broadcast and include outbound, but fail to post outbound tracker", + // chain: chains.BitcoinMainnet, + // nonce: uint64(148), + // failTracker: true, + // }, + // { + // name: "should skip broadcasting RBF tx", + // chain: chains.BitcoinMainnet, + // nonce: uint64(148), + // rbfTx: true, + // skipRBFTx: true, + // }, } for _, tt := range tests { diff --git a/zetaclient/common/constant.go b/zetaclient/common/constant.go index 0c5a4bf1b5..b3020d7b0e 100644 --- a/zetaclient/common/constant.go +++ b/zetaclient/common/constant.go @@ -3,14 +3,17 @@ package common import "time" const ( - // DefaultGasPriceMultiplier is the default gas price multiplier for all chains - DefaultGasPriceMultiplier = 1.0 + // DefaultGasPriceMultiplierFeeCharge is the default gas price multiplier to charge fees from users + DefaultGasPriceMultiplierFeeCharge = 1.0 - // EVMOutboundGasPriceMultiplier is the default gas price multiplier for EVM-chain outbond txs - EVMOutboundGasPriceMultiplier = 1.2 + // EVMGasPriceMultiplierFeeCharge is the default gas price multiplier to charge fees from users + EVMGasPriceMultiplierFeeCharge = 1.2 - // BTCOutboundGasPriceMultiplier is the default gas price multiplier for BTC outbond txs - BTCOutboundGasPriceMultiplier = 2.0 + // BTCGasPriceMultiplierFeeCharge is the default gas price multiplier to charge fees from users + BTCGasPriceMultiplierFeeCharge = 2.0 + + // BTCGasPriceMultiplierSendTx is the default gas price multiplier to send out BTC TSS txs + BTCGasPriceMultiplierSendTx = 1.5 // RPCStatusCheckInterval is the interval to check RPC status, 1 minute RPCStatusCheckInterval = time.Minute diff --git a/zetaclient/zetacore/client_vote.go b/zetaclient/zetacore/client_vote.go index 491ec892e0..1350e15bb3 100644 --- a/zetaclient/zetacore/client_vote.go +++ b/zetaclient/zetacore/client_vote.go @@ -21,7 +21,7 @@ func (c *Client) PostVoteGasPrice( gasPrice uint64, priorityFee, blockNum uint64, ) (string, error) { // get gas price multiplier for the chain - multiplier := GasPriceMultiplier(chain) + multiplier := GasPriceMultiplierFeeCharge(chain) // #nosec G115 always in range gasPrice = uint64(float64(gasPrice) * multiplier) diff --git a/zetaclient/zetacore/tx.go b/zetaclient/zetacore/tx.go index 3d06feb202..7e5ab05124 100644 --- a/zetaclient/zetacore/tx.go +++ b/zetaclient/zetacore/tx.go @@ -57,15 +57,15 @@ func GetInboundVoteMessage( return msg } -// GasPriceMultiplier returns the gas price multiplier for the given chain -func GasPriceMultiplier(chain chains.Chain) float64 { +// GasPriceMultiplierFeeCharge returns the fee-charging gas price multiplier for the given chain +func GasPriceMultiplierFeeCharge(chain chains.Chain) float64 { switch chain.Consensus { case chains.Consensus_ethereum: - return clientcommon.EVMOutboundGasPriceMultiplier + return clientcommon.EVMGasPriceMultiplierFeeCharge case chains.Consensus_bitcoin: - return clientcommon.BTCOutboundGasPriceMultiplier + return clientcommon.BTCGasPriceMultiplierFeeCharge default: - return clientcommon.DefaultGasPriceMultiplier + return clientcommon.DefaultGasPriceMultiplierFeeCharge } } diff --git a/zetaclient/zetacore/tx_test.go b/zetaclient/zetacore/tx_test.go index 14bbb039ad..3b632079bf 100644 --- a/zetaclient/zetacore/tx_test.go +++ b/zetaclient/zetacore/tx_test.go @@ -25,7 +25,7 @@ const ( ethBlockHash = "1a17bcc359e84ba8ae03b17ec425f97022cd11c3e279f6bdf7a96fcffa12b366" ) -func Test_GasPriceMultiplier(t *testing.T) { +func Test_GasPriceMultiplierFeeCharge(t *testing.T) { tt := []struct { name string chain chains.Chain @@ -84,7 +84,7 @@ func Test_GasPriceMultiplier(t *testing.T) { } for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { - multiplier := GasPriceMultiplier(tc.chain) + multiplier := GasPriceMultiplierFeeCharge(tc.chain) require.Equal(t, tc.multiplier, multiplier) }) } From 209e27089565e50e88954670b303cfdcfa7439c0 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 15:27:58 -0600 Subject: [PATCH 18/74] disable RBF test by default; it require one-line code change --- cmd/zetae2e/local/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 820c1d0b7a..8105505078 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -317,7 +317,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { e2etests.TestBitcoinWithdrawP2WSHName, e2etests.TestBitcoinWithdrawMultipleName, e2etests.TestBitcoinWithdrawRestrictedName, - e2etests.TestBitcoinWithdrawRBFName, + //e2etests.TestBitcoinWithdrawRBFName, } if !light { From 3f5087b2b7a10b24e7ce5831dd65fcdf0f385216 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 16:20:56 -0600 Subject: [PATCH 19/74] include coin type to error message; make code cleaner --- zetaclient/chains/bitcoin/signer/outbound_data.go | 4 ++-- zetaclient/chains/bitcoin/signer/outbound_data_test.go | 4 ++-- zetaclient/chains/bitcoin/signer/sign.go | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index 7caecef55c..f036bd02a6 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -63,7 +63,7 @@ func NewOutboundData( // support gas token only for Bitcoin outbound if cctx.InboundParams.CoinType != coin.CoinType_Gas { - return nil, errors.New("can only send gas token to a Bitcoin network") + return nil, fmt.Errorf("invalid coin type %s", cctx.InboundParams.CoinType.String()) } // initial fee rate @@ -85,7 +85,7 @@ func NewOutboundData( return nil, errors.Wrapf(err, "cannot decode receiver address %s", params.Receiver) } if !chains.IsBtcAddressSupported(to) { - return nil, fmt.Errorf("unsupported receiver address %s", params.Receiver) + return nil, fmt.Errorf("unsupported receiver address %s", to.EncodeAddress()) } // amount in BTC and satoshis diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index 9ec6991178..023146ca96 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -103,13 +103,13 @@ func Test_NewOutboundData(t *testing.T) { errMsg: "cctx is nil", }, { - name: "coin type is not gas", + name: "invalid coin types", cctx: sample.CrossChainTx(t, "0x123"), cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { cctx.InboundParams.CoinType = coin.CoinType_ERC20 }, expected: nil, - errMsg: "can only send gas token to a Bitcoin network", + errMsg: "invalid coin type", }, { name: "invalid gas price", diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index 97f04b3c9b..b405b85db0 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -93,14 +93,12 @@ func (signer *Signer) SignWithdrawTx( // add tx outputs inputValue := selected.Value - err = signer.AddWithdrawTxOutputs(tx, txData.to, inputValue, txData.amountSats, nonceMark, fees, txData.cancelTx) - if err != nil { + if err := signer.AddWithdrawTxOutputs(tx, txData.to, inputValue, txData.amountSats, nonceMark, fees, txData.cancelTx); err != nil { return nil, err } // sign the tx - err = signer.SignTx(ctx, tx, inAmounts, txData.height, txData.nonce) - if err != nil { + if err := signer.SignTx(ctx, tx, inAmounts, txData.height, txData.nonce); err != nil { return nil, errors.Wrap(err, "SignTx failed") } From c31020b2b76d609d545347f9464597bf39982b00 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 16:51:28 -0600 Subject: [PATCH 20/74] return error if failed to get signer address; add BTCPayToAddrScript as a method of TSS PubKey --- zetaclient/chains/bitcoin/signer/sign.go | 4 ++-- zetaclient/chains/bitcoin/signer/signer.go | 15 +++------------ zetaclient/tss/crypto.go | 12 +++++++++++- zetaclient/tss/crypto_test.go | 7 +++++++ 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index b405b85db0..91fdc75523 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -161,7 +161,7 @@ func (signer *Signer) AddWithdrawTxOutputs( } // 1st output: the nonce-mark btc to TSS self - payToSelfScript, err := signer.TSSToPkScript() + payToSelfScript, err := signer.TSS().PubKey().BTCPayToAddrScript(signer.Chain().ChainId) if err != nil { return err } @@ -197,7 +197,7 @@ func (signer *Signer) SignTx( height uint64, nonce uint64, ) error { - pkScript, err := signer.TSSToPkScript() + pkScript, err := signer.TSS().PubKey().BTCPayToAddrScript(signer.Chain().ChainId) if err != nil { return err } diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index f09b587ae2..f9448bf741 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -10,7 +10,6 @@ import ( "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" "github.com/pkg/errors" @@ -68,15 +67,6 @@ func (signer *Signer) Broadcast(ctx context.Context, signedTx *wire.MsgTx) error return nil } -// TSSToPkScript returns the TSS pkScript -func (signer *Signer) TSSToPkScript() ([]byte, error) { - tssAddrP2WPKH, err := signer.TSS().PubKey().AddressBTC(signer.Chain().ChainId) - if err != nil { - return nil, err - } - return txscript.PayToAddrScript(tssAddrP2WPKH) -} - // TryProcessOutbound signs and broadcasts a BTC transaction from a new outbound func (signer *Signer) TryProcessOutbound( ctx context.Context, @@ -104,9 +94,10 @@ func (signer *Signer) TryProcessOutbound( logs.FieldNonce: params.TssNonce, } signerAddress, err := zetacoreClient.GetKeys().GetAddress() - if err == nil { - lf["signer"] = signerAddress.String() + if err != nil { + return } + lf["signer"] = signerAddress.String() logger := signer.Logger().Std.With().Fields(lf).Logger() // query network info to get minRelayFee (typically 1000 satoshis) diff --git a/zetaclient/tss/crypto.go b/zetaclient/tss/crypto.go index 170f433657..731b4344e3 100644 --- a/zetaclient/tss/crypto.go +++ b/zetaclient/tss/crypto.go @@ -11,6 +11,7 @@ import ( "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" eth "github.com/ethereum/go-ethereum/common" @@ -111,11 +112,20 @@ func (k PubKey) Bech32String() string { return v } -// AddressBTC returns the bitcoin address of the public key. +// AddressBTC returns the Bitcoin address of the public key. func (k PubKey) AddressBTC(chainID int64) (*btcutil.AddressWitnessPubKeyHash, error) { return bitcoinP2WPKH(k.Bytes(true), chainID) } +// BTCPayToAddrScript returns the script for the Bitcoin TSS address. +func (k PubKey) BTCPayToAddrScript(chainID int64) ([]byte, error) { + tssAddrP2WPKH, err := k.AddressBTC(chainID) + if err != nil { + return nil, err + } + return txscript.PayToAddrScript(tssAddrP2WPKH) +} + // AddressEVM returns the ethereum address of the public key. func (k PubKey) AddressEVM() eth.Address { return crypto.PubkeyToAddress(*k.ecdsaPubKey) diff --git a/zetaclient/tss/crypto_test.go b/zetaclient/tss/crypto_test.go index b34e0dc21e..287a39947b 100644 --- a/zetaclient/tss/crypto_test.go +++ b/zetaclient/tss/crypto_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/btcsuite/btcd/txscript" "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -46,6 +47,12 @@ func TestPubKey(t *testing.T) { addrBTC, err := pk.AddressBTC(chains.BitcoinMainnet.ChainId) require.NoError(t, err) + expectedPkScript, err := txscript.PayToAddrScript(addrBTC) + require.NoError(t, err) + pkScript, err := pk.BTCPayToAddrScript(chains.BitcoinMainnet.ChainId) + require.NoError(t, err) + assert.Equal(t, expectedPkScript, pkScript) + assert.Equal(t, sample, pk.Bech32String()) assert.Equal(t, "0x70e967acfcc17c3941e87562161406d41676fd83", strings.ToLower(addrEVM.Hex())) assert.Equal(t, "bc1qm24wp577nk8aacckv8np465z3dvmu7ry45el6y", addrBTC.String()) From 58fd3401fcc5b7130d38f421b3b39c9fa515adab Mon Sep 17 00:00:00 2001 From: Charlie Chen <34498985+ws4charlie@users.noreply.github.com> Date: Tue, 28 Jan 2025 17:02:43 -0600 Subject: [PATCH 21/74] Update zetaclient/chains/bitcoin/signer/signer.go Replace 'Msgf' with 'Msg' Co-authored-by: Dmitry S <11892559+swift1337@users.noreply.github.com> --- zetaclient/chains/bitcoin/signer/signer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index f9448bf741..d2203ae9fd 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -103,7 +103,7 @@ func (signer *Signer) TryProcessOutbound( // query network info to get minRelayFee (typically 1000 satoshis) networkInfo, err := signer.rpc.GetNetworkInfo(ctx) if err != nil { - logger.Error().Err(err).Msgf("failed get bitcoin network info") + logger.Error().Err(err).Msg("failed get bitcoin network info") return } minRelayFee := networkInfo.RelayFee From 0f0b9aa88b50a9d6efc6c811647608305d423204 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 17:20:31 -0600 Subject: [PATCH 22/74] remove redundant test functions; use testlog for unit test --- zetaclient/chains/bitcoin/signer/signer.go | 2 +- .../chains/bitcoin/signer/signer_test.go | 35 +++---------------- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index d2203ae9fd..e276f430f6 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -144,7 +144,7 @@ func (signer *Signer) BroadcastOutbound( // prepare logger fields lf := map[string]any{ - logs.FieldMethod: "broadcastOutbound", + logs.FieldMethod: "BroadcastOutbound", logs.FieldNonce: nonce, logs.FieldTx: txHash, logs.FieldCctx: cctx.Index, diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index 72db8b38b9..b767511dd0 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -13,13 +13,11 @@ import ( "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" - "github.com/ethereum/go-ethereum/crypto" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/pkg/chains" - crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" @@ -29,9 +27,9 @@ import ( "github.com/zeta-chain/node/zetaclient/db" "github.com/zeta-chain/node/zetaclient/keys" "github.com/zeta-chain/node/zetaclient/metrics" - "github.com/zeta-chain/node/zetaclient/outboundprocessor" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/zetaclient/testutils/mocks" + "github.com/zeta-chain/node/zetaclient/testutils/testlog" ) // the relative path to the testdata directory @@ -63,11 +61,11 @@ func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { WithZetaChain() // create logger - testLogger := zerolog.New(zerolog.NewTestWriter(t)) - logger := base.Logger{Std: testLogger, Compliance: testLogger} + logger := testlog.New(t) + baseLogger := base.Logger{Std: logger.Logger, Compliance: logger.Logger} // create signer - baseSigner := base.NewSigner(chain, tss, logger) + baseSigner := base.NewSigner(chain, tss, baseLogger) signer := signer.New(baseSigner, rpcClient) return &testSuite{ @@ -78,20 +76,6 @@ func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { } } -func Test_NewSigner(t *testing.T) { - // test private key with EVM address - // EVM: 0x236C7f53a90493Bb423411fe4117Cb4c2De71DfB - // BTC testnet: muGe9prUBjQwEnX19zG26fVRHNi8z7kSPo - skHex := "7b8507ba117e069f4a3f456f505276084f8c92aee86ac78ae37b4d1801d35fa8" - privateKey, err := crypto.HexToECDSA(skHex) - require.NoError(t, err) - tss := mocks.NewTSSFromPrivateKey(t, privateKey) - - baseSigner := base.NewSigner(chains.BitcoinMainnet, tss, base.DefaultLogger()) - signer := signer.New(baseSigner, mocks.NewBitcoinClient(t)) - require.NotNil(t, signer) -} - func Test_BroadcastOutbound(t *testing.T) { // test cases tests := []struct { @@ -346,17 +330,6 @@ func makeCtx(t *testing.T) context.Context { return zctx.WithAppContext(context.Background(), app) } -// getCCTX returns a CCTX for testing -func getCCTX(t *testing.T) *crosschaintypes.CrossChainTx { - return testutils.LoadCctxByNonce(t, 8332, 148) -} - -// getNewOutboundProcessor creates a new outbound processor for testing -func getNewOutboundProcessor() *outboundprocessor.Processor { - logger := zerolog.Logger{} - return outboundprocessor.NewProcessor(logger) -} - // getNewObserver creates a new BTC chain observer for testing func (s *testSuite) getNewObserver(t *testing.T) *observer.Observer { // prepare mock arguments to create observer From 54bb4dea1b0da23e47b688c117088ff5d7bbd54f Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 17:34:50 -0600 Subject: [PATCH 23/74] create observer in test suite; use testlog package --- .../chains/bitcoin/signer/signer_test.go | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index b767511dd0..dcc30644f6 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -37,6 +37,7 @@ var TestDataDir = "../../../" type testSuite struct { *signer.Signer + observer *observer.Observer tss *mocks.TSS client *mocks.BitcoinClient zetacoreClient *mocks.ZetacoreClient @@ -68,12 +69,16 @@ func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { baseSigner := base.NewSigner(chain, tss, baseLogger) signer := signer.New(baseSigner, rpcClient) - return &testSuite{ + // create test suite and observer + suite := &testSuite{ Signer: signer, tss: tss, client: rpcClient, zetacoreClient: zetacoreClient, } + suite.createObserver(t) + + return suite } func Test_BroadcastOutbound(t *testing.T) { @@ -101,7 +106,6 @@ func Test_BroadcastOutbound(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // setup signer and observer s := newTestSuite(t, tt.chain) - observer := s.getNewObserver(t) // load tx and result chainID := tt.chain.ChainId @@ -126,7 +130,7 @@ func Test_BroadcastOutbound(t *testing.T) { // mock the previous tx as included // this is necessary to allow the 'checkTSSVin' function to pass - observer.SetIncludedTx(tt.nonce-1, &btcjson.GetTransactionResult{ + s.observer.SetIncludedTx(tt.nonce-1, &btcjson.GetTransactionResult{ TxID: rawResult.Vin[0].Txid, }) @@ -136,12 +140,12 @@ func Test_BroadcastOutbound(t *testing.T) { msgTx, tt.nonce, cctx, - observer, + s.observer, s.zetacoreClient, ) // check if outbound is included - gotResult := observer.GetIncludedTx(tt.nonce) + gotResult := s.observer.GetIncludedTx(tt.nonce) require.Equal(t, txResult, gotResult) }) } @@ -330,8 +334,8 @@ func makeCtx(t *testing.T) context.Context { return zctx.WithAppContext(context.Background(), app) } -// getNewObserver creates a new BTC chain observer for testing -func (s *testSuite) getNewObserver(t *testing.T) *observer.Observer { +// createObserver creates a new BTC chain observer for test suite +func (s *testSuite) createObserver(t *testing.T) { // prepare mock arguments to create observer params := mocks.MockChainParams(s.Chain().ChainId, 2) ts := &metrics.TelemetryServer{} @@ -341,16 +345,15 @@ func (s *testSuite) getNewObserver(t *testing.T) *observer.Observer { require.NoError(t, err) // create logger - testLogger := zerolog.New(zerolog.NewTestWriter(t)) - logger := base.Logger{Std: testLogger, Compliance: testLogger} + logger := testlog.New(t) + baseLogger := base.Logger{Std: logger.Logger, Compliance: logger.Logger} // create observer - baseObserver, err := base.NewObserver(s.Chain(), params, s.zetacoreClient, s.tss, 100, ts, database, logger) + baseObserver, err := base.NewObserver(s.Chain(), params, s.zetacoreClient, s.tss, 100, ts, database, baseLogger) require.NoError(t, err) - ob, err := observer.New(s.Chain(), baseObserver, s.client) + s.observer, err = observer.New(s.Chain(), baseObserver, s.client) require.NoError(t, err) - return ob } func hashFromTXID(t *testing.T, txid string) *chainhash.Hash { From 93b3a38e367b17a7ae8c1f6cf2eaf4c23cbaaf19 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 17:44:03 -0600 Subject: [PATCH 24/74] add description to fee estimation formula --- zetaclient/chains/bitcoin/signer/sign.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index 91fdc75523..3d978c95f1 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -34,6 +34,10 @@ func (signer *Signer) SignWithdrawTx( ob *observer.Observer, ) (*wire.MsgTx, error) { nonceMark := chains.NonceMarkAmount(txData.nonce) + + // we don't know how many UTXOs will be used beforehand, so we do + // a conservative estimation using the maximum size of the outbound tx: + // estimateFee = feeRate * maxTxSize estimateFee := float64(txData.feeRate*common.OutboundBytesMax) / 1e8 totalAmount := txData.amount + estimateFee + float64(nonceMark)*1e-8 From 4f83fdb8a296f5ff87499f8b9d266b5a64052085 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 22:08:14 -0600 Subject: [PATCH 25/74] use structured logs --- zetaclient/chains/bitcoin/signer/sign.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index 3d978c95f1..9e74fa0e54 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -17,6 +17,7 @@ import ( "github.com/zeta-chain/node/pkg/constant" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" + "github.com/zeta-chain/node/zetaclient/logs" ) const ( @@ -72,18 +73,17 @@ func (signer *Signer) SignWithdrawTx( if err != nil { return nil, err } - if txData.txSize < common.BtcOutboundBytesWithdrawer { // ZRC20 'withdraw' charged less fee from end user - signer.Logger().Std.Info(). - Msgf("txSize %d is less than BtcOutboundBytesWithdrawer %d for nonce %d", txData.txSize, txSize, txData.nonce) - } + logger := signer.Logger().Std.With(). + Int64("txData.txSize", txData.txSize). + Int64("tx.size", txSize). + Uint64(logs.FieldNonce, txData.nonce). + Logger() if txSize < common.OutboundBytesMin { // outbound shouldn't be blocked by low sizeLimit - signer.Logger().Std.Warn(). - Msgf("txSize %d is less than outboundBytesMin %d; use outboundBytesMin", txSize, common.OutboundBytesMin) + logger.Warn().Msg("txSize is less than outboundBytesMin") txSize = common.OutboundBytesMin } if txSize > common.OutboundBytesMax { // in case of accident - signer.Logger().Std.Warn(). - Msgf("txSize %d is greater than outboundBytesMax %d; use outboundBytesMax", txSize, common.OutboundBytesMax) + logger.Warn().Msgf("txSize is greater than outboundBytesMax") txSize = common.OutboundBytesMax } From 6fd51dd81e84e1851a5bba6712a8fcac074cbcb6 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 22:13:24 -0600 Subject: [PATCH 26/74] make AddTxInputs independent method --- zetaclient/chains/bitcoin/signer/sign.go | 4 ++-- zetaclient/chains/bitcoin/signer/sign_test.go | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index 9e74fa0e54..91d446451e 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -62,7 +62,7 @@ func (signer *Signer) SignWithdrawTx( // build tx and add inputs tx := wire.NewMsgTx(wire.TxVersion) - inAmounts, err := signer.AddTxInputs(tx, selected.UTXOs) + inAmounts, err := AddTxInputs(tx, selected.UTXOs) if err != nil { return nil, err } @@ -110,7 +110,7 @@ func (signer *Signer) SignWithdrawTx( } // AddTxInputs adds the inputs to the tx and returns input amounts -func (signer *Signer) AddTxInputs(tx *wire.MsgTx, utxos []btcjson.ListUnspentResult) ([]int64, error) { +func AddTxInputs(tx *wire.MsgTx, utxos []btcjson.ListUnspentResult) ([]int64, error) { amounts := make([]int64, len(utxos)) for i, utxo := range utxos { hash, err := chainhash.NewHashFromStr(utxo.TxID) diff --git a/zetaclient/chains/bitcoin/signer/sign_test.go b/zetaclient/chains/bitcoin/signer/sign_test.go index ba0aa8631c..5c0a0f46bf 100644 --- a/zetaclient/chains/bitcoin/signer/sign_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_test.go @@ -72,12 +72,9 @@ func Test_AddTxInputs(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // setup signer - s := newTestSuite(t, chains.BitcoinMainnet) - // create tx msg and add inputs tx := wire.NewMsgTx(wire.TxVersion) - inAmounts, err := s.AddTxInputs(tx, tt.utxos) + inAmounts, err := signer.AddTxInputs(tx, tt.utxos) // assert if tt.fail { @@ -306,7 +303,7 @@ func Test_SignTx(t *testing.T) { Amount: amount, }) } - inAmounts, err := s.AddTxInputs(tx, utxos) + inAmounts, err := signer.AddTxInputs(tx, utxos) require.NoError(t, err) require.Len(t, inAmounts, len(tt.inputs)) From a69d58b23868b5f67f80a8d000c3d578f9211edc Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 22:27:17 -0600 Subject: [PATCH 27/74] add comments to explain function arguments; improve error wrapping; code simplification --- zetaclient/chains/bitcoin/signer/sign.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index 91d446451e..ca285340a7 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -137,6 +137,9 @@ func AddTxInputs(tx *wire.MsgTx, utxos []btcjson.ListUnspentResult) ([]int64, er // 1st output: the nonce-mark btc to TSS itself // 2nd output: the payment to the recipient // 3rd output: the remaining btc to TSS itself +// +// Note: float64 is used for for 'inputValue' because UTXOs struct uses float64. +// But we need to use 'int64' for the outputs because NewTxOut expects int64. func (signer *Signer) AddWithdrawTxOutputs( tx *wire.MsgTx, to btcutil.Address, @@ -220,9 +223,12 @@ func (signer *Signer) SignTx( // sign the tx with TSS sig65Bs, err := signer.TSS().SignBatch(ctx, witnessHashes, height, nonce, signer.Chain().ChainId) if err != nil { - return fmt.Errorf("SignBatch failed: %v", err) + return errors.Wrap(err, "SignBatch failed") } + // add witnesses to the tx + pkCompressed := signer.TSS().PubKey().Bytes(true) + hashType := txscript.SigHashAll for ix := range tx.TxIn { sig65B := sig65Bs[ix] R := &btcec.ModNScalar{} @@ -231,8 +237,6 @@ func (signer *Signer) SignTx( S.SetBytes((*[32]byte)(sig65B[32:64])) sig := btcecdsa.NewSignature(R, S) - pkCompressed := signer.TSS().PubKey().Bytes(true) - hashType := txscript.SigHashAll txWitness := wire.TxWitness{append(sig.Serialize(), byte(hashType)), pkCompressed} tx.TxIn[ix].Witness = txWitness } From 5c09d396c6b4417c11df05da7a883d88461747dc Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 22:48:58 -0600 Subject: [PATCH 28/74] replace ifs with switch case; return original err without overwriting --- zetaclient/chains/bitcoin/client/helpers.go | 12 ++++++------ zetaclient/chains/bitcoin/observer/utxos.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 395c8ccd4f..7ea16c1f2c 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -114,7 +114,7 @@ func (c *Client) GetRawTransactionResult(ctx context.Context, } } -// GetEstimatedFeeRate gets estimated smart fee rate (BTC/Kb) targeting given block confirmation +// GetEstimatedFeeRate gets estimated smart fee rate (sat/vB) targeting given block confirmation func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) { // RPC 'EstimateSmartFee' is not available in regnet if regnet { @@ -122,15 +122,15 @@ func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regn } feeResult, err := c.EstimateSmartFee(ctx, confTarget, &types.EstimateModeEconomical) - if err != nil { + switch { + case err != nil: return 0, errors.Wrap(err, "unable to estimate smart fee") - } - if feeResult.Errors != nil { + case feeResult.Errors != nil: return 0, fmt.Errorf("fee result contains errors: %s", feeResult.Errors) - } - if feeResult.FeeRate == nil { + case feeResult.FeeRate == nil: return 0, errors.New("nil fee rate") } + feeRate := *feeResult.FeeRate if feeRate <= 0 || feeRate >= maxBTCSupply { return 0, fmt.Errorf("invalid fee rate: %f", feeRate) diff --git a/zetaclient/chains/bitcoin/observer/utxos.go b/zetaclient/chains/bitcoin/observer/utxos.go index 3413d62511..2f5b884941 100644 --- a/zetaclient/chains/bitcoin/observer/utxos.go +++ b/zetaclient/chains/bitcoin/observer/utxos.go @@ -41,7 +41,7 @@ func (ob *Observer) FetchUTXOs(ctx context.Context) error { // list all unspent UTXOs (160ms) tssAddr, err := ob.TSS().PubKey().AddressBTC(ob.Chain().ChainId) if err != nil { - return fmt.Errorf("error getting bitcoin tss address") + return err } utxos, err := ob.rpc.ListUnspentMinMaxAddresses(ctx, 0, 9999999, []btcutil.Address{tssAddr}) if err != nil { From c19fa2bf32921459cde28b16229c7cee4a5f6804 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 22:54:52 -0600 Subject: [PATCH 29/74] seems safe to remove panic recovery in FetchUTXOs --- zetaclient/chains/bitcoin/observer/utxos.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/utxos.go b/zetaclient/chains/bitcoin/observer/utxos.go index 2f5b884941..0bf1d909aa 100644 --- a/zetaclient/chains/bitcoin/observer/utxos.go +++ b/zetaclient/chains/bitcoin/observer/utxos.go @@ -29,12 +29,6 @@ type SelectedUTXOs struct { // FetchUTXOs fetches TSS-owned UTXOs from the Bitcoin node func (ob *Observer) FetchUTXOs(ctx context.Context) error { - defer func() { - if err := recover(); err != nil { - ob.logger.UTXOs.Error().Msgf("BTC FetchUTXOs: caught panic error: %v", err) - } - }() - // this is useful when a zetaclient's pending nonce lagged behind for whatever reason. ob.refreshPendingNonce(ctx) From 0d41413feb89dfffd4170ad334798edee444babc Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 28 Jan 2025 22:57:31 -0600 Subject: [PATCH 30/74] move Telemetry update to the line before acquiring observer lock --- zetaclient/chains/bitcoin/observer/utxos.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zetaclient/chains/bitcoin/observer/utxos.go b/zetaclient/chains/bitcoin/observer/utxos.go index 0bf1d909aa..c3f86673e2 100644 --- a/zetaclient/chains/bitcoin/observer/utxos.go +++ b/zetaclient/chains/bitcoin/observer/utxos.go @@ -69,8 +69,8 @@ func (ob *Observer) FetchUTXOs(ctx context.Context) error { utxosFiltered = append(utxosFiltered, utxo) } - ob.Mu().Lock() ob.TelemetryServer().SetNumberOfUTXOs(len(utxosFiltered)) + ob.Mu().Lock() ob.utxos = utxosFiltered ob.Mu().Unlock() return nil From c0a2a235b80e47633eff678be180078815bb30d0 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 29 Jan 2025 00:36:07 -0600 Subject: [PATCH 31/74] use testlog package --- zetaclient/chains/bitcoin/observer/observer_test.go | 8 ++++---- zetaclient/chains/solana/observer/outbound_test.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index e00a6b4355..6a6937a2ef 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -9,7 +9,6 @@ import ( "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/wire" - "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/zetaclient/db" @@ -24,6 +23,7 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/metrics" "github.com/zeta-chain/node/zetaclient/testutils/mocks" + "github.com/zeta-chain/node/zetaclient/testutils/testlog" clienttypes "github.com/zeta-chain/node/zetaclient/types" ) @@ -299,8 +299,8 @@ func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { require.NoError(t, err) // create logger - testLogger := zerolog.New(zerolog.NewTestWriter(t)) - logger := base.Logger{Std: testLogger, Compliance: testLogger} + logger := testlog.New(t) + baseLogger := base.Logger{Std: logger.Logger, Compliance: logger.Logger} baseObserver, err := base.NewObserver( chain, @@ -310,7 +310,7 @@ func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { 100, nil, database, - logger, + baseLogger, ) require.NoError(t, err) diff --git a/zetaclient/chains/solana/observer/outbound_test.go b/zetaclient/chains/solana/observer/outbound_test.go index ee51a81db5..67ee3b5e63 100644 --- a/zetaclient/chains/solana/observer/outbound_test.go +++ b/zetaclient/chains/solana/observer/outbound_test.go @@ -7,7 +7,6 @@ import ( "github.com/gagliardetto/solana-go" "github.com/pkg/errors" - "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/pkg/chains" @@ -20,6 +19,7 @@ import ( "github.com/zeta-chain/node/zetaclient/db" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/zetaclient/testutils/mocks" + "github.com/zeta-chain/node/zetaclient/testutils/testlog" ) const ( @@ -55,13 +55,13 @@ func createTestObserver( database, err := db.NewFromSqliteInMemory(true) require.NoError(t, err) - testLogger := zerolog.New(zerolog.NewTestWriter(t)) - logger := base.Logger{Std: testLogger, Compliance: testLogger} + logger := testlog.New(t) + baseLogger := base.Logger{Std: logger.Logger, Compliance: logger.Logger} // create observer chainParams := sample.ChainParams(chain.ChainId) chainParams.GatewayAddress = GatewayAddressTest - ob, err := observer.NewObserver(chain, solClient, *chainParams, nil, tss, database, logger, nil) + ob, err := observer.NewObserver(chain, solClient, *chainParams, nil, tss, database, baseLogger, nil) require.NoError(t, err) return ob From 8644a335bfc3945a20d755fad7b89ebd095e13e1 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 29 Jan 2025 15:12:15 -0600 Subject: [PATCH 32/74] use retry package for Bitcoin tx broadcasting; let SaveBroadcastedTx return error --- zetaclient/chains/bitcoin/observer/db.go | 14 ++-- zetaclient/chains/bitcoin/observer/db_test.go | 65 ++++++++++++----- zetaclient/chains/bitcoin/signer/signer.go | 69 +++++++++---------- 3 files changed, 89 insertions(+), 59 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/db.go b/zetaclient/chains/bitcoin/observer/db.go index b36a58d4a4..45360fe3f5 100644 --- a/zetaclient/chains/bitcoin/observer/db.go +++ b/zetaclient/chains/bitcoin/observer/db.go @@ -6,11 +6,12 @@ import ( "github.com/pkg/errors" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/zetaclient/logs" clienttypes "github.com/zeta-chain/node/zetaclient/types" ) // SaveBroadcastedTx saves successfully broadcasted transaction -func (ob *Observer) SaveBroadcastedTx(txHash string, nonce uint64) { +func (ob *Observer) SaveBroadcastedTx(txHash string, nonce uint64) error { outboundID := ob.OutboundID(nonce) ob.Mu().Lock() ob.tssOutboundHashes[txHash] = true @@ -19,11 +20,14 @@ func (ob *Observer) SaveBroadcastedTx(txHash string, nonce uint64) { broadcastEntry := clienttypes.ToOutboundHashSQLType(txHash, outboundID) if err := ob.DB().Client().Save(&broadcastEntry).Error; err != nil { - ob.logger.Outbound.Error(). - Err(err). - Msgf("SaveBroadcastedTx: error saving broadcasted txHash %s for outbound %s", txHash, outboundID) + return errors.Wrapf(err, "failed to save broadcasted outbound hash %s for %s", txHash, outboundID) } - ob.logger.Outbound.Info().Msgf("SaveBroadcastedTx: saved broadcasted txHash %s for outbound %s", txHash, outboundID) + ob.logger.Outbound.Info(). + Str(logs.FieldTx, txHash). + Str(logs.FieldOutboundID, outboundID). + Msg("saved broadcasted outbound hash to db") + + return nil } // LoadLastBlockScanned loads the last scanned block from the database diff --git a/zetaclient/chains/bitcoin/observer/db_test.go b/zetaclient/chains/bitcoin/observer/db_test.go index 1a126b7d68..948aed5826 100644 --- a/zetaclient/chains/bitcoin/observer/db_test.go +++ b/zetaclient/chains/bitcoin/observer/db_test.go @@ -15,25 +15,52 @@ import ( ) func Test_SaveBroadcastedTx(t *testing.T) { - t.Run("should be able to save broadcasted tx", func(t *testing.T) { - // test data - nonce := uint64(1) - txHash := sample.BtcHash().String() - - // create observer and open db - ob := newTestSuite(t, chains.BitcoinMainnet, "") - - // save a test tx - ob.SaveBroadcastedTx(txHash, nonce) - - // check if the txHash is a TSS outbound - require.True(t, ob.IsTSSTransaction(txHash)) - - // get the broadcasted tx - gotHash, found := ob.GetBroadcastedTx(nonce) - require.True(t, found) - require.Equal(t, txHash, gotHash) - }) + tests := []struct { + name string + wantErr string + }{ + { + name: "should be able to save broadcasted tx", + wantErr: "", + }, + { + name: "should fail on db error", + wantErr: "failed to save broadcasted outbound hash", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + // ARRANGE + // test data + nonce := uint64(1) + txHash := sample.BtcHash().String() + dbPath := sample.CreateTempDir(t) + ob := newTestSuite(t, chains.BitcoinMainnet, dbPath) + if tt.wantErr != "" { + // delete db to simulate db error + os.RemoveAll(dbPath) + } + + // ACT + // save a test tx + err := ob.SaveBroadcastedTx(txHash, nonce) + + // ASSERT + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + + // should always save broadcasted outbound to memory + gotHash, found := ob.GetBroadcastedTx(nonce) + require.True(t, found) + require.Equal(t, txHash, gotHash) + require.True(t, ob.IsTSSTransaction(txHash)) + }) + } } func Test_LoadLastBlockScanned(t *testing.T) { diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index e276f430f6..d242a34c88 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -11,8 +11,10 @@ import ( "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" + "github.com/cenkalti/backoff/v4" "github.com/pkg/errors" + "github.com/zeta-chain/node/pkg/retry" "github.com/zeta-chain/node/x/crosschain/types" "github.com/zeta-chain/node/zetaclient/chains/base" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" @@ -22,11 +24,11 @@ import ( ) const ( - // broadcastBackoff is the initial backoff duration for retrying broadcast - broadcastBackoff = 1000 * time.Millisecond + // broadcastBackoff is the backoff duration for retrying broadcast + broadcastBackoff = time.Second * 6 // broadcastRetries is the maximum number of retries for broadcasting a transaction - broadcastRetries = 5 + broadcastRetries = 10 ) type RPC interface { @@ -151,39 +153,36 @@ func (signer *Signer) BroadcastOutbound( } logger := signer.Logger().Std - // try broacasting tx with increasing backoff (1s, 2s, 4s, 8s, 16s) in case of RPC error - backOff := broadcastBackoff - for i := 0; i < broadcastRetries; i++ { - time.Sleep(backOff) - - // broadcast tx - err := signer.Broadcast(ctx, tx) - if err != nil { - logger.Warn().Err(err).Fields(lf).Msgf("broadcasting Bitcoin outbound, retry %d", i) - backOff *= 2 - continue - } - logger.Info().Fields(lf).Msg("broadcasted Bitcoin outbound successfully") - - // save tx local db - ob.SaveBroadcastedTx(txHash, nonce) - - // add tx to outbound tracker so that all observers know about it - zetaHash, err := zetacoreClient.PostOutboundTracker(ctx, ob.Chain().ChainId, nonce, txHash) - if err != nil { - logger.Err(err).Fields(lf).Msg("unable to add Bitcoin outbound tracker") - } else { - lf[logs.FieldZetaTx] = zetaHash - logger.Info().Fields(lf).Msg("add Bitcoin outbound tracker successfully") - } + // try broacasting tx with backoff in case of RPC error + broadcast := func() error { + return retry.Retry(signer.Broadcast(ctx, tx)) + } - // try including this outbound as early as possible - _, included := ob.TryIncludeOutbound(ctx, cctx, txHash) - if included { - logger.Info().Fields(lf).Msg("included newly broadcasted Bitcoin outbound") - } + bo := backoff.NewConstantBackOff(broadcastBackoff) + boWithMaxRetries := backoff.WithMaxRetries(bo, broadcastRetries) + if err := retry.DoWithBackoff(broadcast, boWithMaxRetries); err != nil { + logger.Error().Err(err).Fields(lf).Msgf("unable to broadcast Bitcoin outbound") + } + logger.Info().Fields(lf).Msg("broadcasted Bitcoin outbound successfully") + + // save tx local db and ignore db error. + // db error is not critical and should not block outbound tracker. + if err := ob.SaveBroadcastedTx(txHash, nonce); err != nil { + logger.Error().Err(err).Fields(lf).Msg("unable to save broadcasted Bitcoin outbound") + } + + // add tx to outbound tracker so that all observers know about it + zetaHash, err := zetacoreClient.PostOutboundTracker(ctx, ob.Chain().ChainId, nonce, txHash) + if err != nil { + logger.Err(err).Fields(lf).Msg("unable to add Bitcoin outbound tracker") + } else { + lf[logs.FieldZetaTx] = zetaHash + logger.Info().Fields(lf).Msg("add Bitcoin outbound tracker successfully") + } - // successful broadcast; no need to retry - break + // try including this outbound as early as possible, no need to wait for outbound tracker + _, included := ob.TryIncludeOutbound(ctx, cctx, txHash) + if included { + logger.Info().Fields(lf).Msg("included newly broadcasted Bitcoin outbound") } } From bf9b4c4d6ac9ef79d0326edd2099f6d46d001f2a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 29 Jan 2025 16:10:19 -0600 Subject: [PATCH 33/74] use named return values to make GetEstimatedFeeRate more readable --- zetaclient/chains/bitcoin/client/helpers.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 7ea16c1f2c..adb079486b 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -115,7 +115,11 @@ func (c *Client) GetRawTransactionResult(ctx context.Context, } // GetEstimatedFeeRate gets estimated smart fee rate (sat/vB) targeting given block confirmation -func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) { +func (c *Client) GetEstimatedFeeRate( + ctx context.Context, + confTarget int64, + regnet bool, +) (satsPerByte int64, err error) { // RPC 'EstimateSmartFee' is not available in regnet if regnet { return FeeRateRegnet, nil From 96004424f3c5afa70a94f09cc84d7d396c7c7f1b Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 29 Jan 2025 23:11:26 -0600 Subject: [PATCH 34/74] move utxo unit tests to utxos.go and improved unit tests --- .../chains/bitcoin/observer/observer_test.go | 2 +- .../chains/bitcoin/observer/outbound_test.go | 260 --------------- .../chains/bitcoin/observer/utxos_test.go | 304 ++++++++++++++++++ 3 files changed, 305 insertions(+), 261 deletions(-) create mode 100644 zetaclient/chains/bitcoin/observer/utxos_test.go diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index 6a6937a2ef..8862d12c67 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -308,7 +308,7 @@ func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { zetacore, tss, 100, - nil, + &metrics.TelemetryServer{}, database, baseLogger, ) diff --git a/zetaclient/chains/bitcoin/observer/outbound_test.go b/zetaclient/chains/bitcoin/observer/outbound_test.go index 94425ae9b9..3ec63b952f 100644 --- a/zetaclient/chains/bitcoin/observer/outbound_test.go +++ b/zetaclient/chains/bitcoin/observer/outbound_test.go @@ -1,9 +1,6 @@ package observer import ( - "context" - "math" - "sort" "testing" "github.com/btcsuite/btcd/btcjson" @@ -66,46 +63,6 @@ func createObserverWithPrivateKey(t *testing.T) *Observer { return ob } -// helper function to create a test Bitcoin observer with UTXOs -func createObserverWithUTXOs(t *testing.T) *Observer { - // Create Bitcoin observer - ob := createObserverWithPrivateKey(t) - tssAddress, err := ob.TSS().PubKey().AddressBTC(chains.BitcoinTestnet.ChainId) - require.NoError(t, err) - - // Create 10 dummy UTXOs (22.44 BTC in total) - ob.utxos = make([]btcjson.ListUnspentResult, 0, 10) - amounts := []float64{0.01, 0.12, 0.18, 0.24, 0.5, 1.26, 2.97, 3.28, 5.16, 8.72} - for _, amount := range amounts { - ob.utxos = append(ob.utxos, btcjson.ListUnspentResult{Address: tssAddress.EncodeAddress(), Amount: amount}) - } - return ob -} - -func mineTxNSetNonceMark(t *testing.T, ob *Observer, nonce uint64, txid string, preMarkIndex int) { - // Mine transaction - outboundID := ob.OutboundID(nonce) - ob.includedTxResults[outboundID] = &btcjson.GetTransactionResult{TxID: txid} - - // Set nonce mark - tssAddress, err := ob.TSS().PubKey().AddressBTC(chains.BitcoinMainnet.ChainId) - require.NoError(t, err) - nonceMark := btcjson.ListUnspentResult{ - TxID: txid, - Address: tssAddress.EncodeAddress(), - Amount: float64(chains.NonceMarkAmount(nonce)) * 1e-8, - } - if preMarkIndex >= 0 { // replace nonce-mark utxo - ob.utxos[preMarkIndex] = nonceMark - - } else { // add nonce-mark utxo directly - ob.utxos = append(ob.utxos, nonceMark) - } - sort.SliceStable(ob.utxos, func(i, j int) bool { - return ob.utxos[i].Amount < ob.utxos[j].Amount - }) -} - func TestCheckTSSVout(t *testing.T) { // the archived outbound raw result file and cctx file // https://blockstream.info/tx/030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0 @@ -256,220 +213,3 @@ func TestCheckTSSVoutCancelled(t *testing.T) { require.ErrorContains(t, err, "not match TSS address") }) } - -func TestSelectUTXOs(t *testing.T) { - ctx := context.Background() - - ob := createObserverWithUTXOs(t) - dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4" - - // Case1: nonce = 0, bootstrap - // input: utxoCap = 5, amount = 0.01, nonce = 0 - // output: [0.01], 0.01 - selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 0, math.MaxUint16, true) - require.NoError(t, err) - require.Equal(t, 0.01, selected.Value) - require.Equal(t, ob.utxos[0:1], selected.UTXOs) - - // Case2: nonce = 1, must FAIL and wait for previous transaction to be mined - // input: utxoCap = 5, amount = 0.5, nonce = 1 - // output: error - selected, err = ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) - require.Error(t, err) - require.Nil(t, selected.UTXOs) - require.Zero(t, selected.Value) - require.Equal(t, "getOutboundHashByNonce: cannot find outbound txid for nonce 0", err.Error()) - mineTxNSetNonceMark(t, ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 - - // Case3: nonce = 1, should pass now - // input: utxoCap = 5, amount = 0.5, nonce = 1 - // output: [0.00002, 0.01, 0.12, 0.18, 0.24], 0.55002 - selected, err = ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) - require.NoError(t, err) - require.Equal(t, 0.55002, selected.Value) - require.Equal(t, ob.utxos[0:5], selected.UTXOs) - mineTxNSetNonceMark(t, ob, 1, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 1 - - // Case4: - // input: utxoCap = 5, amount = 1.0, nonce = 2 - // output: [0.00002001, 0.01, 0.12, 0.18, 0.24, 0.5], 1.05002001 - selected, err = ob.SelectUTXOs(ctx, 1.0, 5, 2, math.MaxUint16, true) - require.NoError(t, err) - require.InEpsilon(t, 1.05002001, selected.Value, 1e-8) - require.Equal(t, ob.utxos[0:6], selected.UTXOs) - mineTxNSetNonceMark(t, ob, 2, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 2 - - // Case5: should include nonce-mark utxo on the LEFT - // input: utxoCap = 5, amount = 8.05, nonce = 3 - // output: [0.00002002, 0.24, 0.5, 1.26, 2.97, 3.28], 8.25002002 - selected, err = ob.SelectUTXOs(ctx, 8.05, 5, 3, math.MaxUint16, true) - require.NoError(t, err) - require.InEpsilon(t, 8.25002002, selected.Value, 1e-8) - expected := append([]btcjson.ListUnspentResult{ob.utxos[0]}, ob.utxos[4:9]...) - require.Equal(t, expected, selected.UTXOs) - mineTxNSetNonceMark(t, ob, 24105431, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 24105431 - - // Case6: should include nonce-mark utxo on the RIGHT - // input: utxoCap = 5, amount = 0.503, nonce = 24105432 - // output: [0.24107432, 0.01, 0.12, 0.18, 0.24], 0.55002002 - selected, err = ob.SelectUTXOs(ctx, 0.503, 5, 24105432, math.MaxUint16, true) - require.NoError(t, err) - require.InEpsilon(t, 0.79107431, selected.Value, 1e-8) - expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:4]...) - require.Equal(t, expected, selected.UTXOs) - mineTxNSetNonceMark(t, ob, 24105432, dummyTxID, 4) // mine a transaction and set nonce-mark utxo for nonce 24105432 - - // Case7: should include nonce-mark utxo in the MIDDLE - // input: utxoCap = 5, amount = 1.0, nonce = 24105433 - // output: [0.24107432, 0.12, 0.18, 0.24, 0.5], 1.28107432 - selected, err = ob.SelectUTXOs(ctx, 1.0, 5, 24105433, math.MaxUint16, true) - require.NoError(t, err) - require.InEpsilon(t, 1.28107432, selected.Value, 1e-8) - expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[1:4]...) - expected = append(expected, ob.utxos[5]) - require.Equal(t, expected, selected.UTXOs) - - // Case8: should work with maximum amount - // input: utxoCap = 5, amount = 16.03 - // output: [0.24107432, 1.26, 2.97, 3.28, 5.16, 8.72], 21.63107432 - selected, err = ob.SelectUTXOs(ctx, 16.03, 5, 24105433, math.MaxUint16, true) - require.NoError(t, err) - require.InEpsilon(t, 21.63107432, selected.Value, 1e-8) - expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[6:11]...) - require.Equal(t, expected, selected.UTXOs) - - // Case9: must FAIL due to insufficient funds - // input: utxoCap = 5, amount = 21.64 - // output: error - selected, err = ob.SelectUTXOs(ctx, 21.64, 5, 24105433, math.MaxUint16, true) - require.Error(t, err) - require.Nil(t, selected.UTXOs) - require.Zero(t, selected.Value) - require.Equal( - t, - "SelectUTXOs: not enough btc in reserve - available : 21.63107432 , tx amount : 21.64", - err.Error(), - ) -} - -func TestUTXOConsolidation(t *testing.T) { - ctx := context.Background() - - dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4" - - t.Run("should not consolidate", func(t *testing.T) { - ob := createObserverWithUTXOs(t) - mineTxNSetNonceMark(t, ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 - - // input: utxoCap = 10, amount = 0.01, nonce = 1, rank = 10 - // output: [0.00002, 0.01], 0.01002 - res, err := ob.SelectUTXOs(ctx, 0.01, 10, 1, 10, true) - require.NoError(t, err) - require.Equal(t, 0.01002, res.Value) - require.Equal(t, ob.utxos[0:2], res.UTXOs) - require.Equal(t, uint16(0), res.ConsolidatedUTXOs) - require.Equal(t, 0.0, res.ConsolidatedValue) - }) - - t.Run("should consolidate 1 utxo", func(t *testing.T) { - ob := createObserverWithUTXOs(t) - mineTxNSetNonceMark(t, ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 - - // input: utxoCap = 9, amount = 0.01, nonce = 1, rank = 9 - // output: [0.00002, 0.01, 0.12], 0.13002 - res, err := ob.SelectUTXOs(ctx, 0.01, 9, 1, 9, true) - require.NoError(t, err) - require.Equal(t, 0.13002, res.Value) - require.Equal(t, ob.utxos[0:3], res.UTXOs) - require.Equal(t, uint16(1), res.ConsolidatedUTXOs) - require.Equal(t, 0.12, res.ConsolidatedValue) - }) - - t.Run("should consolidate 3 utxos", func(t *testing.T) { - ob := createObserverWithUTXOs(t) - mineTxNSetNonceMark(t, ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 - - // input: utxoCap = 5, amount = 0.01, nonce = 0, rank = 5 - // output: [0.00002, 0.014, 1.26, 0.5, 0.2], 2.01002 - res, err := ob.SelectUTXOs(ctx, 0.01, 5, 1, 5, true) - require.NoError(t, err) - require.Equal(t, 2.01002, res.Value) - expected := make([]btcjson.ListUnspentResult, 2) - copy(expected, ob.utxos[0:2]) - for i := 6; i >= 4; i-- { // append consolidated utxos in descending order - expected = append(expected, ob.utxos[i]) - } - require.Equal(t, expected, res.UTXOs) - require.Equal(t, uint16(3), res.ConsolidatedUTXOs) - require.Equal(t, 2.0, res.ConsolidatedValue) - }) - - t.Run("should consolidate all utxos using rank 1", func(t *testing.T) { - ob := createObserverWithUTXOs(t) - mineTxNSetNonceMark(t, ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 - - // input: utxoCap = 12, amount = 0.01, nonce = 0, rank = 1 - // output: [0.00002, 0.01, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18, 0.12], 22.44002 - res, err := ob.SelectUTXOs(ctx, 0.01, 12, 1, 1, true) - require.NoError(t, err) - require.Equal(t, 22.44002, res.Value) - expected := make([]btcjson.ListUnspentResult, 2) - copy(expected, ob.utxos[0:2]) - for i := 10; i >= 2; i-- { // append consolidated utxos in descending order - expected = append(expected, ob.utxos[i]) - } - require.Equal(t, expected, res.UTXOs) - require.Equal(t, uint16(9), res.ConsolidatedUTXOs) - require.Equal(t, 22.43, res.ConsolidatedValue) - }) - - t.Run("should consolidate 3 utxos sparse", func(t *testing.T) { - ob := createObserverWithUTXOs(t) - mineTxNSetNonceMark( - t, - ob, - 24105431, - dummyTxID, - -1, - ) // mine a transaction and set nonce-mark utxo for nonce 24105431 - - // input: utxoCap = 5, amount = 0.13, nonce = 24105432, rank = 5 - // output: [0.24107431, 0.01, 0.12, 1.26, 0.5, 0.24], 2.37107431 - res, err := ob.SelectUTXOs(ctx, 0.13, 5, 24105432, 5, true) - require.NoError(t, err) - require.InEpsilon(t, 2.37107431, res.Value, 1e-8) - expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...) - expected = append(expected, ob.utxos[6]) - expected = append(expected, ob.utxos[5]) - expected = append(expected, ob.utxos[3]) - require.Equal(t, expected, res.UTXOs) - require.Equal(t, uint16(3), res.ConsolidatedUTXOs) - require.Equal(t, 2.0, res.ConsolidatedValue) - }) - - t.Run("should consolidate all utxos sparse", func(t *testing.T) { - ob := createObserverWithUTXOs(t) - mineTxNSetNonceMark( - t, - ob, - 24105431, - dummyTxID, - -1, - ) // mine a transaction and set nonce-mark utxo for nonce 24105431 - - // input: utxoCap = 12, amount = 0.13, nonce = 24105432, rank = 1 - // output: [0.24107431, 0.01, 0.12, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18], 22.68107431 - res, err := ob.SelectUTXOs(ctx, 0.13, 12, 24105432, 1, true) - require.NoError(t, err) - require.InEpsilon(t, 22.68107431, res.Value, 1e-8) - expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...) - for i := 10; i >= 5; i-- { // append consolidated utxos in descending order - expected = append(expected, ob.utxos[i]) - } - expected = append(expected, ob.utxos[3]) - expected = append(expected, ob.utxos[2]) - require.Equal(t, expected, res.UTXOs) - require.Equal(t, uint16(8), res.ConsolidatedUTXOs) - require.Equal(t, 22.31, res.ConsolidatedValue) - }) -} diff --git a/zetaclient/chains/bitcoin/observer/utxos_test.go b/zetaclient/chains/bitcoin/observer/utxos_test.go new file mode 100644 index 0000000000..7038f8be3a --- /dev/null +++ b/zetaclient/chains/bitcoin/observer/utxos_test.go @@ -0,0 +1,304 @@ +package observer_test + +import ( + "context" + "math" + "testing" + + "github.com/btcsuite/btcd/btcjson" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + observertypes "github.com/zeta-chain/node/x/observer/types" + + "github.com/zeta-chain/node/pkg/chains" +) + +func Test_SelectUTXOs(t *testing.T) { + ctx := context.Background() + dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4" + + t.Run("noce = 0, should bootstrap", func(t *testing.T) { + // input: utxoCap = 5, amount = 0.01, nonce = 0 + // output: [0.01], 0.01 + ob, utxos := newTestSuitWithUTXOs(t) + selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 0, math.MaxUint16, true) + require.NoError(t, err) + require.Equal(t, 0.01, selected.Value) + require.Equal(t, utxos[0:1], selected.UTXOs) + }) + + t.Run("nonce = 1, must FAIL and wait for previous transaction to be mined", func(t *testing.T) { + // input: utxoCap = 5, amount = 0.5, nonce = 1 + // output: error + ob, _ := newTestSuitWithUTXOs(t) + selected, err := ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) + require.Error(t, err) + require.Nil(t, selected.UTXOs) + require.Zero(t, selected.Value) + require.ErrorContains(t, err, "cannot find outbound txid for nonce 0") + }) + + t.Run("nonce = 1, should pass when nonce mark 0 is set", func(t *testing.T) { + // input: utxoCap = 5, amount = 0.5, nonce = 1 + // output: [0.00002, 0.01, 0.12, 0.18, 0.24], 0.55002 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 0, dummyTxID) + selected, err := ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) + require.NoError(t, err) + require.Equal(t, 0.55002, selected.Value) + require.Equal(t, utxos[0:5], selected.UTXOs) + }) + + t.Run("nonce = 2, should pass when nonce mark 1 is set", func(t *testing.T) { + // input: utxoCap = 5, amount = 1.0, nonce = 2 + // output: [0.00002001, 0.01, 0.12, 0.18, 0.24, 0.5], 1.05002001 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 1, dummyTxID) + selected, err := ob.SelectUTXOs(ctx, 1.0, 5, 2, math.MaxUint16, true) + require.NoError(t, err) + require.InEpsilon(t, 1.05002001, selected.Value, 1e-8) + require.Equal(t, utxos[0:6], selected.UTXOs) + }) + + t.Run("nonce = 3, should select nonce-mark utxo on the LEFT", func(t *testing.T) { + // input: utxoCap = 5, amount = 8.05, nonce = 3 + // output: [0.00002002, 0.24, 0.5, 1.26, 2.97, 3.28], 8.25002002 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 2, dummyTxID) + selected, err := ob.SelectUTXOs(ctx, 8.05, 5, 3, math.MaxUint16, true) + require.NoError(t, err) + require.InEpsilon(t, 8.25002002, selected.Value, 1e-8) + expected := append([]btcjson.ListUnspentResult{utxos[0]}, utxos[4:9]...) + require.Equal(t, expected, selected.UTXOs) + }) + + t.Run("nonce = 24105432, should select nonce-mark utxo on the RIGHT", func(t *testing.T) { + // input: utxoCap = 5, amount = 0.503, nonce = 24105432 + // output: [0.24107432, 0.01, 0.12, 0.18, 0.24], 0.7910731 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105431, dummyTxID) + selected, err := ob.SelectUTXOs(ctx, 0.503, 5, 24105432, math.MaxUint16, true) + require.NoError(t, err) + require.InEpsilon(t, 0.79107431, selected.Value, 1e-8) + expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[0:4]...) + require.Equal(t, expected, selected.UTXOs) + }) + + t.Run("nonce = 24105433, should select nonce-mark utxo in the MIDDLE", func(t *testing.T) { + // input: utxoCap = 5, amount = 1.0, nonce = 24105433 + // output: [0.24107432, 0.12, 0.18, 0.24, 0.5], 1.28107432 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105432, dummyTxID) + selected, err := ob.SelectUTXOs(ctx, 1.0, 5, 24105433, math.MaxUint16, true) + require.NoError(t, err) + require.InEpsilon(t, 1.28107432, selected.Value, 1e-8) + expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[1:4]...) + expected = append(expected, utxos[5]) + require.Equal(t, expected, selected.UTXOs) + }) + + t.Run("nonce = 24105433, should select biggest utxos to maximize amount", func(t *testing.T) { + // input: utxoCap = 5, amount = 16.03 + // output: [0.24107432, 1.26, 2.97, 3.28, 5.16, 8.72], 21.63107432 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105432, dummyTxID) + selected, err := ob.SelectUTXOs(ctx, 16.03, 5, 24105433, math.MaxUint16, true) + require.NoError(t, err) + require.InEpsilon(t, 21.63107432, selected.Value, 1e-8) + expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[6:11]...) + require.Equal(t, expected, selected.UTXOs) + }) + + t.Run("nonce = 24105433, should fail due to insufficient funds", func(t *testing.T) { + // input: utxoCap = 5, amount = 21.64 + // output: error + ob, _ := createTestSuitWithUTXOsAndNonceMark(t, 24105432, dummyTxID) + selected, err := ob.SelectUTXOs(ctx, 21.64, 5, 24105433, math.MaxUint16, true) + require.Error(t, err) + require.Nil(t, selected.UTXOs) + require.Zero(t, selected.Value) + require.ErrorContains(t, err, "not enough btc in reserve - available : 21.63107432 , tx amount : 21.64") + }) +} + +func Test_SelectUTXOs_Consolidation(t *testing.T) { + ctx := context.Background() + dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4" + + t.Run("should not consolidate", func(t *testing.T) { + // create test suite and set nonce-mark utxo for nonce 0 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 0, dummyTxID) + + // input: utxoCap = 10, amount = 0.01, nonce = 1, rank = 10 + // output: [0.00002, 0.01], 0.01002 + res, err := ob.SelectUTXOs(ctx, 0.01, 10, 1, 10, true) + require.NoError(t, err) + require.Equal(t, 0.01002, res.Value) + require.Equal(t, utxos[0:2], res.UTXOs) + require.Zero(t, res.ConsolidatedUTXOs) + require.Zero(t, res.ConsolidatedValue) + }) + + t.Run("should consolidate 1 utxo", func(t *testing.T) { + // create test suite and set nonce-mark utxo for nonce 0 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 0, dummyTxID) + + // input: utxoCap = 9, amount = 0.01, nonce = 1, rank = 9 + // output: [0.00002, 0.01, 0.12], 0.13002 + selected, err := ob.SelectUTXOs(ctx, 0.01, 9, 1, 9, true) + require.NoError(t, err) + require.Equal(t, 0.13002, selected.Value) + require.Equal(t, utxos[0:3], selected.UTXOs) + require.Equal(t, uint16(1), selected.ConsolidatedUTXOs) + require.Equal(t, 0.12, selected.ConsolidatedValue) + }) + + t.Run("should consolidate 3 utxos", func(t *testing.T) { + // create test suite and set nonce-mark utxo for nonce 0 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 0, dummyTxID) + + // input: utxoCap = 5, amount = 0.01, nonce = 0, rank = 5 + // output: [0.00002, 0.014, 1.26, 0.5, 0.2], 2.01002 + selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 1, 5, true) + require.NoError(t, err) + require.Equal(t, 2.01002, selected.Value) + expected := make([]btcjson.ListUnspentResult, 2) + copy(expected, utxos[0:2]) + for i := 6; i >= 4; i-- { // append consolidated utxos in descending order + expected = append(expected, utxos[i]) + } + require.Equal(t, expected, selected.UTXOs) + require.Equal(t, uint16(3), selected.ConsolidatedUTXOs) + require.Equal(t, 2.0, selected.ConsolidatedValue) + }) + + t.Run("should consolidate all utxos using rank 1", func(t *testing.T) { + // create test suite and set nonce-mark utxo for nonce 0 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 0, dummyTxID) + + // input: utxoCap = 12, amount = 0.01, nonce = 0, rank = 1 + // output: [0.00002, 0.01, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18, 0.12], 22.44002 + selected, err := ob.SelectUTXOs(ctx, 0.01, 12, 1, 1, true) + require.NoError(t, err) + require.Equal(t, 22.44002, selected.Value) + expected := make([]btcjson.ListUnspentResult, 2) + copy(expected, utxos[0:2]) + for i := 10; i >= 2; i-- { // append consolidated utxos in descending order + expected = append(expected, utxos[i]) + } + require.Equal(t, expected, selected.UTXOs) + require.Equal(t, uint16(9), selected.ConsolidatedUTXOs) + require.Equal(t, 22.43, selected.ConsolidatedValue) + }) + + t.Run("should consolidate 3 utxos sparse", func(t *testing.T) { + // create test suite and set nonce-mark utxo for nonce 24105431 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105431, dummyTxID) + + // input: utxoCap = 5, amount = 0.13, nonce = 24105432, rank = 5 + // output: [0.24107431, 0.01, 0.12, 1.26, 0.5, 0.24], 2.37107431 + selected, err := ob.SelectUTXOs(ctx, 0.13, 5, 24105432, 5, true) + require.NoError(t, err) + require.InEpsilon(t, 2.37107431, selected.Value, 1e-8) + expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[0:2]...) + expected = append(expected, utxos[6]) + expected = append(expected, utxos[5]) + expected = append(expected, utxos[3]) + require.Equal(t, expected, selected.UTXOs) + require.Equal(t, uint16(3), selected.ConsolidatedUTXOs) + require.Equal(t, 2.0, selected.ConsolidatedValue) + }) + + t.Run("should consolidate all utxos sparse", func(t *testing.T) { + // create test suite and set nonce-mark utxo for nonce 24105431 + ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105431, dummyTxID) + + // input: utxoCap = 12, amount = 0.13, nonce = 24105432, rank = 1 + // output: [0.24107431, 0.01, 0.12, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18], 22.68107431 + selected, err := ob.SelectUTXOs(ctx, 0.13, 12, 24105432, 1, true) + require.NoError(t, err) + require.InEpsilon(t, 22.68107431, selected.Value, 1e-8) + expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[0:2]...) + for i := 10; i >= 5; i-- { // append consolidated utxos in descending order + expected = append(expected, utxos[i]) + } + expected = append(expected, utxos[3]) + expected = append(expected, utxos[2]) + require.Equal(t, expected, selected.UTXOs) + require.Equal(t, uint16(8), selected.ConsolidatedUTXOs) + require.Equal(t, 22.31, selected.ConsolidatedValue) + }) +} + +// helper function to create a test suite with UTXOs +func newTestSuitWithUTXOs(t *testing.T) (*testSuite, []btcjson.ListUnspentResult) { + // create test observer + ob := newTestSuite(t, chains.BitcoinMainnet, "") + + // get test UTXOs + tssAddress, err := ob.TSS().PubKey().AddressBTC(ob.Chain().ChainId) + require.NoError(t, err) + utxos := getTestUTXOs(tssAddress.EncodeAddress()) + + // mock up pending nonces and UTXOs + pendingNonces := observertypes.PendingNonces{} + ob.zetacore.On("GetPendingNoncesByChain", mock.Anything, mock.Anything).Maybe().Return(pendingNonces, nil) + ob.client.On("ListUnspentMinMaxAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Maybe(). + Return(utxos, nil) + + // update UTXOs + err = ob.FetchUTXOs(context.Background()) + require.NoError(t, err) + + return ob, utxos +} + +// helper function to create a test suite with UTXOs and nonce mark +func createTestSuitWithUTXOsAndNonceMark( + t *testing.T, + nonce uint64, + txid string, +) (*testSuite, []btcjson.ListUnspentResult) { + // create test observer + ob := newTestSuite(t, chains.BitcoinMainnet, "") + + // make a nonce mark UTXO + tssAddress, err := ob.TSS().PubKey().AddressBTC(ob.Chain().ChainId) + require.NoError(t, err) + nonceMark := btcjson.ListUnspentResult{ + TxID: txid, + Address: tssAddress.EncodeAddress(), + Amount: float64(chains.NonceMarkAmount(nonce)) * 1e-8, + Confirmations: 1, + } + + // get test UTXOs and append nonce-mark UTXO + utxos := getTestUTXOs(tssAddress.EncodeAddress()) + utxos = append(utxos, nonceMark) + + // mock up pending nonces and UTXOs + pendingNonces := observertypes.PendingNonces{} + ob.zetacore.On("GetPendingNoncesByChain", mock.Anything, mock.Anything).Maybe().Return(pendingNonces, nil) + ob.client.On("ListUnspentMinMaxAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Maybe(). + Return(utxos, nil) + + // update UTXOs + err = ob.FetchUTXOs(context.Background()) + require.NoError(t, err) + + // set nonce-mark + ob.Observer.SetIncludedTx(nonce, &btcjson.GetTransactionResult{TxID: txid}) + + return ob, utxos +} + +// getTestUTXOs returns a list of constant UTXOs for testing +func getTestUTXOs(owner string) []btcjson.ListUnspentResult { + // create 10 constant dummy UTXOs (22.44 BTC in total) + utxos := make([]btcjson.ListUnspentResult, 0, 10) + amounts := []float64{0.01, 0.12, 0.18, 0.24, 0.5, 1.26, 2.97, 3.28, 5.16, 8.72} + for _, amount := range amounts { + utxos = append(utxos, btcjson.ListUnspentResult{ + Address: owner, + Amount: amount, + Confirmations: 1, + }) + } + return utxos +} From 8040ac137d121d28602d5dae45aca1271aafc1f7 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 29 Jan 2025 23:16:35 -0600 Subject: [PATCH 35/74] wrap RPC error in LoadLastBlockScanned --- zetaclient/chains/bitcoin/observer/db.go | 2 +- zetaclient/chains/bitcoin/observer/db_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/db.go b/zetaclient/chains/bitcoin/observer/db.go index 45360fe3f5..eb80752354 100644 --- a/zetaclient/chains/bitcoin/observer/db.go +++ b/zetaclient/chains/bitcoin/observer/db.go @@ -43,7 +43,7 @@ func (ob *Observer) LoadLastBlockScanned(ctx context.Context) error { if ob.LastBlockScanned() == 0 { blockNumber, err := ob.rpc.GetBlockCount(ctx) if err != nil { - return errors.Wrapf(err, "error GetBlockCount for chain %d", ob.Chain().ChainId) + return errors.Wrap(err, "unable to get block count") } // #nosec G115 always positive ob.WithLastBlockScanned(uint64(blockNumber)) diff --git a/zetaclient/chains/bitcoin/observer/db_test.go b/zetaclient/chains/bitcoin/observer/db_test.go index 948aed5826..29b94deb39 100644 --- a/zetaclient/chains/bitcoin/observer/db_test.go +++ b/zetaclient/chains/bitcoin/observer/db_test.go @@ -105,7 +105,7 @@ func Test_LoadLastBlockScanned(t *testing.T) { // load last block scanned err := obOther.LoadLastBlockScanned(ctx) - require.ErrorContains(t, err, "rpc error") + require.ErrorContains(t, err, "unable to get block count") }) t.Run("should use hardcode block 100 for regtest", func(t *testing.T) { // use regtest chain From a64f73805cfbc1a94c9ddddbd8fc1554dbf70f39 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 29 Jan 2025 23:43:37 -0600 Subject: [PATCH 36/74] move last scanned block to log field; use Opt function for test suite --- zetaclient/chains/bitcoin/observer/db.go | 2 +- zetaclient/chains/bitcoin/observer/db_test.go | 14 +++--- .../chains/bitcoin/observer/event_test.go | 6 +-- .../chains/bitcoin/observer/inbound_test.go | 2 +- .../chains/bitcoin/observer/observer_test.go | 47 ++++++++++++------- .../chains/bitcoin/observer/utxos_test.go | 4 +- zetaclient/chains/evm/observer/observer.go | 2 +- 7 files changed, 45 insertions(+), 32 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/db.go b/zetaclient/chains/bitcoin/observer/db.go index eb80752354..122e8d2aad 100644 --- a/zetaclient/chains/bitcoin/observer/db.go +++ b/zetaclient/chains/bitcoin/observer/db.go @@ -53,7 +53,7 @@ func (ob *Observer) LoadLastBlockScanned(ctx context.Context) error { if chains.IsBitcoinRegnet(ob.Chain().ChainId) { ob.WithLastBlockScanned(RegnetStartBlock) } - ob.Logger().Chain.Info().Msgf("chain %d starts scanning from block %d", ob.Chain().ChainId, ob.LastBlockScanned()) + ob.Logger().Chain.Info().Uint64("last_block_scanned", ob.LastBlockScanned()).Msg("LoadLastBlockScanned succeed") return nil } diff --git a/zetaclient/chains/bitcoin/observer/db_test.go b/zetaclient/chains/bitcoin/observer/db_test.go index 29b94deb39..86bc1d3989 100644 --- a/zetaclient/chains/bitcoin/observer/db_test.go +++ b/zetaclient/chains/bitcoin/observer/db_test.go @@ -37,7 +37,7 @@ func Test_SaveBroadcastedTx(t *testing.T) { nonce := uint64(1) txHash := sample.BtcHash().String() dbPath := sample.CreateTempDir(t) - ob := newTestSuite(t, chains.BitcoinMainnet, dbPath) + ob := newTestSuite(t, chains.BitcoinMainnet, optDBPath(dbPath)) if tt.wantErr != "" { // delete db to simulate db error os.RemoveAll(dbPath) @@ -71,7 +71,7 @@ func Test_LoadLastBlockScanned(t *testing.T) { t.Run("should load last block scanned", func(t *testing.T) { // create observer and write 199 as last block scanned - ob := newTestSuite(t, chain, "") + ob := newTestSuite(t, chain) ob.WriteLastBlockScannedToDB(199) // load last block scanned @@ -81,7 +81,7 @@ func Test_LoadLastBlockScanned(t *testing.T) { }) t.Run("should fail on invalid env var", func(t *testing.T) { // create observer - ob := newTestSuite(t, chain, "") + ob := newTestSuite(t, chain) // set invalid environment variable envvar := base.EnvVarLatestBlockByChain(chain) @@ -94,7 +94,7 @@ func Test_LoadLastBlockScanned(t *testing.T) { }) t.Run("should fail on RPC error", func(t *testing.T) { // create observer on separate path, as we need to reset last block scanned - obOther := newTestSuite(t, chain, "") + obOther := newTestSuite(t, chain) // reset last block scanned to 0 so that it will be loaded from RPC obOther.WithLastBlockScanned(0) @@ -109,7 +109,7 @@ func Test_LoadLastBlockScanned(t *testing.T) { }) t.Run("should use hardcode block 100 for regtest", func(t *testing.T) { // use regtest chain - obRegnet := newTestSuite(t, chains.BitcoinRegtest, "") + obRegnet := newTestSuite(t, chains.BitcoinRegtest) // load last block scanned err := obRegnet.LoadLastBlockScanned(ctx) @@ -126,11 +126,11 @@ func Test_LoadBroadcastedTxMap(t *testing.T) { // create observer and save a test tx dbPath := sample.CreateTempDir(t) - obOld := newTestSuite(t, chains.BitcoinMainnet, dbPath) + obOld := newTestSuite(t, chains.BitcoinMainnet, optDBPath(dbPath)) obOld.SaveBroadcastedTx(txHash, nonce) // create new observer using same db path - obNew := newTestSuite(t, chains.BitcoinMainnet, dbPath) + obNew := newTestSuite(t, chains.BitcoinMainnet, optDBPath(dbPath)) // load broadcasted tx map to new observer err := obNew.LoadBroadcastedTxMap() diff --git a/zetaclient/chains/bitcoin/observer/event_test.go b/zetaclient/chains/bitcoin/observer/event_test.go index 4d3492105f..744a4f033a 100644 --- a/zetaclient/chains/bitcoin/observer/event_test.go +++ b/zetaclient/chains/bitcoin/observer/event_test.go @@ -308,7 +308,7 @@ func Test_IsEventProcessable(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain, "") + ob := newTestSuite(t, chain) // setup compliance config cfg := config.Config{ @@ -356,7 +356,7 @@ func Test_NewInboundVoteFromLegacyMemo(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain, "") + ob := newTestSuite(t, chain) ob.zetacore.WithKeys(&keys.Keys{}).WithZetaChain() t.Run("should create new inbound vote msg V1", func(t *testing.T) { @@ -396,7 +396,7 @@ func Test_NewInboundVoteFromStdMemo(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain, "") + ob := newTestSuite(t, chain) ob.zetacore.WithKeys(&keys.Keys{}).WithZetaChain() t.Run("should create new inbound vote msg with standard memo", func(t *testing.T) { diff --git a/zetaclient/chains/bitcoin/observer/inbound_test.go b/zetaclient/chains/bitcoin/observer/inbound_test.go index fce2137d55..815b800806 100644 --- a/zetaclient/chains/bitcoin/observer/inbound_test.go +++ b/zetaclient/chains/bitcoin/observer/inbound_test.go @@ -158,7 +158,7 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { chain := chains.BitcoinMainnet // create test observer - ob := newTestSuite(t, chain, "") + ob := newTestSuite(t, chain) ob.zetacore.WithKeys(&keys.Keys{}).WithZetaChain() // test cases diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index 8862d12c67..cf86f67ed5 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -167,7 +167,7 @@ func Test_NewObserver(t *testing.T) { func Test_BlockCache(t *testing.T) { t.Run("should add and get block from cache", func(t *testing.T) { // create observer - ob := newTestSuite(t, chains.BitcoinMainnet, "") + ob := newTestSuite(t, chains.BitcoinMainnet) // feed block hash, header and block to btc client hash := sample.BtcHash() @@ -191,7 +191,7 @@ func Test_BlockCache(t *testing.T) { }) t.Run("should fail if stored type is not BlockNHeader", func(t *testing.T) { // create observer - ob := newTestSuite(t, chains.BitcoinMainnet, "") + ob := newTestSuite(t, chains.BitcoinMainnet) // add a string to cache blockNumber := int64(100) @@ -206,7 +206,7 @@ func Test_BlockCache(t *testing.T) { func Test_SetPendingNonce(t *testing.T) { // create observer - ob := newTestSuite(t, chains.BitcoinMainnet, "") + ob := newTestSuite(t, chains.BitcoinMainnet) // ensure pending nonce is 0 require.Zero(t, ob.GetPendingNonce()) @@ -219,7 +219,7 @@ func Test_SetPendingNonce(t *testing.T) { func TestConfirmationThreshold(t *testing.T) { chain := chains.BitcoinMainnet - ob := newTestSuite(t, chain, "") + ob := newTestSuite(t, chain) t.Run("should return confirmations in chain param", func(t *testing.T) { ob.SetChainParams(observertypes.ChainParams{ConfirmationCount: 3}) @@ -266,13 +266,26 @@ type testSuite struct { client *mocks.BitcoinClient zetacore *mocks.ZetacoreClient db *db.DB + dbPath string } -func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { - ctx := context.Background() +type Opt func(t *testSuite) - require.True(t, chain.IsBitcoinChain()) +// optDBPath is an option to set custom db path +func optDBPath(dbPath string) Opt { + return func(t *testSuite) { + t.dbPath = dbPath + } +} + +func newTestSuite(t *testing.T, chain chains.Chain, opts ...Opt) *testSuite { + // create test suite with options + s := &testSuite{ctx: context.Background()} + for _, opt := range opts { + opt(s) + } + require.True(t, chain.IsBitcoinChain()) chainParams := mocks.MockChainParams(chain.ChainId, 10) client := mocks.NewBitcoinClient(t) @@ -290,11 +303,11 @@ func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { // create test database var err error var database *db.DB - if dbPath == "" { + if s.dbPath == "" { database, err = db.NewFromSqliteInMemory(true) } else { - database, err = db.NewFromSqlite(dbPath, "test.db", true) - t.Cleanup(func() { os.RemoveAll(dbPath) }) + database, err = db.NewFromSqlite(s.dbPath, "test.db", true) + t.Cleanup(func() { os.RemoveAll(s.dbPath) }) } require.NoError(t, err) @@ -317,11 +330,11 @@ func newTestSuite(t *testing.T, chain chains.Chain, dbPath string) *testSuite { ob, err := observer.New(chain, baseObserver, client) require.NoError(t, err) - return &testSuite{ - ctx: ctx, - Observer: ob, - client: client, - zetacore: zetacore, - db: database, - } + // set test suite fields + s.Observer = ob + s.client = client + s.zetacore = zetacore + s.db = database + + return s } diff --git a/zetaclient/chains/bitcoin/observer/utxos_test.go b/zetaclient/chains/bitcoin/observer/utxos_test.go index 7038f8be3a..91f84e8156 100644 --- a/zetaclient/chains/bitcoin/observer/utxos_test.go +++ b/zetaclient/chains/bitcoin/observer/utxos_test.go @@ -227,7 +227,7 @@ func Test_SelectUTXOs_Consolidation(t *testing.T) { // helper function to create a test suite with UTXOs func newTestSuitWithUTXOs(t *testing.T) (*testSuite, []btcjson.ListUnspentResult) { // create test observer - ob := newTestSuite(t, chains.BitcoinMainnet, "") + ob := newTestSuite(t, chains.BitcoinMainnet) // get test UTXOs tssAddress, err := ob.TSS().PubKey().AddressBTC(ob.Chain().ChainId) @@ -255,7 +255,7 @@ func createTestSuitWithUTXOsAndNonceMark( txid string, ) (*testSuite, []btcjson.ListUnspentResult) { // create test observer - ob := newTestSuite(t, chains.BitcoinMainnet, "") + ob := newTestSuite(t, chains.BitcoinMainnet) // make a nonce mark UTXO tssAddress, err := ob.TSS().PubKey().AddressBTC(ob.Chain().ChainId) diff --git a/zetaclient/chains/evm/observer/observer.go b/zetaclient/chains/evm/observer/observer.go index f6cce87820..0997e7daec 100644 --- a/zetaclient/chains/evm/observer/observer.go +++ b/zetaclient/chains/evm/observer/observer.go @@ -262,7 +262,7 @@ func (ob *Observer) LoadLastBlockScanned(ctx context.Context) error { } ob.WithLastBlockScanned(blockNumber) } - ob.Logger().Chain.Info().Msgf("chain %d starts scanning from block %d", ob.Chain().ChainId, ob.LastBlockScanned()) + ob.Logger().Chain.Info().Uint64("last_block_scanned", ob.LastBlockScanned()).Msg("LoadLastBlockScanned succeed") return nil } From bb9593559bcca539974fe414b23ae21eddad6a8d Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 30 Jan 2025 00:15:27 -0600 Subject: [PATCH 37/74] move values to log fields --- zetaclient/chains/bitcoin/observer/outbound.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index d7ef25e85e..290e987ba4 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -220,10 +220,12 @@ func (ob *Observer) VoteOutboundIfConfirmed(ctx context.Context, cctx *crosschai // 1. The zetaclient gets restarted. // 2. The tracker is missing in zetacore. func (ob *Observer) refreshPendingNonce(ctx context.Context) { + logger := ob.logger.Outbound.With().Str(logs.FieldMethod, "refreshPendingNonce").Logger() + // get pending nonces from zetacore p, err := ob.ZetacoreClient().GetPendingNoncesByChain(ctx, ob.Chain().ChainId) if err != nil { - ob.logger.Chain.Error().Err(err).Msg("refreshPendingNonce: error getting pending nonces") + logger.Error().Err(err).Msg("error getting pending nonces") } // increase pending nonce if lagged behind @@ -233,13 +235,12 @@ func (ob *Observer) refreshPendingNonce(ctx context.Context) { // get the last included outbound hash txid, err := ob.getOutboundHashByNonce(ctx, nonceLow-1, false) if err != nil { - ob.logger.Chain.Error().Err(err).Msg("refreshPendingNonce: error getting last outbound txid") + logger.Error().Err(err).Msg("error getting last outbound txid") } // set 'NonceLow' as the new pending nonce ob.SetPendingNonce(nonceLow) - ob.logger.Chain.Info(). - Msgf("refreshPendingNonce: increase pending nonce to %d with txid %s", nonceLow, txid) + logger.Info().Uint64("pending_nonce", nonceLow).Str(logs.FieldTx, txid).Msg("increased pending nonce") } } @@ -343,13 +344,14 @@ func (ob *Observer) SetIncludedTx(nonce uint64, getTxResult *btcjson.GetTransact if nonce >= ob.pendingNonce { ob.pendingNonce = nonce + 1 } - ob.logger.Outbound.Info().Fields(lf).Msgf("included new bitcoin outbound, pending nonce %d", ob.pendingNonce) + lf["pending_nonce"] = ob.pendingNonce + ob.logger.Outbound.Info().Fields(lf).Msg("included new bitcoin outbound") } else if txHash == res.TxID { // for existing hash: // - update tx result because confirmations may increase ob.includedTxResults[outboundID] = getTxResult if getTxResult.Confirmations > res.Confirmations { - ob.logger.Outbound.Info().Msgf("bitcoin outbound got %d confirmations", getTxResult.Confirmations) + ob.logger.Outbound.Info().Fields(lf).Msgf("bitcoin outbound got %d confirmations", getTxResult.Confirmations) } } else { // for other hash: @@ -357,7 +359,8 @@ func (ob *Observer) SetIncludedTx(nonce uint64, getTxResult *btcjson.GetTransact // we can't tell which txHash is true, so we remove all to be safe delete(ob.tssOutboundHashes, res.TxID) delete(ob.includedTxResults, outboundID) - ob.logger.Outbound.Error().Msgf("setIncludedTx: duplicate payment by bitcoin outbound %s outboundID %s, prior outbound %s", txHash, outboundID, res.TxID) + lf["prior_outbound"] = res.TxID + ob.logger.Outbound.Error().Fields(lf).Msg("be alert for duplicate payment") } } From 13efedc55a4b641b780fd5c27882ab836db79ebb Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 30 Jan 2025 10:27:43 -0600 Subject: [PATCH 38/74] add unit test for FetchUTXOs --- .../chains/bitcoin/observer/utxos_test.go | 17 +++++++++++++++++ zetaclient/metrics/telemetry.go | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/zetaclient/chains/bitcoin/observer/utxos_test.go b/zetaclient/chains/bitcoin/observer/utxos_test.go index 91f84e8156..b00109b51f 100644 --- a/zetaclient/chains/bitcoin/observer/utxos_test.go +++ b/zetaclient/chains/bitcoin/observer/utxos_test.go @@ -4,15 +4,25 @@ import ( "context" "math" "testing" + "time" "github.com/btcsuite/btcd/btcjson" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" observertypes "github.com/zeta-chain/node/x/observer/types" + "golang.org/x/exp/rand" "github.com/zeta-chain/node/pkg/chains" ) +func Test_FetchUTXOs(t *testing.T) { + // create test suite + ob, utxos := newTestSuitWithUTXOs(t) + + // check number of UTXOs again + require.Equal(t, len(utxos), ob.TelemetryServer().GetNumberOfUTXOs()) +} + func Test_SelectUTXOs(t *testing.T) { ctx := context.Background() dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4" @@ -300,5 +310,12 @@ func getTestUTXOs(owner string) []btcjson.ListUnspentResult { Confirmations: 1, }) } + + // shuffle the UTXOs, zetaclient will always sort them + rand.Seed(uint64(time.Now().Second())) + rand.Shuffle(len(utxos), func(i, j int) { + utxos[i], utxos[j] = utxos[j], utxos[i] + }) + return utxos } diff --git a/zetaclient/metrics/telemetry.go b/zetaclient/metrics/telemetry.go index d225e1ccf5..5a4bf4c27f 100644 --- a/zetaclient/metrics/telemetry.go +++ b/zetaclient/metrics/telemetry.go @@ -152,6 +152,13 @@ func (t *TelemetryServer) SetNumberOfUTXOs(numberOfUTXOs int) { t.mu.Unlock() } +// GetNumberOfUTXOs returns number of UTXOs +func (t *TelemetryServer) GetNumberOfUTXOs() int { + t.mu.Lock() + defer t.mu.Unlock() + return t.status.BTCNumberOfUTXOs +} + // AddFeeEntry adds fee entry func (t *TelemetryServer) AddFeeEntry(block int64, amount int64) { t.mu.Lock() From 416bd03acf307c01a86d7416cddc689e732567f2 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 30 Jan 2025 16:09:18 -0600 Subject: [PATCH 39/74] add unit for SignWithdrawTx; use structured log --- e2e/runner/run.go | 2 +- .../chains/bitcoin/signer/outbound_data.go | 11 +- .../bitcoin/signer/outbound_data_test.go | 11 +- zetaclient/chains/bitcoin/signer/sign.go | 33 +++--- zetaclient/chains/bitcoin/signer/sign_test.go | 111 ++++++++++++++++++ zetaclient/chains/bitcoin/signer/signer.go | 3 +- 6 files changed, 135 insertions(+), 36 deletions(-) diff --git a/e2e/runner/run.go b/e2e/runner/run.go index 318e5734c0..d9cefe58ba 100644 --- a/e2e/runner/run.go +++ b/e2e/runner/run.go @@ -30,7 +30,7 @@ func (r *E2ERunner) RunE2ETests(e2eTests []E2ETest) (err error) { func (r *E2ERunner) RunE2ETest(e2eTest E2ETest, checkAccounting bool) error { startTime := time.Now() // note: spacing is padded to width of completed message - r.Logger.Print("⏳ running - %s", e2eTest.Name) + r.Logger.Print("⏳ running - %s", e2eTest.Description) // run e2e test, if args are not provided, use default args args := e2eTest.Args diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index f036bd02a6..4867456822 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -19,9 +19,6 @@ import ( // OutboundData is a data structure containing necessary data to construct a BTC outbound transaction type OutboundData struct { - // chainID is the external chain ID - chainID int64 - // to is the recipient address to btcutil.Address @@ -51,7 +48,6 @@ type OutboundData struct { // NewOutboundData creates OutboundData from the given CCTX. func NewOutboundData( cctx *types.CrossChainTx, - chainID int64, height uint64, minRelayFee float64, logger, loggerCompliance zerolog.Logger, @@ -93,6 +89,10 @@ func NewOutboundData( amountSats := params.Amount.BigInt().Int64() // check gas limit + if params.CallOptions == nil { + // never happens, 'GetCurrentOutboundParam' will create it + return nil, errors.New("call options is nil") + } if params.CallOptions.GasLimit > math.MaxInt64 { return nil, fmt.Errorf("invalid gas limit %d", params.CallOptions.GasLimit) } @@ -106,7 +106,7 @@ func NewOutboundData( restrictedCCTX := compliance.IsCctxRestricted(cctx) if restrictedCCTX { compliance.PrintComplianceLog(logger, loggerCompliance, - true, chainID, cctx.Index, cctx.InboundParams.Sender, params.Receiver, "BTC") + true, params.ReceiverChainId, cctx.Index, cctx.InboundParams.Sender, params.Receiver, "BTC") } // check dust amount @@ -123,7 +123,6 @@ func NewOutboundData( } return &OutboundData{ - chainID: chainID, to: to, amount: amount, amountSats: amountSats, diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index 023146ca96..e6f4678d61 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -32,7 +32,6 @@ func Test_NewOutboundData(t *testing.T) { name string cctx *crosschaintypes.CrossChainTx cctxModifier func(cctx *crosschaintypes.CrossChainTx) - chainID int64 height uint64 minRelayFee float64 expected *OutboundData @@ -50,11 +49,9 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte cctx.GetCurrentOutboundParam().TssNonce = 1 }, - chainID: chain.ChainId, height: 101, minRelayFee: 0.00001, // 1000 sat/KB expected: &OutboundData{ - chainID: chain.ChainId, to: receiver, amount: 0.1, amountSats: 10000000, @@ -79,11 +76,9 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().GasPriorityFee = "15" // 15 sats/vByte cctx.GetCurrentOutboundParam().TssNonce = 1 }, - chainID: chain.ChainId, height: 101, minRelayFee: 0.00001, // 1000 sat/KB expected: &OutboundData{ - chainID: chain.ChainId, to: receiver, amount: 0.1, amountSats: 10000000, @@ -177,11 +172,9 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte cctx.GetCurrentOutboundParam().TssNonce = 1 }, - chainID: chain.ChainId, height: 101, minRelayFee: 0.00001, // 1000 sat/KB expected: &OutboundData{ - chainID: chain.ChainId, to: receiver, amount: 0, // should cancel the tx amountSats: 0, @@ -204,11 +197,9 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte cctx.GetCurrentOutboundParam().TssNonce = 1 }, - chainID: chain.ChainId, height: 101, minRelayFee: 0.00001, // 1000 sat/KB expected: &OutboundData{ - chainID: chain.ChainId, to: receiver, amount: 0, // should cancel the tx amountSats: 0, @@ -228,7 +219,7 @@ func Test_NewOutboundData(t *testing.T) { tt.cctxModifier(tt.cctx) } - outboundData, err := NewOutboundData(tt.cctx, tt.chainID, tt.height, tt.minRelayFee, log.Logger, log.Logger) + outboundData, err := NewOutboundData(tt.cctx, tt.height, tt.minRelayFee, log.Logger, log.Logger) if tt.errMsg != "" { require.Nil(t, outboundData) require.ErrorContains(t, err, tt.errMsg) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index ca285340a7..229fcf094c 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -44,7 +44,11 @@ func (signer *Signer) SignWithdrawTx( // refresh unspent UTXOs and continue with keysign regardless of error if err := ob.FetchUTXOs(ctx); err != nil { - signer.Logger().Std.Error().Err(err).Uint64("nonce", txData.nonce).Msg("SignWithdrawTx: FetchUTXOs failed") + signer.Logger(). + Std.Error(). + Err(err). + Uint64(logs.FieldNonce, txData.nonce). + Msg("FetchUTXOs failed") } // select N UTXOs to cover the total expense @@ -73,33 +77,28 @@ func (signer *Signer) SignWithdrawTx( if err != nil { return nil, err } - logger := signer.Logger().Std.With(). - Int64("txData.txSize", txData.txSize). - Int64("tx.size", txSize). - Uint64(logs.FieldNonce, txData.nonce). - Logger() - if txSize < common.OutboundBytesMin { // outbound shouldn't be blocked by low sizeLimit - logger.Warn().Msg("txSize is less than outboundBytesMin") - txSize = common.OutboundBytesMin - } - if txSize > common.OutboundBytesMax { // in case of accident - logger.Warn().Msgf("txSize is greater than outboundBytesMax") + logger := signer.Logger().Std.With().Uint64("tx.nonce", txData.nonce).Int64("tx.size", txSize).Logger() + if txSize > common.OutboundBytesMax { + // in case of accident + logger.Warn().Msg("tx size is greater than outboundBytesMax") txSize = common.OutboundBytesMax } // fee calculation - // #nosec G115 always in range (checked above) fees := txSize * txData.feeRate - signer.Logger(). - Std.Info(). - Msgf("bitcoin outbound nonce %d feeRate %d size %d fees %d consolidated %d utxos of value %v", - txData.nonce, txData.feeRate, txSize, fees, selected.ConsolidatedUTXOs, selected.ConsolidatedValue) // add tx outputs inputValue := selected.Value if err := signer.AddWithdrawTxOutputs(tx, txData.to, inputValue, txData.amountSats, nonceMark, fees, txData.cancelTx); err != nil { return nil, err } + signer.Logger(). + Std.Info(). + Int64("tx.rate", txData.feeRate). + Int64("tx.fees", fees). + Uint16("tx.consolidated_utxos", selected.ConsolidatedUTXOs). + Float64("tx.consolidated_value", selected.ConsolidatedValue). + Msg("signing bitcoin outbound") // sign the tx if err := signer.SignTx(ctx, tx, inAmounts, txData.height, txData.nonce); err != nil { diff --git a/zetaclient/chains/bitcoin/signer/sign_test.go b/zetaclient/chains/bitcoin/signer/sign_test.go index 5c0a0f46bf..fe68de3c9c 100644 --- a/zetaclient/chains/bitcoin/signer/sign_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_test.go @@ -6,21 +6,132 @@ import ( "reflect" "testing" + sdkmath "cosmossdk.io/math" "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/testutil/sample" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/coin" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" + observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" "github.com/zeta-chain/node/zetaclient/testutils/mocks" ) +func Test_SignWithdrawTx(t *testing.T) { + net := &chaincfg.MainNetParams + + // make sample cctx + mkCCTX := func(t *testing.T) *crosschaintypes.CrossChainTx { + cctx := sample.CrossChainTx(t, "0x123") + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().GasPrice = "10" + cctx.GetCurrentOutboundParam().Receiver = sample.BTCAddressP2WPKH(t, sample.Rand(), net).String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chains.BitcoinMainnet.ChainId + cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC + cctx.GetCurrentOutboundParam().CallOptions = &crosschaintypes.CallOptions{GasLimit: 254} + cctx.GetCurrentOutboundParam().TssNonce = 0 + return cctx + } + + // helper function to create tx data + mkTxData := func(height uint64, minRelayFee float64) signer.OutboundData { + cctx := mkCCTX(t) + txData, err := signer.NewOutboundData(cctx, height, minRelayFee, zerolog.Nop(), zerolog.Nop()) + require.NoError(t, err) + return *txData + } + + tests := []struct { + name string + chain chains.Chain + txData signer.OutboundData + failFetchUTXOs bool + failSignTx bool + fail bool + }{ + { + name: "should sign withdraw tx successfully", + chain: chains.BitcoinMainnet, + txData: mkTxData(101, 0.00001), + }, + { + name: "should fail if no UTXOs fetched due to RPC error", + chain: chains.BitcoinMainnet, + txData: mkTxData(101, 0.00001), + failFetchUTXOs: true, + fail: true, + }, + { + name: "should fail if TSS keysign fails", + chain: chains.BitcoinMainnet, + txData: mkTxData(101, 0.00001), + failSignTx: true, + fail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // ARRANGE + // setup signer + s := newTestSuite(t, tt.chain) + btcAddress, err := s.TSS().PubKey().AddressBTC(tt.chain.ChainId) + require.NoError(t, err) + tssAddress := btcAddress.EncodeAddress() + + // mock up pending nonces + pendingNonces := observertypes.PendingNonces{} + s.zetacoreClient.On("GetPendingNoncesByChain", mock.Anything, mock.Anything). + Maybe(). + Return(pendingNonces, nil) + + // mock up utxos + utxos := []btcjson.ListUnspentResult{} + utxos = append(utxos, btcjson.ListUnspentResult{Address: tssAddress, Amount: 1.0, Confirmations: 1}) + if !tt.failFetchUTXOs { + s.client.On("ListUnspentMinMaxAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(utxos, nil) + } else { + s.client.On("ListUnspentMinMaxAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("rpc error")) + } + + // mock up TSS SignBatch error + if tt.failSignTx { + s.tss.Pause() + } + + // ACT + // sign withdraw tx + ctx := context.Background() + tx, err := s.SignWithdrawTx(ctx, &tt.txData, s.observer) + + // ASSERT + if tt.fail { + require.Error(t, err) + require.Nil(t, tx) + return + } + require.NoError(t, err) + + // check tx signature + for i := range tx.TxIn { + require.Len(t, tx.TxIn[i].Witness, 2) + } + }) + } +} + func Test_AddTxInputs(t *testing.T) { r := sample.Rand() net := &chaincfg.MainNetParams diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 7e10a4cc64..0cdeb75552 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -88,7 +88,6 @@ func (signer *Signer) TryProcessOutbound( }() // prepare logger - chain := signer.Chain() params := cctx.GetCurrentOutboundParam() lf := map[string]any{ logs.FieldMethod: "TryProcessOutbound", @@ -115,7 +114,7 @@ func (signer *Signer) TryProcessOutbound( } // setup outbound data - txData, err := NewOutboundData(cctx, chain.ChainId, height, minRelayFee, logger, signer.Logger().Compliance) + txData, err := NewOutboundData(cctx, height, minRelayFee, logger, signer.Logger().Compliance) if err != nil { logger.Error().Err(err).Msg("failed to setup Bitcoin outbound data") return From 79db16e3fd2f2dcf0421daaf299cf32bb60a1410 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 30 Jan 2025 17:11:28 -0600 Subject: [PATCH 40/74] avoid creating log field map and add log fields right on the logger --- zetaclient/chains/bitcoin/signer/signer.go | 26 ++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 0cdeb75552..d24d8cf308 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -144,13 +144,12 @@ func (signer *Signer) BroadcastOutbound( txHash := tx.TxID() // prepare logger fields - lf := map[string]any{ - logs.FieldMethod: "BroadcastOutbound", - logs.FieldNonce: nonce, - logs.FieldTx: txHash, - logs.FieldCctx: cctx.Index, - } - logger := signer.Logger().Std + logger := signer.Logger().Std.With(). + Str(logs.FieldMethod, "BroadcastOutbound"). + Uint64(logs.FieldNonce, nonce). + Str(logs.FieldTx, txHash). + Str(logs.FieldCctx, cctx.Index). + Logger() // try broacasting tx with backoff in case of RPC error broadcast := func() error { @@ -160,28 +159,27 @@ func (signer *Signer) BroadcastOutbound( bo := backoff.NewConstantBackOff(broadcastBackoff) boWithMaxRetries := backoff.WithMaxRetries(bo, broadcastRetries) if err := retry.DoWithBackoff(broadcast, boWithMaxRetries); err != nil { - logger.Error().Err(err).Fields(lf).Msgf("unable to broadcast Bitcoin outbound") + logger.Error().Err(err).Msgf("unable to broadcast Bitcoin outbound") } - logger.Info().Fields(lf).Msg("broadcasted Bitcoin outbound successfully") + logger.Info().Msg("broadcasted Bitcoin outbound successfully") // save tx local db and ignore db error. // db error is not critical and should not block outbound tracker. if err := ob.SaveBroadcastedTx(txHash, nonce); err != nil { - logger.Error().Err(err).Fields(lf).Msg("unable to save broadcasted Bitcoin outbound") + logger.Error().Err(err).Msg("unable to save broadcasted Bitcoin outbound") } // add tx to outbound tracker so that all observers know about it zetaHash, err := zetacoreClient.PostOutboundTracker(ctx, ob.Chain().ChainId, nonce, txHash) if err != nil { - logger.Err(err).Fields(lf).Msg("unable to add Bitcoin outbound tracker") + logger.Err(err).Msg("unable to add Bitcoin outbound tracker") } else { - lf[logs.FieldZetaTx] = zetaHash - logger.Info().Fields(lf).Msg("add Bitcoin outbound tracker successfully") + logger.Info().Str(logs.FieldZetaTx, zetaHash).Msg("add Bitcoin outbound tracker successfully") } // try including this outbound as early as possible, no need to wait for outbound tracker _, included := ob.TryIncludeOutbound(ctx, cctx, txHash) if included { - logger.Info().Fields(lf).Msg("included newly broadcasted Bitcoin outbound") + logger.Info().Msg("included newly broadcasted Bitcoin outbound") } } From 03492462084262745db57db43f682573368568ab Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:05:26 +0100 Subject: [PATCH 41/74] fix client.GetEstimatedFeeRate --- zetaclient/chains/bitcoin/client/client.go | 8 +++++++- zetaclient/chains/bitcoin/client/helpers.go | 8 ++------ zetaclient/chains/bitcoin/client/mockgen.go | 2 +- .../chains/bitcoin/observer/gas_price.go | 2 +- .../chains/bitcoin/observer/observer.go | 2 +- zetaclient/chains/bitcoin/signer/signer.go | 2 +- zetaclient/testutils/mocks/bitcoin_client.go | 20 +++++++++---------- 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/client.go b/zetaclient/chains/bitcoin/client/client.go index c501cd5cdb..f99a95569b 100644 --- a/zetaclient/chains/bitcoin/client/client.go +++ b/zetaclient/chains/bitcoin/client/client.go @@ -38,6 +38,7 @@ import ( "github.com/rs/zerolog" "github.com/tendermint/btcd/chaincfg" + pkgchains "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/zetaclient/config" "github.com/zeta-chain/node/zetaclient/logs" "github.com/zeta-chain/node/zetaclient/metrics" @@ -48,6 +49,7 @@ type Client struct { hostURL string client *http.Client clientName string + isRegnet bool config config.BTCConfig params chains.Params logger zerolog.Logger @@ -81,7 +83,10 @@ func New(cfg config.BTCConfig, chainID int64, logger zerolog.Logger, opts ...Opt return nil, errors.Wrap(err, "unable to resolve chain params") } - clientName := fmt.Sprintf("btc:%d", chainID) + var ( + clientName = fmt.Sprintf("btc:%d", chainID) + isRegnet = pkgchains.IsBitcoinRegnet(chainID) + ) c := &Client{ hostURL: normalizeHostURL(cfg.RPCHost, true), @@ -89,6 +94,7 @@ func New(cfg config.BTCConfig, chainID int64, logger zerolog.Logger, opts ...Opt config: cfg, params: params, clientName: clientName, + isRegnet: isRegnet, logger: logger.With(). Str(logs.FieldModule, "btc_client"). Int64(logs.FieldChain, chainID). diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index adb079486b..c6128b73e2 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -115,13 +115,9 @@ func (c *Client) GetRawTransactionResult(ctx context.Context, } // GetEstimatedFeeRate gets estimated smart fee rate (sat/vB) targeting given block confirmation -func (c *Client) GetEstimatedFeeRate( - ctx context.Context, - confTarget int64, - regnet bool, -) (satsPerByte int64, err error) { +func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (satsPerByte int64, err error) { // RPC 'EstimateSmartFee' is not available in regnet - if regnet { + if c.isRegnet { return FeeRateRegnet, nil } diff --git a/zetaclient/chains/bitcoin/client/mockgen.go b/zetaclient/chains/bitcoin/client/mockgen.go index 69cfe1cd6b..ca7093aa73 100644 --- a/zetaclient/chains/bitcoin/client/mockgen.go +++ b/zetaclient/chains/bitcoin/client/mockgen.go @@ -45,7 +45,7 @@ type client interface { SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*hash.Hash, error) - GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) GetTransactionFeeAndRate(ctx context.Context, tx *types.TxRawResult) (int64, int64, error) EstimateSmartFee( ctx context.Context, diff --git a/zetaclient/chains/bitcoin/observer/gas_price.go b/zetaclient/chains/bitcoin/observer/gas_price.go index ea6d85e6bd..8071702a2c 100644 --- a/zetaclient/chains/bitcoin/observer/gas_price.go +++ b/zetaclient/chains/bitcoin/observer/gas_price.go @@ -30,7 +30,7 @@ func (ob *Observer) PostGasPrice(ctx context.Context) error { return errors.Wrapf(err, "unable to get recent fee rate") } case chains.NetworkType_mainnet: - feeRateEstimated, err = ob.rpc.GetEstimatedFeeRate(ctx, 1, false) + feeRateEstimated, err = ob.rpc.GetEstimatedFeeRate(ctx, 1) if err != nil { return errors.Wrap(err, "unable to get estimated fee rate") } diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 7b26f9998e..1718b7b195 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -35,7 +35,7 @@ type RPC interface { res *btcjson.GetTransactionResult, ) (btcjson.TxRawResult, error) - GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) GetTransactionFeeAndRate(ctx context.Context, tx *btcjson.TxRawResult) (int64, int64, error) EstimateSmartFee( diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index d24d8cf308..6884336d5a 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -33,7 +33,7 @@ const ( type RPC interface { GetNetworkInfo(ctx context.Context) (*btcjson.GetNetworkInfoResult, error) GetRawTransaction(ctx context.Context, hash *chainhash.Hash) (*btcutil.Tx, error) - GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*chainhash.Hash, error) } diff --git a/zetaclient/testutils/mocks/bitcoin_client.go b/zetaclient/testutils/mocks/bitcoin_client.go index e921e1a54a..5f76cdbc42 100644 --- a/zetaclient/testutils/mocks/bitcoin_client.go +++ b/zetaclient/testutils/mocks/bitcoin_client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.51.0. DO NOT EDIT. +// Code generated by mockery v2.45.0. DO NOT EDIT. package mocks @@ -357,9 +357,9 @@ func (_m *BitcoinClient) GetBlockVerboseByStr(ctx context.Context, blockHash str return r0, r1 } -// GetEstimatedFeeRate provides a mock function with given fields: ctx, confTarget, regnet -func (_m *BitcoinClient) GetEstimatedFeeRate(ctx context.Context, confTarget int64, regnet bool) (int64, error) { - ret := _m.Called(ctx, confTarget, regnet) +// GetEstimatedFeeRate provides a mock function with given fields: ctx, confTarget +func (_m *BitcoinClient) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) { + ret := _m.Called(ctx, confTarget) if len(ret) == 0 { panic("no return value specified for GetEstimatedFeeRate") @@ -367,17 +367,17 @@ func (_m *BitcoinClient) GetEstimatedFeeRate(ctx context.Context, confTarget int var r0 int64 var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int64, bool) (int64, error)); ok { - return rf(ctx, confTarget, regnet) + if rf, ok := ret.Get(0).(func(context.Context, int64) (int64, error)); ok { + return rf(ctx, confTarget) } - if rf, ok := ret.Get(0).(func(context.Context, int64, bool) int64); ok { - r0 = rf(ctx, confTarget, regnet) + if rf, ok := ret.Get(0).(func(context.Context, int64) int64); ok { + r0 = rf(ctx, confTarget) } else { r0 = ret.Get(0).(int64) } - if rf, ok := ret.Get(1).(func(context.Context, int64, bool) error); ok { - r1 = rf(ctx, confTarget, regnet) + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, confTarget) } else { r1 = ret.Error(1) } From 6e8f9005c5990cd37af15f22275cb3f3f72d5b52 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:11:56 +0100 Subject: [PATCH 42/74] Fix loadBroadcastedTxMap --- zetaclient/chains/bitcoin/observer/db.go | 13 ++++++++----- zetaclient/chains/bitcoin/observer/db_test.go | 4 ---- zetaclient/chains/bitcoin/observer/observer.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/db.go b/zetaclient/chains/bitcoin/observer/db.go index 122e8d2aad..0ea2c5fdc4 100644 --- a/zetaclient/chains/bitcoin/observer/db.go +++ b/zetaclient/chains/bitcoin/observer/db.go @@ -58,16 +58,19 @@ func (ob *Observer) LoadLastBlockScanned(ctx context.Context) error { return nil } -// LoadBroadcastedTxMap loads broadcasted transactions from the database -func (ob *Observer) LoadBroadcastedTxMap() error { +// loadBroadcastedTxMap loads broadcasted transactions from the database +func (ob *Observer) loadBroadcastedTxMap() error { var broadcastedTransactions []clienttypes.OutboundHashSQLType - if err := ob.DB().Client().Find(&broadcastedTransactions).Error; err != nil { - ob.logger.Chain.Error().Err(err).Msgf("error iterating over db for chain %d", ob.Chain().ChainId) - return err + + tx := ob.DB().Client().Find(&broadcastedTransactions) + if tx.Error != nil { + return errors.Wrap(tx.Error, "unable to find broadcasted txs") } + for _, entry := range broadcastedTransactions { ob.tssOutboundHashes[entry.Hash] = true ob.broadcastedTx[entry.Key] = entry.Hash } + return nil } diff --git a/zetaclient/chains/bitcoin/observer/db_test.go b/zetaclient/chains/bitcoin/observer/db_test.go index 86bc1d3989..dc9760c8a5 100644 --- a/zetaclient/chains/bitcoin/observer/db_test.go +++ b/zetaclient/chains/bitcoin/observer/db_test.go @@ -132,10 +132,6 @@ func Test_LoadBroadcastedTxMap(t *testing.T) { // create new observer using same db path obNew := newTestSuite(t, chains.BitcoinMainnet, optDBPath(dbPath)) - // load broadcasted tx map to new observer - err := obNew.LoadBroadcastedTxMap() - require.NoError(t, err) - // check if the txHash is a TSS outbound require.True(t, obNew.IsTSSTransaction(txHash)) diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 1718b7b195..6d9e59dd2b 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -148,7 +148,7 @@ func New(chain chains.Chain, baseObserver *base.Observer, rpc RPC) (*Observer, e } // load broadcasted transactions - if err = ob.LoadBroadcastedTxMap(); err != nil { + if err = ob.loadBroadcastedTxMap(); err != nil { return nil, errors.Wrap(err, "unable to load broadcasted tx map") } From 8cddb38935c705b18439b0ca4df83f40330de97c Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:00:35 +0100 Subject: [PATCH 43/74] Fix SelectedUTXOs --- zetaclient/chains/bitcoin/observer/db_test.go | 6 +- .../chains/bitcoin/observer/observer_test.go | 69 +++++++++++-------- .../chains/bitcoin/observer/outbound.go | 51 +++++++------- zetaclient/chains/bitcoin/observer/utxos.go | 3 +- .../chains/bitcoin/observer/utxos_test.go | 32 ++++----- zetaclient/chains/bitcoin/signer/sign.go | 1 - 6 files changed, 86 insertions(+), 76 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/db_test.go b/zetaclient/chains/bitcoin/observer/db_test.go index dc9760c8a5..1a3ad0c206 100644 --- a/zetaclient/chains/bitcoin/observer/db_test.go +++ b/zetaclient/chains/bitcoin/observer/db_test.go @@ -37,7 +37,7 @@ func Test_SaveBroadcastedTx(t *testing.T) { nonce := uint64(1) txHash := sample.BtcHash().String() dbPath := sample.CreateTempDir(t) - ob := newTestSuite(t, chains.BitcoinMainnet, optDBPath(dbPath)) + ob := newTestSuite(t, chains.BitcoinMainnet, withDatabasePath(dbPath)) if tt.wantErr != "" { // delete db to simulate db error os.RemoveAll(dbPath) @@ -126,11 +126,11 @@ func Test_LoadBroadcastedTxMap(t *testing.T) { // create observer and save a test tx dbPath := sample.CreateTempDir(t) - obOld := newTestSuite(t, chains.BitcoinMainnet, optDBPath(dbPath)) + obOld := newTestSuite(t, chains.BitcoinMainnet, withDatabasePath(dbPath)) obOld.SaveBroadcastedTx(txHash, nonce) // create new observer using same db path - obNew := newTestSuite(t, chains.BitcoinMainnet, optDBPath(dbPath)) + obNew := newTestSuite(t, chains.BitcoinMainnet, withDatabasePath(dbPath)) // check if the txHash is a TSS outbound require.True(t, obNew.IsTSSTransaction(txHash)) diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index cf86f67ed5..de0453481f 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -2,6 +2,7 @@ package observer_test import ( "context" + "errors" "math/big" "os" "strconv" @@ -17,6 +18,7 @@ import ( "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/testutil/sample" + "github.com/zeta-chain/node/x/crosschain/types" observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" @@ -266,31 +268,30 @@ type testSuite struct { client *mocks.BitcoinClient zetacore *mocks.ZetacoreClient db *db.DB - dbPath string } -type Opt func(t *testSuite) +type testSuiteOpts struct { + dbPath string +} -// optDBPath is an option to set custom db path -func optDBPath(dbPath string) Opt { - return func(t *testSuite) { - t.dbPath = dbPath - } +type opt func(t *testSuiteOpts) + +// withDatabasePath is an option to set custom db path +func withDatabasePath(dbPath string) opt { + return func(t *testSuiteOpts) { t.dbPath = dbPath } } -func newTestSuite(t *testing.T, chain chains.Chain, opts ...Opt) *testSuite { +func newTestSuite(t *testing.T, chain chains.Chain, opts ...opt) *testSuite { // create test suite with options - s := &testSuite{ctx: context.Background()} + var testOpts testSuiteOpts for _, opt := range opts { - opt(s) + opt(&testOpts) } require.True(t, chain.IsBitcoinChain()) chainParams := mocks.MockChainParams(chain.ChainId, 10) client := mocks.NewBitcoinClient(t) - client.On("GetBlockCount", mock.Anything).Maybe().Return(int64(100), nil).Maybe() - zetacore := mocks.NewZetacoreClient(t) var tss interfaces.TSSSigner @@ -300,20 +301,22 @@ func newTestSuite(t *testing.T, chain chains.Chain, opts ...Opt) *testSuite { tss = mocks.NewTSS(t).FakePubKey(testutils.TSSPubkeyAthens3) } - // create test database - var err error + // create logger + logger := testlog.New(t) + baseLogger := base.Logger{Std: logger.Logger, Compliance: logger.Logger} + var database *db.DB - if s.dbPath == "" { + var err error + if testOpts.dbPath == "" { database, err = db.NewFromSqliteInMemory(true) + require.NoError(t, err) } else { - database, err = db.NewFromSqlite(s.dbPath, "test.db", true) - t.Cleanup(func() { os.RemoveAll(s.dbPath) }) + database, err = db.NewFromSqlite(testOpts.dbPath, "test.db", true) + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(testOpts.dbPath) }) } - require.NoError(t, err) - // create logger - logger := testlog.New(t) - baseLogger := base.Logger{Std: logger.Logger, Compliance: logger.Logger} + client.On("GetBlockCount", mock.Anything).Maybe().Return(int64(100), nil).Maybe() baseObserver, err := base.NewObserver( chain, @@ -330,11 +333,23 @@ func newTestSuite(t *testing.T, chain chains.Chain, opts ...Opt) *testSuite { ob, err := observer.New(chain, baseObserver, client) require.NoError(t, err) - // set test suite fields - s.Observer = ob - s.client = client - s.zetacore = zetacore - s.db = database + ts := &testSuite{ + ctx: context.Background(), + client: client, + zetacore: zetacore, + db: database, + Observer: ob, + } + + ts.zetacore. + On("GetCctxByNonce", mock.Anything, mock.Anything, mock.Anything). + Return(ts.mockGetCCTXByNonce). + Maybe() + + return ts +} - return s +func (ts *testSuite) mockGetCCTXByNonce(_ context.Context, chainID int64, nonce uint64) (*types.CrossChainTx, error) { + // implement custom logic here if needed (e.g. mock) + return nil, errors.New("not implemented") } diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index 290e987ba4..56dfdef206 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -233,7 +233,7 @@ func (ob *Observer) refreshPendingNonce(ctx context.Context) { nonceLow := uint64(p.NonceLow) if nonceLow > ob.GetPendingNonce() { // get the last included outbound hash - txid, err := ob.getOutboundHashByNonce(ctx, nonceLow-1, false) + txid, err := ob.getOutboundHashByNonce(ctx, nonceLow-1) if err != nil { logger.Error().Err(err).Msg("error getting last outbound txid") } @@ -246,38 +246,35 @@ func (ob *Observer) refreshPendingNonce(ctx context.Context) { // getOutboundHashByNonce gets the outbound hash for given nonce. // test is true for unit test only -func (ob *Observer) getOutboundHashByNonce(ctx context.Context, nonce uint64, test bool) (string, error) { +func (ob *Observer) getOutboundHashByNonce(ctx context.Context, nonce uint64) (string, error) { // There are 2 types of txids an observer can trust // 1. The ones had been verified and saved by observer self. // 2. The ones had been finalized in zetacore based on majority vote. if res := ob.GetIncludedTx(nonce); res != nil { return res.TxID, nil } - if !test { // if not unit test, get cctx from zetacore - send, err := ob.ZetacoreClient().GetCctxByNonce(ctx, ob.Chain().ChainId, nonce) - if err != nil { - return "", errors.Wrapf(err, "getOutboundHashByNonce: error getting cctx for nonce %d", nonce) - } - txid := send.GetCurrentOutboundParam().Hash - if txid == "" { - return "", fmt.Errorf("getOutboundHashByNonce: cannot find outbound txid for nonce %d", nonce) - } - // make sure it's a real Bitcoin txid - _, getTxResult, err := ob.rpc.GetTransactionByStr(ctx, txid) - if err != nil { - return "", errors.Wrapf( - err, - "getOutboundHashByNonce: error getting outbound result for nonce %d hash %s", - nonce, - txid, - ) - } - if getTxResult.Confirmations <= 0 { // just a double check - return "", fmt.Errorf("getOutboundHashByNonce: outbound txid %s for nonce %d is not included", txid, nonce) - } - return txid, nil + + send, err := ob.ZetacoreClient().GetCctxByNonce(ctx, ob.Chain().ChainId, nonce) + if err != nil { + return "", errors.Wrapf(err, "error getting cctx for nonce %d", nonce) + } + + txid := send.GetCurrentOutboundParam().Hash + if txid == "" { + return "", fmt.Errorf("cannot find outbound txid for nonce %d", nonce) + } + + // make sure it's a real Bitcoin txid + _, getTxResult, err := ob.rpc.GetTransactionByStr(ctx, txid) + switch { + case err != nil: + return "", errors.Wrapf(err, "error getting outbound result for nonce %d hash %s", nonce, txid) + case getTxResult.Confirmations <= 0: + // just a double check + return "", fmt.Errorf("outbound txid %s for nonce %d is not included", txid, nonce) } - return "", fmt.Errorf("getOutboundHashByNonce: cannot find outbound txid for nonce %d", nonce) + + return txid, nil } // checkTxInclusion checks if a txHash is included and returns (txResult, included) @@ -432,7 +429,7 @@ func (ob *Observer) checkTSSVin(ctx context.Context, vins []btcjson.Vin, nonce u } // 1st vin: nonce-mark MUST come from prior TSS outbound if nonce > 0 && i == 0 { - preTxid, err := ob.getOutboundHashByNonce(ctx, nonce-1, false) + preTxid, err := ob.getOutboundHashByNonce(ctx, nonce-1) if err != nil { return fmt.Errorf("checkTSSVin: error findTxIDByNonce %d", nonce-1) } diff --git a/zetaclient/chains/bitcoin/observer/utxos.go b/zetaclient/chains/bitcoin/observer/utxos.go index c3f86673e2..84905cc779 100644 --- a/zetaclient/chains/bitcoin/observer/utxos.go +++ b/zetaclient/chains/bitcoin/observer/utxos.go @@ -92,7 +92,6 @@ func (ob *Observer) SelectUTXOs( utxosToSpend uint16, nonce uint64, consolidateRank uint16, - test bool, ) (SelectedUTXOs, error) { idx := -1 if nonce == 0 { @@ -101,7 +100,7 @@ func (ob *Observer) SelectUTXOs( defer ob.Mu().Unlock() } else { // for nonce > 0; we proceed only when we see the nonce-mark utxo - preTxid, err := ob.getOutboundHashByNonce(ctx, nonce-1, test) + preTxid, err := ob.getOutboundHashByNonce(ctx, nonce-1) if err != nil { return SelectedUTXOs{}, err } diff --git a/zetaclient/chains/bitcoin/observer/utxos_test.go b/zetaclient/chains/bitcoin/observer/utxos_test.go index b00109b51f..57d32a43e7 100644 --- a/zetaclient/chains/bitcoin/observer/utxos_test.go +++ b/zetaclient/chains/bitcoin/observer/utxos_test.go @@ -31,7 +31,7 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 0.01, nonce = 0 // output: [0.01], 0.01 ob, utxos := newTestSuitWithUTXOs(t) - selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 0, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 0, math.MaxUint16) require.NoError(t, err) require.Equal(t, 0.01, selected.Value) require.Equal(t, utxos[0:1], selected.UTXOs) @@ -41,18 +41,18 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 0.5, nonce = 1 // output: error ob, _ := newTestSuitWithUTXOs(t) - selected, err := ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16) require.Error(t, err) require.Nil(t, selected.UTXOs) require.Zero(t, selected.Value) - require.ErrorContains(t, err, "cannot find outbound txid for nonce 0") + require.ErrorContains(t, err, "error getting cctx for nonce 0") }) t.Run("nonce = 1, should pass when nonce mark 0 is set", func(t *testing.T) { // input: utxoCap = 5, amount = 0.5, nonce = 1 // output: [0.00002, 0.01, 0.12, 0.18, 0.24], 0.55002 ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 0, dummyTxID) - selected, err := ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 0.5, 5, 1, math.MaxUint16) require.NoError(t, err) require.Equal(t, 0.55002, selected.Value) require.Equal(t, utxos[0:5], selected.UTXOs) @@ -62,7 +62,7 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 1.0, nonce = 2 // output: [0.00002001, 0.01, 0.12, 0.18, 0.24, 0.5], 1.05002001 ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 1, dummyTxID) - selected, err := ob.SelectUTXOs(ctx, 1.0, 5, 2, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 1.0, 5, 2, math.MaxUint16) require.NoError(t, err) require.InEpsilon(t, 1.05002001, selected.Value, 1e-8) require.Equal(t, utxos[0:6], selected.UTXOs) @@ -72,7 +72,7 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 8.05, nonce = 3 // output: [0.00002002, 0.24, 0.5, 1.26, 2.97, 3.28], 8.25002002 ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 2, dummyTxID) - selected, err := ob.SelectUTXOs(ctx, 8.05, 5, 3, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 8.05, 5, 3, math.MaxUint16) require.NoError(t, err) require.InEpsilon(t, 8.25002002, selected.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{utxos[0]}, utxos[4:9]...) @@ -83,7 +83,7 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 0.503, nonce = 24105432 // output: [0.24107432, 0.01, 0.12, 0.18, 0.24], 0.7910731 ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105431, dummyTxID) - selected, err := ob.SelectUTXOs(ctx, 0.503, 5, 24105432, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 0.503, 5, 24105432, math.MaxUint16) require.NoError(t, err) require.InEpsilon(t, 0.79107431, selected.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[0:4]...) @@ -94,7 +94,7 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 1.0, nonce = 24105433 // output: [0.24107432, 0.12, 0.18, 0.24, 0.5], 1.28107432 ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105432, dummyTxID) - selected, err := ob.SelectUTXOs(ctx, 1.0, 5, 24105433, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 1.0, 5, 24105433, math.MaxUint16) require.NoError(t, err) require.InEpsilon(t, 1.28107432, selected.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[1:4]...) @@ -106,7 +106,7 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 16.03 // output: [0.24107432, 1.26, 2.97, 3.28, 5.16, 8.72], 21.63107432 ob, utxos := createTestSuitWithUTXOsAndNonceMark(t, 24105432, dummyTxID) - selected, err := ob.SelectUTXOs(ctx, 16.03, 5, 24105433, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 16.03, 5, 24105433, math.MaxUint16) require.NoError(t, err) require.InEpsilon(t, 21.63107432, selected.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[6:11]...) @@ -117,7 +117,7 @@ func Test_SelectUTXOs(t *testing.T) { // input: utxoCap = 5, amount = 21.64 // output: error ob, _ := createTestSuitWithUTXOsAndNonceMark(t, 24105432, dummyTxID) - selected, err := ob.SelectUTXOs(ctx, 21.64, 5, 24105433, math.MaxUint16, true) + selected, err := ob.SelectUTXOs(ctx, 21.64, 5, 24105433, math.MaxUint16) require.Error(t, err) require.Nil(t, selected.UTXOs) require.Zero(t, selected.Value) @@ -135,7 +135,7 @@ func Test_SelectUTXOs_Consolidation(t *testing.T) { // input: utxoCap = 10, amount = 0.01, nonce = 1, rank = 10 // output: [0.00002, 0.01], 0.01002 - res, err := ob.SelectUTXOs(ctx, 0.01, 10, 1, 10, true) + res, err := ob.SelectUTXOs(ctx, 0.01, 10, 1, 10) require.NoError(t, err) require.Equal(t, 0.01002, res.Value) require.Equal(t, utxos[0:2], res.UTXOs) @@ -149,7 +149,7 @@ func Test_SelectUTXOs_Consolidation(t *testing.T) { // input: utxoCap = 9, amount = 0.01, nonce = 1, rank = 9 // output: [0.00002, 0.01, 0.12], 0.13002 - selected, err := ob.SelectUTXOs(ctx, 0.01, 9, 1, 9, true) + selected, err := ob.SelectUTXOs(ctx, 0.01, 9, 1, 9) require.NoError(t, err) require.Equal(t, 0.13002, selected.Value) require.Equal(t, utxos[0:3], selected.UTXOs) @@ -163,7 +163,7 @@ func Test_SelectUTXOs_Consolidation(t *testing.T) { // input: utxoCap = 5, amount = 0.01, nonce = 0, rank = 5 // output: [0.00002, 0.014, 1.26, 0.5, 0.2], 2.01002 - selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 1, 5, true) + selected, err := ob.SelectUTXOs(ctx, 0.01, 5, 1, 5) require.NoError(t, err) require.Equal(t, 2.01002, selected.Value) expected := make([]btcjson.ListUnspentResult, 2) @@ -182,7 +182,7 @@ func Test_SelectUTXOs_Consolidation(t *testing.T) { // input: utxoCap = 12, amount = 0.01, nonce = 0, rank = 1 // output: [0.00002, 0.01, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18, 0.12], 22.44002 - selected, err := ob.SelectUTXOs(ctx, 0.01, 12, 1, 1, true) + selected, err := ob.SelectUTXOs(ctx, 0.01, 12, 1, 1) require.NoError(t, err) require.Equal(t, 22.44002, selected.Value) expected := make([]btcjson.ListUnspentResult, 2) @@ -201,7 +201,7 @@ func Test_SelectUTXOs_Consolidation(t *testing.T) { // input: utxoCap = 5, amount = 0.13, nonce = 24105432, rank = 5 // output: [0.24107431, 0.01, 0.12, 1.26, 0.5, 0.24], 2.37107431 - selected, err := ob.SelectUTXOs(ctx, 0.13, 5, 24105432, 5, true) + selected, err := ob.SelectUTXOs(ctx, 0.13, 5, 24105432, 5) require.NoError(t, err) require.InEpsilon(t, 2.37107431, selected.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[0:2]...) @@ -219,7 +219,7 @@ func Test_SelectUTXOs_Consolidation(t *testing.T) { // input: utxoCap = 12, amount = 0.13, nonce = 24105432, rank = 1 // output: [0.24107431, 0.01, 0.12, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18], 22.68107431 - selected, err := ob.SelectUTXOs(ctx, 0.13, 12, 24105432, 1, true) + selected, err := ob.SelectUTXOs(ctx, 0.13, 12, 24105432, 1) require.NoError(t, err) require.InEpsilon(t, 22.68107431, selected.Value, 1e-8) expected := append([]btcjson.ListUnspentResult{utxos[4]}, utxos[0:2]...) diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index 229fcf094c..de76cb3057 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -58,7 +58,6 @@ func (signer *Signer) SignWithdrawTx( MaxNoOfInputsPerTx, txData.nonce, consolidationRank, - false, ) if err != nil { return nil, err From 01e50dfbd6b4bce7ff1e8b1a22e6cda61e3be2db Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:04:25 +0100 Subject: [PATCH 44/74] Fix log naming --- zetaclient/chains/bitcoin/observer/outbound.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index 56dfdef206..6b331969e9 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -220,7 +220,7 @@ func (ob *Observer) VoteOutboundIfConfirmed(ctx context.Context, cctx *crosschai // 1. The zetaclient gets restarted. // 2. The tracker is missing in zetacore. func (ob *Observer) refreshPendingNonce(ctx context.Context) { - logger := ob.logger.Outbound.With().Str(logs.FieldMethod, "refreshPendingNonce").Logger() + logger := ob.logger.Outbound.With().Str(logs.FieldMethod, "refresh_pending_nonce").Logger() // get pending nonces from zetacore p, err := ob.ZetacoreClient().GetPendingNoncesByChain(ctx, ob.Chain().ChainId) From d556b7e81992c6f275a76aa0fc96c801074a19c1 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:05:40 +0100 Subject: [PATCH 45/74] fix e2e logging --- e2e/runner/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/runner/run.go b/e2e/runner/run.go index d9cefe58ba..318e5734c0 100644 --- a/e2e/runner/run.go +++ b/e2e/runner/run.go @@ -30,7 +30,7 @@ func (r *E2ERunner) RunE2ETests(e2eTests []E2ETest) (err error) { func (r *E2ERunner) RunE2ETest(e2eTest E2ETest, checkAccounting bool) error { startTime := time.Now() // note: spacing is padded to width of completed message - r.Logger.Print("⏳ running - %s", e2eTest.Description) + r.Logger.Print("⏳ running - %s", e2eTest.Name) // run e2e test, if args are not provided, use default args args := e2eTest.Args From 4c6b5bb962e612b3cc4b875a959842a0065bcbff Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:17:38 +0100 Subject: [PATCH 46/74] Fix setPendingNonce --- zetaclient/chains/bitcoin/observer/observer.go | 3 +-- zetaclient/chains/bitcoin/observer/observer_test.go | 13 ------------- zetaclient/chains/bitcoin/observer/outbound.go | 2 +- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 6d9e59dd2b..4b6cfec6f4 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -163,8 +163,7 @@ func (ob *Observer) GetPendingNonce() uint64 { return ob.pendingNonce } -// SetPendingNonce sets the artificial pending nonce -func (ob *Observer) SetPendingNonce(nonce uint64) { +func (ob *Observer) setPendingNonce(nonce uint64) { ob.Mu().Lock() defer ob.Mu().Unlock() ob.pendingNonce = nonce diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index de0453481f..beac15e0b4 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -206,19 +206,6 @@ func Test_BlockCache(t *testing.T) { }) } -func Test_SetPendingNonce(t *testing.T) { - // create observer - ob := newTestSuite(t, chains.BitcoinMainnet) - - // ensure pending nonce is 0 - require.Zero(t, ob.GetPendingNonce()) - - // set and get pending nonce - nonce := uint64(100) - ob.SetPendingNonce(nonce) - require.Equal(t, nonce, ob.GetPendingNonce()) -} - func TestConfirmationThreshold(t *testing.T) { chain := chains.BitcoinMainnet ob := newTestSuite(t, chain) diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index 6b331969e9..78b7f0bd97 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -239,7 +239,7 @@ func (ob *Observer) refreshPendingNonce(ctx context.Context) { } // set 'NonceLow' as the new pending nonce - ob.SetPendingNonce(nonceLow) + ob.setPendingNonce(nonceLow) logger.Info().Uint64("pending_nonce", nonceLow).Str(logs.FieldTx, txid).Msg("increased pending nonce") } } From 3ab710ec7cb5d868fdbc48dc7e990ef5ff5f6b59 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 31 Jan 2025 12:45:10 -0600 Subject: [PATCH 47/74] don't use GasPriorityFee as it's always empty --- .../chains/bitcoin/signer/outbound_data.go | 9 +------ .../bitcoin/signer/outbound_data_test.go | 27 ------------------- .../chains/bitcoin/signer/signer_test.go | 4 ++- 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index 4867456822..ed62195707 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -62,19 +62,12 @@ func NewOutboundData( return nil, fmt.Errorf("invalid coin type %s", cctx.InboundParams.CoinType.String()) } - // initial fee rate + // parse fee rate feeRate, err := strconv.ParseInt(params.GasPrice, 10, 64) if err != nil || feeRate <= 0 { return nil, fmt.Errorf("invalid fee rate %s", params.GasPrice) } - // use current gas rate if fed by zetacore - newRate, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) - if err == nil && newRate > 0 && newRate != feeRate { - logger.Info().Msgf("use new fee rate %d sat/vB instead of %d sat/vB", newRate, feeRate) - feeRate = newRate - } - // check receiver address to, err := chains.DecodeBtcAddress(params.Receiver, params.ReceiverChainId) if err != nil { diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index e6f4678d61..faf94920f1 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -63,33 +63,6 @@ func Test_NewOutboundData(t *testing.T) { }, errMsg: "", }, - { - name: "create new outbound data using current gas rate instead of old rate", - cctx: sample.CrossChainTx(t, "0x123"), - cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { - cctx.InboundParams.CoinType = coin.CoinType_Gas - cctx.GetCurrentOutboundParam().Receiver = receiver.String() - cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId - cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC - cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes - cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte - cctx.GetCurrentOutboundParam().GasPriorityFee = "15" // 15 sats/vByte - cctx.GetCurrentOutboundParam().TssNonce = 1 - }, - height: 101, - minRelayFee: 0.00001, // 1000 sat/KB - expected: &OutboundData{ - to: receiver, - amount: 0.1, - amountSats: 10000000, - feeRate: 16, // 15 + 1 (minRelayFee) - txSize: 254, - nonce: 1, - height: 101, - cancelTx: false, - }, - errMsg: "", - }, { name: "cctx is nil", cctx: nil, diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index dcc30644f6..a6c65e6932 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -18,6 +18,7 @@ import ( "github.com/stretchr/testify/require" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/testutil/sample" observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" @@ -327,7 +328,8 @@ func makeCtx(t *testing.T) context.Context { map[int64]*observertypes.ChainParams{ chain.ChainId: &btcParams, }, - observertypes.CrosschainFlags{}, + *sample.CrosschainFlags(), + sample.OperationalFlags(), ) require.NoError(t, err, "unable to update app context") From 03d366c84c7dccd5da21f2d67647a49059fd1e8a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Mon, 3 Feb 2025 13:32:46 -0600 Subject: [PATCH 48/74] check if fee rate is bumped or not in zetaclient --- .../chains/bitcoin/signer/outbound_data.go | 19 +++++++++--- .../bitcoin/signer/outbound_data_test.go | 30 ++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index 99e2851688..c594cf4a03 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -31,6 +31,9 @@ type OutboundData struct { // feeRate is the fee rate in satoshis/vByte feeRate int64 + // feeRateBumpped is a flag to indicate if the fee rate in CCTX is bumped by zetacore + feeRateBumped bool + // txSize is the average size of a BTC outbound transaction // user is charged (in ZRC20 contract) at a static txSize on each withdrawal txSize int64 @@ -68,6 +71,13 @@ func NewOutboundData( return nil, fmt.Errorf("invalid fee rate %s", params.GasPrice) } + // check if the fee rate is bumped by zetacore + // 'GasPriorityFee' is always empty for Bitcoin unless zetacore bumps the fee rate + feeRateBumped := params.GasPrice == params.GasPriorityFee + if feeRateBumped { + logger.Info().Msgf("fee rate is bumped by zetacore: %s", params.GasPriorityFee) + } + // apply outbound fee rate multiplier feeRate = common.OutboundFeeRateFromCCTXRate(feeRate) @@ -122,10 +132,11 @@ func NewOutboundData( } return &OutboundData{ - to: to, - amount: amount, - amountSats: amountSats, - feeRate: feeRate, + to: to, + amount: amount, + amountSats: amountSats, + feeRate: feeRate, + feeRateBumped: feeRateBumped, // #nosec G115 checked in range txSize: int64(params.CallOptions.GasLimit), nonce: params.TssNonce, diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index 950713c534..115e2a1ab9 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -38,7 +38,7 @@ func Test_NewOutboundData(t *testing.T) { errMsg string }{ { - name: "create new outbound data successfully", + name: "create new outbound data successfully, no fee bump", cctx: sample.CrossChainTx(t, "0x123"), cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { cctx.InboundParams.CoinType = coin.CoinType_Gas @@ -63,6 +63,34 @@ func Test_NewOutboundData(t *testing.T) { }, errMsg: "", }, + { + name: "create new outbound data successfully, fee bumped", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().Receiver = receiver.String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId + cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC + cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes + cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().GasPriorityFee = "10" // 10 sats/vByte, bumped by zetacore + cctx.GetCurrentOutboundParam().TssNonce = 1 + }, + height: 101, + minRelayFee: 0.00001, // 1000 sat/KB + expected: &OutboundData{ + to: receiver, + amount: 0.1, + amountSats: 10000000, + feeRate: 8, // Round(7.5) + feeRateBumped: true, + txSize: 254, + nonce: 1, + height: 101, + cancelTx: false, + }, + errMsg: "", + }, { name: "cctx is nil", cctx: nil, From f9528484356d6392638d9a8f85166c0b864d06fb Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Mon, 17 Mar 2025 23:38:15 -0500 Subject: [PATCH 49/74] revert a few renamings because we will continue to use one single BTC gas price multiplier --- pkg/math/integer.go | 2 -- zetaclient/chains/bitcoin/client/client_test.go | 2 +- zetaclient/chains/bitcoin/common/fee.go | 4 ++-- zetaclient/chains/bitcoin/observer/inbound_test.go | 11 ++++++----- zetaclient/chains/bitcoin/observer/witness_test.go | 2 +- zetaclient/common/constant.go | 12 ++++++------ zetaclient/zetacore/client_vote.go | 2 +- zetaclient/zetacore/tx.go | 10 +++++----- zetaclient/zetacore/tx_test.go | 4 ++-- 9 files changed, 24 insertions(+), 25 deletions(-) diff --git a/pkg/math/integer.go b/pkg/math/integer.go index 8d74b5dc27..dcd858f55d 100644 --- a/pkg/math/integer.go +++ b/pkg/math/integer.go @@ -9,8 +9,6 @@ import ( // IncreaseIntByPercent is a function that increases integer by a percentage. // Example1: IncreaseIntByPercent(10, 15) = 10 * 1.15 = 11 // Example2: IncreaseIntByPercent(-10, 15) = -10 * 1.15 = -11 -// -// Note: use with caution if passing negative values. func IncreaseIntByPercent(value int64, percent uint32) int64 { if percent == 0 { return value diff --git a/zetaclient/chains/bitcoin/client/client_test.go b/zetaclient/chains/bitcoin/client/client_test.go index 87e9ee003a..55091ebbef 100644 --- a/zetaclient/chains/bitcoin/client/client_test.go +++ b/zetaclient/chains/bitcoin/client/client_test.go @@ -464,7 +464,7 @@ func TestClientLive(t *testing.T) { // the actual fee rate is 860 sat/vByte // #nosec G115 always in range - expectedRate := int64(float64(860) * common.BTCGasPriceMultiplierFeeCharge) + expectedRate := int64(float64(860) * common.BTCOutboundGasPriceMultiplier) expectedFee := btc.DepositorFee(expectedRate) require.Equal(t, expectedFee, depositorFee) }) diff --git a/zetaclient/chains/bitcoin/common/fee.go b/zetaclient/chains/bitcoin/common/fee.go index 92cc9d1083..6e87b09fec 100644 --- a/zetaclient/chains/bitcoin/common/fee.go +++ b/zetaclient/chains/bitcoin/common/fee.go @@ -173,7 +173,7 @@ func DepositorFee(satPerByte int64) float64 { // // TSS will make profit from the difference between fee charged from users and fee paid to Bitcoin network func OutboundFeeRateFromCCTXRate(cctxRate int64) int64 { - marketRate := float64(cctxRate) / common.BTCGasPriceMultiplierFeeCharge + marketRate := float64(cctxRate) / common.BTCOutboundGasPriceMultiplier return int64(math.Round(marketRate * common.BTCGasPriceMultiplierSendTx)) } @@ -260,7 +260,7 @@ func CalcDepositorFee( // apply gas price multiplier // #nosec G115 always in range - feeRate = int64(float64(feeRate) * common.BTCGasPriceMultiplierFeeCharge) + feeRate = int64(float64(feeRate) * common.BTCOutboundGasPriceMultiplier) return DepositorFee(feeRate), nil } diff --git a/zetaclient/chains/bitcoin/observer/inbound_test.go b/zetaclient/chains/bitcoin/observer/inbound_test.go index 444888d6ab..a33ad03946 100644 --- a/zetaclient/chains/bitcoin/observer/inbound_test.go +++ b/zetaclient/chains/bitcoin/observer/inbound_test.go @@ -3,16 +3,17 @@ package observer_test import ( "bytes" "context" - cosmosmath "cosmossdk.io/math" "encoding/hex" - "github.com/zeta-chain/node/pkg/coin" - "github.com/zeta-chain/node/pkg/memo" "math" "math/big" "path" "strings" "testing" + cosmosmath "cosmossdk.io/math" + "github.com/zeta-chain/node/pkg/coin" + "github.com/zeta-chain/node/pkg/memo" + "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" @@ -299,7 +300,7 @@ func TestGetBtcEvent(t *testing.T) { net := &chaincfg.MainNetParams // fee rate of above tx is 28 sat/vB - depositorFee := common.DepositorFee(28 * clientcommon.BTCGasPriceMultiplierFeeCharge) + depositorFee := common.DepositorFee(28 * clientcommon.BTCOutboundGasPriceMultiplier) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) // expected result @@ -657,7 +658,7 @@ func TestGetBtcEventErrors(t *testing.T) { blockNumber := uint64(835640) // fee rate of above tx is 28 sat/vB - depositorFee := common.DepositorFee(28 * clientcommon.BTCGasPriceMultiplierFeeCharge) + depositorFee := common.DepositorFee(28 * clientcommon.BTCOutboundGasPriceMultiplier) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) t.Run("should return error if RPC client fails to get raw tx", func(t *testing.T) { diff --git a/zetaclient/chains/bitcoin/observer/witness_test.go b/zetaclient/chains/bitcoin/observer/witness_test.go index 4fbd463a3f..469c7aee22 100644 --- a/zetaclient/chains/bitcoin/observer/witness_test.go +++ b/zetaclient/chains/bitcoin/observer/witness_test.go @@ -64,7 +64,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { net := &chaincfg.MainNetParams // fee rate of above tx is 28 sat/vB - depositorFee := common.DepositorFee(28 * clientcommon.BTCGasPriceMultiplierFeeCharge) + depositorFee := common.DepositorFee(28 * clientcommon.BTCOutboundGasPriceMultiplier) feeCalculator := mockDepositFeeCalculator(depositorFee, nil) t.Run("decode OP_RETURN ok", func(t *testing.T) { diff --git a/zetaclient/common/constant.go b/zetaclient/common/constant.go index b3020d7b0e..69eb7b5b01 100644 --- a/zetaclient/common/constant.go +++ b/zetaclient/common/constant.go @@ -3,14 +3,14 @@ package common import "time" const ( - // DefaultGasPriceMultiplierFeeCharge is the default gas price multiplier to charge fees from users - DefaultGasPriceMultiplierFeeCharge = 1.0 + // DefaultGasPriceMultiplier is the default gas price multiplier for all chains + DefaultGasPriceMultiplier = 1.0 - // EVMGasPriceMultiplierFeeCharge is the default gas price multiplier to charge fees from users - EVMGasPriceMultiplierFeeCharge = 1.2 + // EVMOutboundGasPriceMultiplier is the default gas price multiplier for EVM-chain outbond txs + EVMOutboundGasPriceMultiplier = 1.2 - // BTCGasPriceMultiplierFeeCharge is the default gas price multiplier to charge fees from users - BTCGasPriceMultiplierFeeCharge = 2.0 + // BTCOutboundGasPriceMultiplier is the default gas price multiplier for BTC outbond txs + BTCOutboundGasPriceMultiplier = 2.0 // BTCGasPriceMultiplierSendTx is the default gas price multiplier to send out BTC TSS txs BTCGasPriceMultiplierSendTx = 1.5 diff --git a/zetaclient/zetacore/client_vote.go b/zetaclient/zetacore/client_vote.go index 704ffc941a..934fa69591 100644 --- a/zetaclient/zetacore/client_vote.go +++ b/zetaclient/zetacore/client_vote.go @@ -21,7 +21,7 @@ func (c *Client) PostVoteGasPrice( gasPrice uint64, priorityFee, blockNum uint64, ) (string, error) { // get gas price multiplier for the chain - multiplier := GasPriceMultiplierFeeCharge(chain) + multiplier := GasPriceMultiplier(chain) // #nosec G115 always in range gasPrice = uint64(float64(gasPrice) * multiplier) diff --git a/zetaclient/zetacore/tx.go b/zetaclient/zetacore/tx.go index 43298c2cbd..2923173bdf 100644 --- a/zetaclient/zetacore/tx.go +++ b/zetaclient/zetacore/tx.go @@ -58,15 +58,15 @@ func GetInboundVoteMessage( return msg } -// GasPriceMultiplierFeeCharge returns the fee-charging gas price multiplier for the given chain -func GasPriceMultiplierFeeCharge(chain chains.Chain) float64 { +// GasPriceMultiplier returns the gas price multiplier for the given chain +func GasPriceMultiplier(chain chains.Chain) float64 { switch chain.Consensus { case chains.Consensus_ethereum: - return clientcommon.EVMGasPriceMultiplierFeeCharge + return clientcommon.EVMOutboundGasPriceMultiplier case chains.Consensus_bitcoin: - return clientcommon.BTCGasPriceMultiplierFeeCharge + return clientcommon.BTCOutboundGasPriceMultiplier default: - return clientcommon.DefaultGasPriceMultiplierFeeCharge + return clientcommon.DefaultGasPriceMultiplier } } diff --git a/zetaclient/zetacore/tx_test.go b/zetaclient/zetacore/tx_test.go index 9bbb854795..f28bc0a7b8 100644 --- a/zetaclient/zetacore/tx_test.go +++ b/zetaclient/zetacore/tx_test.go @@ -25,7 +25,7 @@ const ( ethBlockHash = "1a17bcc359e84ba8ae03b17ec425f97022cd11c3e279f6bdf7a96fcffa12b366" ) -func Test_GasPriceMultiplierFeeCharge(t *testing.T) { +func Test_GasPriceMultiplier(t *testing.T) { tt := []struct { name string chain chains.Chain @@ -84,7 +84,7 @@ func Test_GasPriceMultiplierFeeCharge(t *testing.T) { } for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { - multiplier := GasPriceMultiplierFeeCharge(tc.chain) + multiplier := GasPriceMultiplier(tc.chain) require.Equal(t, tc.multiplier, multiplier) }) } From a50132e9d836bda88c04eedee0afbca7d7e5ac02 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 18 Mar 2025 16:57:49 -0500 Subject: [PATCH 50/74] remove the method OutboundFeeRateFromCCTXRate; add additional field in OutboundData to handle fee bumping --- zetaclient/chains/bitcoin/client/helpers.go | 5 ++ zetaclient/chains/bitcoin/client/mockgen.go | 1 + zetaclient/chains/bitcoin/common/fee.go | 10 --- zetaclient/chains/bitcoin/common/fee_test.go | 21 ----- .../chains/bitcoin/signer/outbound_data.go | 57 ++++++------- .../bitcoin/signer/outbound_data_test.go | 78 +++++++----------- zetaclient/chains/bitcoin/signer/sign_rbf.go | 41 ++++------ .../chains/bitcoin/signer/sign_rbf_test.go | 82 +++++++++++-------- zetaclient/chains/bitcoin/signer/signer.go | 36 ++++---- zetaclient/testutils/mocks/bitcoin_client.go | 66 +++++++++------ 10 files changed, 184 insertions(+), 213 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index ed5dcf5853..f27a35d216 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -26,6 +26,11 @@ const ( maxBTCSupply = 21000000.0 ) +// IsRegnet returns true if the chain is regnet +func (c *Client) IsRegnet() bool { + return c.isRegnet +} + // GetBlockVerboseByStr alias for GetBlockVerbose func (c *Client) GetBlockVerboseByStr(ctx context.Context, blockHash string) (*types.GetBlockVerboseTxResult, error) { h, err := strToHash(blockHash) diff --git a/zetaclient/chains/bitcoin/client/mockgen.go b/zetaclient/chains/bitcoin/client/mockgen.go index 3a9621dde3..06cd21b147 100644 --- a/zetaclient/chains/bitcoin/client/mockgen.go +++ b/zetaclient/chains/bitcoin/client/mockgen.go @@ -18,6 +18,7 @@ import ( //go:generate mockery --name client --structname BitcoinClient --filename bitcoin_client.go --output ../../../testutils/mocks type client interface { Ping(ctx context.Context) error + IsRegnet() bool Healthcheck(ctx context.Context, tssAddress btcutil.Address) (time.Time, error) GetNetworkInfo(ctx context.Context) (*types.GetNetworkInfoResult, error) diff --git a/zetaclient/chains/bitcoin/common/fee.go b/zetaclient/chains/bitcoin/common/fee.go index 6e87b09fec..d4f9687609 100644 --- a/zetaclient/chains/bitcoin/common/fee.go +++ b/zetaclient/chains/bitcoin/common/fee.go @@ -167,16 +167,6 @@ func DepositorFee(satPerByte int64) float64 { return float64(satPerByte) * float64(BtcOutboundBytesDepositor) / btcutil.SatoshiPerBitcoin } -// OutboundFeeRateFromCCTXRate calculates the outbound fee rate from the median rate -// Example: OutboundFeeRateFromCCTXRate(10) => 10 / 2.0 * 1.5 = 7.5 ≈ 8 -// Example: OutboundFeeRateFromCCTXRate(20) => 20 / 2.0 * 1.5 = 15 -// -// TSS will make profit from the difference between fee charged from users and fee paid to Bitcoin network -func OutboundFeeRateFromCCTXRate(cctxRate int64) int64 { - marketRate := float64(cctxRate) / common.BTCOutboundGasPriceMultiplier - return int64(math.Round(marketRate * common.BTCGasPriceMultiplierSendTx)) -} - // CalcBlockAvgFeeRate calculates the average gas rate (in sat/vByte) for a given block func CalcBlockAvgFeeRate(blockVb *btcjson.GetBlockVerboseTxResult, netParams *chaincfg.Params) (int64, error) { // sanity check diff --git a/zetaclient/chains/bitcoin/common/fee_test.go b/zetaclient/chains/bitcoin/common/fee_test.go index 0ab0d9f9ca..4465ed6645 100644 --- a/zetaclient/chains/bitcoin/common/fee_test.go +++ b/zetaclient/chains/bitcoin/common/fee_test.go @@ -1,7 +1,6 @@ package common import ( - "fmt" "math/rand" "testing" @@ -476,26 +475,6 @@ func TestOutboundSizeBreakdown(t *testing.T) { require.Equal(t, depositFee, 0.00001360) } -func TestOutboundFeeRateFromCCTXRate(t *testing.T) { - tests := []struct { - inputRate int64 - outputRate int64 - }{ - {inputRate: 0, outputRate: 0}, - {inputRate: 1, outputRate: 1}, - {inputRate: 2, outputRate: 2}, - {inputRate: 10, outputRate: 8}, - {inputRate: 20, outputRate: 15}, - } - - for i, test := range tests { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - output := OutboundFeeRateFromCCTXRate(test.inputRate) - require.Equal(t, test.outputRate, output) - }) - } -} - func TestOutboundSizeMinMaxError(t *testing.T) { // P2TR output is the largest in size; P2WPKH is the smallest toP2TR := getTestAddrScript(t, ScriptTypeP2TR) diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index f954c36136..8688fc416c 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -2,7 +2,6 @@ package signer import ( "fmt" - "math" "strconv" "github.com/btcsuite/btcd/btcutil" @@ -31,19 +30,22 @@ type OutboundData struct { // feeRate is the fee rate in satoshis/vByte feeRate int64 + // feeRateLatest is the latest median fee rate in satoshis/vByte + // this value is fed by the zetacore when it bumps the gas price with gas stability pool + feeRateLatest int64 + // feeRateBumpped is a flag to indicate if the fee rate in CCTX is bumped by zetacore feeRateBumped bool - // txSize is the average size of a BTC outbound transaction - // user is charged (in ZRC20 contract) at a static txSize on each withdrawal - txSize int64 - - // nonce is the nonce of the outbound - nonce uint64 + // minRelayFee is the minimum relay fee in unit of BTC + minRelayFee float64 // height is the ZetaChain block height height uint64 + // nonce is the nonce of the outbound + nonce uint64 + // cancelTx is a flag to indicate if this outbound should be cancelled cancelTx bool } @@ -71,16 +73,19 @@ func NewOutboundData( return nil, fmt.Errorf("invalid fee rate %s", params.GasPrice) } - // check if the fee rate is bumped by zetacore + // check if zetacore has bumped the fee rate // 'GasPriorityFee' is always empty for Bitcoin unless zetacore bumps the fee rate - feeRateBumped := params.GasPrice == params.GasPriorityFee - if feeRateBumped { - logger.Info().Msgf("fee rate is bumped by zetacore: %s", params.GasPriorityFee) + var ( + feeRateBumped bool + feeRateLatest int64 + ) + gasPriorityFee, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) + if err == nil && gasPriorityFee > 0 { + feeRateBumped = true + feeRateLatest = gasPriorityFee + logger.Info().Str("latest_fee_rate", params.GasPriorityFee).Msg("fee rate is bumped by zetacore") } - // apply outbound fee rate multiplier - feeRate = common.OutboundFeeRateFromCCTXRate(feeRate) - // to avoid minRelayTxFee error, please do not use the minimum rate (1 sat/vB by default). // we simply add additional 1 sat/vB to 'minRate' to avoid tx rejection by Bitcoin core. // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/policy.h#L35 @@ -99,18 +104,10 @@ func NewOutboundData( } // amount in BTC and satoshis + // the float64 'amount' is used later to select UTXOs, precision does not matter amount := float64(params.Amount.Uint64()) / 1e8 amountSats := params.Amount.BigInt().Int64() - // check gas limit - if params.CallOptions == nil { - // never happens, 'GetCurrentOutboundParam' will create it - return nil, errors.New("call options is nil") - } - if params.CallOptions.GasLimit > math.MaxInt64 { - return nil, fmt.Errorf("invalid gas limit %d", params.CallOptions.GasLimit) - } - // compliance check restrictedCCTX := compliance.IsCctxRestricted(cctx) if restrictedCCTX { @@ -119,9 +116,9 @@ func NewOutboundData( } // check dust amount - dustAmount := params.Amount.Uint64() < constant.BTCWithdrawalDustAmount + dustAmount := amountSats < constant.BTCWithdrawalDustAmount if dustAmount { - logger.Warn().Msgf("dust amount %d sats, canceling tx", params.Amount.Uint64()) + logger.Warn().Int64("amount", amountSats).Msg("outbound will be cancelled due to dust amount") } // set the amount to 0 when the tx should be cancelled @@ -136,11 +133,11 @@ func NewOutboundData( amount: amount, amountSats: amountSats, feeRate: feeRate, + feeRateLatest: feeRateLatest, feeRateBumped: feeRateBumped, - // #nosec G115 checked in range - txSize: int64(params.CallOptions.GasLimit), - nonce: params.TssNonce, - height: height, - cancelTx: cancelTx, + minRelayFee: minRelayFee, + height: height, + nonce: params.TssNonce, + cancelTx: cancelTx, }, nil } diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index d4c43b3733..3e635daaf1 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -1,7 +1,6 @@ package signer import ( - "math" "testing" sdkmath "cosmossdk.io/math" @@ -45,21 +44,20 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().Receiver = receiver.String() cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC - cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes - cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().GasPrice = "8" // 8 sats/vByte cctx.GetCurrentOutboundParam().TssNonce = 1 }, height: 101, minRelayFee: 0.00001, // 1000 sat/KB expected: &OutboundData{ - to: receiver, - amount: 0.1, - amountSats: 10000000, - feeRate: 8, // Round(7.5) - txSize: 254, - nonce: 1, - height: 101, - cancelTx: false, + to: receiver, + amount: 0.1, + amountSats: 10000000, + feeRate: 8, + nonce: 1, + minRelayFee: 0.00001, + height: 101, + cancelTx: false, }, errMsg: "", }, @@ -71,8 +69,7 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().Receiver = receiver.String() cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC - cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes - cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().GasPrice = "8" // 8 sats/vByte cctx.GetCurrentOutboundParam().GasPriorityFee = "10" // 10 sats/vByte, bumped by zetacore cctx.GetCurrentOutboundParam().TssNonce = 1 }, @@ -82,10 +79,11 @@ func Test_NewOutboundData(t *testing.T) { to: receiver, amount: 0.1, amountSats: 10000000, - feeRate: 8, // Round(7.5) + feeRate: 8, + feeRateLatest: 10, feeRateBumped: true, - txSize: 254, nonce: 1, + minRelayFee: 0.00001, height: 101, cancelTx: false, }, @@ -148,18 +146,6 @@ func Test_NewOutboundData(t *testing.T) { expected: nil, errMsg: "unsupported receiver address", }, - { - name: "invalid gas limit", - cctx: sample.CrossChainTx(t, "0x123"), - cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { - cctx.InboundParams.CoinType = coin.CoinType_Gas - cctx.GetCurrentOutboundParam().Receiver = receiver.String() - cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId - cctx.GetCurrentOutboundParam().CallOptions.GasLimit = math.MaxInt64 + 1 - }, - expected: nil, - errMsg: "invalid gas limit", - }, { name: "should cancel restricted CCTX", cctx: sample.CrossChainTx(t, "0x123"), @@ -169,21 +155,20 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().Receiver = receiver.String() cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC - cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes - cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().GasPrice = "8" // 8 sats/vByte cctx.GetCurrentOutboundParam().TssNonce = 1 }, height: 101, minRelayFee: 0.00001, // 1000 sat/KB expected: &OutboundData{ - to: receiver, - amount: 0, // should cancel the tx - amountSats: 0, - feeRate: 8, // Round(7.5) - txSize: 254, - nonce: 1, - height: 101, - cancelTx: true, + to: receiver, + amount: 0, // should cancel the tx + amountSats: 0, + feeRate: 8, + nonce: 1, + minRelayFee: 0.00001, + height: 101, + cancelTx: true, }, }, { @@ -194,21 +179,20 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().Receiver = receiver.String() cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(constant.BTCWithdrawalDustAmount - 1) - cctx.GetCurrentOutboundParam().CallOptions.GasLimit = 254 // 254 bytes - cctx.GetCurrentOutboundParam().GasPrice = "10" // 10 sats/vByte + cctx.GetCurrentOutboundParam().GasPrice = "8" // 8 sats/vByte cctx.GetCurrentOutboundParam().TssNonce = 1 }, height: 101, minRelayFee: 0.00001, // 1000 sat/KB expected: &OutboundData{ - to: receiver, - amount: 0, // should cancel the tx - amountSats: 0, - feeRate: 8, // Round(7.5) - txSize: 254, - nonce: 1, - height: 101, - cancelTx: true, + to: receiver, + amount: 0, // should cancel the tx + amountSats: 0, + feeRate: 8, + nonce: 1, + minRelayFee: 0.00001, + height: 101, + cancelTx: true, }, }, } diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go index efc022f269..8898dfbf6d 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -2,16 +2,12 @@ package signer import ( "context" - "fmt" - "strconv" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/wire" "github.com/pkg/errors" - "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/zetaclient/logs" ) @@ -20,35 +16,23 @@ import ( // The key points: // - It reuses the stuck tx's inputs and outputs but gives a higher fee to miners. // - Funding the last stuck outbound will be considered as CPFP (child-pays-for-parent) by miners. -func (signer *Signer) SignRBFTx( - ctx context.Context, - height uint64, - nonce uint64, - lastTx *btcutil.Tx, - latestRateStr string, - minRelayFee float64, -) (*wire.MsgTx, error) { +func (signer *Signer) SignRBFTx(ctx context.Context, txData *OutboundData, lastTx *btcutil.Tx) (*wire.MsgTx, error) { var ( lf = map[string]any{ logs.FieldMethod: "SignRBFTx", - logs.FieldNonce: nonce, + logs.FieldNonce: txData.nonce, logs.FieldTx: lastTx.MsgTx().TxID(), } logger = signer.Logger().Std.With().Fields(lf).Logger() + + // use latest fee rate specified by zetacore + cctxRate = txData.feeRateLatest ) - var cctxRate int64 - switch signer.Chain().ChainId { - case chains.BitcoinRegtest.ChainId: - // hardcode for regnet E2E test, zetacore won't feed it to CCTX + // use hardcoded value for regnet E2E test because it takes 40 + // minutes for zetacore to bump gas rate and we should not wait that long + if signer.rpc.IsRegnet() { cctxRate = client.FeeRateRegnetRBF - default: - // parse latest fee rate from CCTX - latestRate, err := strconv.ParseInt(latestRateStr, 10, 64) - if err != nil || latestRate <= 0 { - return nil, fmt.Errorf("invalid fee rate %s", latestRateStr) - } - cctxRate = common.OutboundFeeRateFromCCTXRate(latestRate) } // create fee bumper @@ -58,7 +42,7 @@ func (signer *Signer) SignRBFTx( signer.Chain(), lastTx, cctxRate, - minRelayFee, + txData.minRelayFee, logger, ) if err != nil { @@ -71,7 +55,10 @@ func (signer *Signer) SignRBFTx( return nil, errors.Wrap(err, "BumpTxFee failed") } logger.Info(). - Msgf("BumpTxFee succeed, additional fees: %d sats, rate: %d => %d sat/vB", additionalFees, fb.AvgFeeRate, newRate) + Int64("old_fee_rate", fb.AvgFeeRate). + Int64("new_fee_rate", newRate). + Int64("additional_fees", additionalFees). + Msg("BumpTxFee succeed") // collect input amounts for signing inAmounts := make([]int64, len(newTx.TxIn)) @@ -85,7 +72,7 @@ func (signer *Signer) SignRBFTx( } // sign the RBF tx - err = signer.SignTx(ctx, newTx, inAmounts, height, nonce) + err = signer.SignTx(ctx, newTx, inAmounts, txData.height, txData.nonce) if err != nil { return nil, errors.Wrap(err, "SignTx failed") } diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index 4a4330e6bb..0bfa6625e7 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -4,26 +4,30 @@ import ( "context" "testing" + sdkmath "cosmossdk.io/math" "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" "github.com/pkg/errors" + "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/testutil/sample" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/coin" ) func Test_SignRBFTx(t *testing.T) { // https://mempool.space/tx/030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0 chain := chains.BitcoinMainnet - nonce := uint64(148) txid := "030cd813443f7b70cc6d8a544d320c6d8465e4528fc0f3410b599dc0b26753a0" msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chain.ChainId, txid) - // inputs + // inputs of the transaction type prevTx struct { hash *chainhash.Hash vout uint32 @@ -80,21 +84,19 @@ func Test_SignRBFTx(t *testing.T) { chain chains.Chain lastTx *btcutil.Tx preTxs []prevTx - minRelayFee float64 - cctxRateStr string + txData signer.OutboundData liveRate int64 memplTxsInfo *mempoolTxsInfo errMsg string expectedTx *wire.MsgTx }{ { - name: "should sign RBF tx successfully", - chain: chains.BitcoinMainnet, - lastTx: btcutil.NewTx(msgTx.Copy()), - preTxs: preTxs, - minRelayFee: 0.00001, - cctxRateStr: "76", // 57 sat/vB as tx rate - liveRate: 59, // 59 sat/vB + name: "should sign RBF tx successfully", + chain: chains.BitcoinMainnet, + lastTx: btcutil.NewTx(msgTx.Copy()), + preTxs: preTxs, + txData: mkTxData(t, 0.00001, "57"), // 57 sat/vB as cctx rate + liveRate: 59, // 59 sat/vB memplTxsInfo: newMempoolTxsInfo( 1, // 1 stuck tx 0.00027213, // fees: 0.00027213 BTC @@ -108,30 +110,20 @@ func Test_SignRBFTx(t *testing.T) { return newTx }(), }, - { - name: "should return error if latest fee rate is not available", - chain: chains.BitcoinMainnet, - lastTx: btcutil.NewTx(msgTx.Copy()), - minRelayFee: 0.00001, - cctxRateStr: "", - errMsg: "invalid fee rate", - }, { name: "should return error if unable to create fee bumper", chain: chains.BitcoinMainnet, lastTx: btcutil.NewTx(msgTx.Copy()), - minRelayFee: 0.00001, - cctxRateStr: "76", - memplTxsInfo: nil, + txData: mkTxData(t, 0.00001, "57"), + memplTxsInfo: nil, // no mempool txs info provided errMsg: "NewCPFPFeeBumper failed", }, { - name: "should return error if live rate is too high", - chain: chains.BitcoinMainnet, - lastTx: btcutil.NewTx(msgTx.Copy()), - minRelayFee: 0.00001, - cctxRateStr: "76", // 57 sat/vB as tx rate - liveRate: 99, // 99 sat/vB is much higher than ccxt rate + name: "should return error if live rate is too high", + chain: chains.BitcoinMainnet, + lastTx: btcutil.NewTx(msgTx.Copy()), + txData: mkTxData(t, 0.00001, "57"), // 57 sat/vB as cctx rate + liveRate: 99, // 99 sat/vB is much higher than ccxt rate memplTxsInfo: newMempoolTxsInfo( 1, // 1 stuck tx 0.00027213, // fees: 0.00027213 BTC @@ -141,13 +133,12 @@ func Test_SignRBFTx(t *testing.T) { errMsg: "BumpTxFee failed", }, { - name: "should return error if unable to get previous tx", - chain: chains.BitcoinMainnet, - lastTx: btcutil.NewTx(msgTx.Copy()), - preTxs: nil, - minRelayFee: 0.00001, - cctxRateStr: "76", // 57 sat/vB as tx rate - liveRate: 59, // 59 sat/vB + name: "should return error if unable to get previous tx", + chain: chains.BitcoinMainnet, + lastTx: btcutil.NewTx(msgTx.Copy()), + txData: mkTxData(t, 0.00001, "57"), // 57 sat/vB as cctx rate + preTxs: nil, // no previous info provided + liveRate: 59, // 59 sat/vB memplTxsInfo: newMempoolTxsInfo( 1, // 1 stuck tx 0.00027213, // fees: 0.00027213 BTC @@ -164,6 +155,9 @@ func Test_SignRBFTx(t *testing.T) { // setup signer s := newTestSuite(t, tt.chain) + // mock isRegnet + s.client.On("IsRegnet").Return(tt.chain.ChainId == chains.BitcoinRegtest.ChainId) + // mock RPC live fee rate if tt.liveRate > 0 { s.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(tt.liveRate, nil) @@ -208,7 +202,7 @@ func Test_SignRBFTx(t *testing.T) { // ACT // sign tx ctx := context.Background() - newTx, err := s.SignRBFTx(ctx, 1, nonce, tt.lastTx, tt.cctxRateStr, tt.minRelayFee) + newTx, err := s.SignRBFTx(ctx, &tt.txData, tt.lastTx) if tt.errMsg != "" { require.ErrorContains(t, err, tt.errMsg) return @@ -224,3 +218,19 @@ func Test_SignRBFTx(t *testing.T) { }) } } + +// mkTxData creates a new outbound data for testing +func mkTxData(t *testing.T, minRelayFee float64, latestFeeRate string) signer.OutboundData { + net := &chaincfg.MainNetParams + cctx := sample.CrossChainTx(t, "0x123") + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().GasPrice = "1" + cctx.GetCurrentOutboundParam().GasPriorityFee = latestFeeRate + cctx.GetCurrentOutboundParam().Receiver = sample.BTCAddressP2WPKH(t, sample.Rand(), net).String() + cctx.GetCurrentOutboundParam().ReceiverChainId = chains.BitcoinMainnet.ChainId + cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC + + txData, err := signer.NewOutboundData(cctx, 1, minRelayFee, zerolog.Nop(), zerolog.Nop()) + require.NoError(t, err) + return *txData +} diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 28417acb8b..03e7016b8e 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -31,6 +31,7 @@ const ( ) type RPC interface { + IsRegnet() bool GetNetworkInfo(ctx context.Context) (*btcjson.GetNetworkInfoResult, error) GetRawTransaction(ctx context.Context, hash *chainhash.Hash) (*btcutil.Tx, error) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) @@ -118,30 +119,21 @@ func (signer *Signer) TryProcessOutbound( return } + // setup outbound data + txData, err := NewOutboundData(cctx, height, minRelayFee, logger, signer.Logger().Compliance) + if err != nil { + logger.Error().Err(err).Msg("failed to setup Bitcoin outbound data") + return + } + var ( - rbfTx = false signedTx *wire.MsgTx stuckTx = observer.GetLastStuckOutbound() + rbfTx = stuckTx != nil && stuckTx.Nonce == txData.nonce && txData.feeRateBumped ) // sign outbound - if stuckTx != nil && params.TssNonce == stuckTx.Nonce { - // sign RBF tx - rbfTx = true - signedTx, err = signer.SignRBFTx(ctx, height, params.TssNonce, stuckTx.Tx, params.GasPriorityFee, minRelayFee) - if err != nil { - logger.Error().Err(err).Msg("SignRBFTx failed") - return - } - logger.Info().Str(logs.FieldTx, signedTx.TxID()).Msg("SignRBFTx succeed") - } else { - // setup outbound data - txData, err := NewOutboundData(cctx, height, minRelayFee, logger, signer.Logger().Compliance) - if err != nil { - logger.Error().Err(err).Msg("failed to setup Bitcoin outbound data") - return - } - + if !rbfTx { // sign withdraw tx signedTx, err = signer.SignWithdrawTx(ctx, txData, observer) if err != nil { @@ -149,6 +141,14 @@ func (signer *Signer) TryProcessOutbound( return } logger.Info().Str(logs.FieldTx, signedTx.TxID()).Msg("SignWithdrawTx succeed") + } else { + // sign RBF tx + signedTx, err = signer.SignRBFTx(ctx, txData, stuckTx.Tx) + if err != nil { + logger.Error().Err(err).Msg("SignRBFTx failed") + return + } + logger.Info().Str(logs.FieldTx, signedTx.TxID()).Msg("SignRBFTx succeed") } // broadcast signed outbound diff --git a/zetaclient/testutils/mocks/bitcoin_client.go b/zetaclient/testutils/mocks/bitcoin_client.go index fe81636cd3..458d48b0fc 100644 --- a/zetaclient/testutils/mocks/bitcoin_client.go +++ b/zetaclient/testutils/mocks/bitcoin_client.go @@ -445,29 +445,29 @@ func (_m *BitcoinClient) GetNetworkInfo(ctx context.Context) (*btcjson.GetNetwor return r0, r1 } -// GetRawMempool provides a mock function with given fields: ctx -func (_m *BitcoinClient) GetRawMempool(ctx context.Context) ([]*chainhash.Hash, error) { - ret := _m.Called(ctx) +// GetNewAddress provides a mock function with given fields: ctx, account +func (_m *BitcoinClient) GetNewAddress(ctx context.Context, account string) (btcutil.Address, error) { + ret := _m.Called(ctx, account) if len(ret) == 0 { - panic("no return value specified for GetRawMempool") + panic("no return value specified for GetNewAddress") } - var r0 []*chainhash.Hash + var r0 btcutil.Address var r1 error - if rf, ok := ret.Get(0).(func(context.Context) ([]*chainhash.Hash, error)); ok { - return rf(ctx) + if rf, ok := ret.Get(0).(func(context.Context, string) (btcutil.Address, error)); ok { + return rf(ctx, account) } - if rf, ok := ret.Get(0).(func(context.Context) []*chainhash.Hash); ok { - r0 = rf(ctx) + if rf, ok := ret.Get(0).(func(context.Context, string) btcutil.Address); ok { + r0 = rf(ctx, account) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]*chainhash.Hash) + r0 = ret.Get(0).(btcutil.Address) } } - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, account) } else { r1 = ret.Error(1) } @@ -475,29 +475,29 @@ func (_m *BitcoinClient) GetRawMempool(ctx context.Context) ([]*chainhash.Hash, return r0, r1 } -// GetNewAddress provides a mock function with given fields: ctx, account -func (_m *BitcoinClient) GetNewAddress(ctx context.Context, account string) (btcutil.Address, error) { - ret := _m.Called(ctx, account) +// GetRawMempool provides a mock function with given fields: ctx +func (_m *BitcoinClient) GetRawMempool(ctx context.Context) ([]*chainhash.Hash, error) { + ret := _m.Called(ctx) if len(ret) == 0 { - panic("no return value specified for GetNewAddress") + panic("no return value specified for GetRawMempool") } - var r0 btcutil.Address + var r0 []*chainhash.Hash var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string) (btcutil.Address, error)); ok { - return rf(ctx, account) + if rf, ok := ret.Get(0).(func(context.Context) ([]*chainhash.Hash, error)); ok { + return rf(ctx) } - if rf, ok := ret.Get(0).(func(context.Context, string) btcutil.Address); ok { - r0 = rf(ctx, account) + if rf, ok := ret.Get(0).(func(context.Context) []*chainhash.Hash); ok { + r0 = rf(ctx) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(btcutil.Address) + r0 = ret.Get(0).([]*chainhash.Hash) } } - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, account) + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) } else { r1 = ret.Error(1) } @@ -840,6 +840,24 @@ func (_m *BitcoinClient) ImportPrivKeyRescan(ctx context.Context, privKeyWIF *bt return r0 } +// IsRegnet provides a mock function with given fields: +func (_m *BitcoinClient) IsRegnet() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for IsRegnet") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + // ListUnspent provides a mock function with given fields: ctx func (_m *BitcoinClient) ListUnspent(ctx context.Context) ([]btcjson.ListUnspentResult, error) { ret := _m.Called(ctx) From 83c78d3b7b274e106c38daa4300350db3f1fcb89 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 18 Mar 2025 23:55:35 -0500 Subject: [PATCH 51/74] adjust fee bumper to use new fee rate fed by zetacore --- .../chains/bitcoin/signer/fee_bumper.go | 64 ++++++++++--------- .../chains/bitcoin/signer/fee_bumper_test.go | 52 ++++++--------- .../chains/bitcoin/signer/sign_rbf_test.go | 10 ++- 3 files changed, 60 insertions(+), 66 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go index 20fe753279..8e828b4af6 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -24,10 +24,10 @@ const ( // see: https://mempool.space/graphs/mempool#3y feeRateCap = 100 - // minCPFPFeeBumpPercent is the minimum percentage by which the CPFP average fee rate should be bumped. - // This value 20% is a heuristic, not mandated by the Bitcoin protocol, designed to balance effectiveness - // in replacing stuck transactions while avoiding excessive sensitivity to fee market fluctuations. - minCPFPFeeBumpPercent = 20 + // decentFeeBumpPercent is a decent percentage for a fee rate bump. + // The value20% is a heuristic, not mandated by the Bitcoin protocol. It is used to measure the gap between + // the old fee rate, new fee rate and live fee rate to emit warning messages during RBF fee bumping. + decentFeeBumpPercent = 20 ) // CPFPFeeBumper is a helper struct to contain CPFP (child-pays-for-parent) fee bumping logic @@ -62,6 +62,8 @@ type CPFPFeeBumper struct { // AvgFeeRate is the average fee rate of all stuck TSS txs AvgFeeRate int64 + + Logger zerolog.Logger } // NewCPFPFeeBumper creates a new CPFPFeeBumper @@ -81,9 +83,10 @@ func NewCPFPFeeBumper( Tx: tx, MinRelayFee: minRelayFee, CCTXRate: cctxRate, + Logger: logger, } - err := fb.FetchFeeBumpInfo(logger) + err := fb.FetchFeeBumpInfo() if err != nil { return nil, err } @@ -98,34 +101,28 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, int64, error) { return nil, 0, 0, errors.New("original tx has no reserved bump fees") } - // tx replacement is triggered only when market fee rate goes 20% higher than current paid rate. - // zetacore updates the cctx fee rate evey 10 minutes, we could hold on and retry later. - minBumpRate := mathpkg.IncreaseIntByPercent(b.AvgFeeRate, minCPFPFeeBumpPercent) - if b.CCTXRate < minBumpRate { - return nil, 0, 0, fmt.Errorf( - "hold on RBF: cctx rate %d is lower than the min bumped rate %d", - b.CCTXRate, - minBumpRate, - ) + // the new fee rate is supposed to be much higher than current paid rate (old rate). + // we print a warning message if it's not the case for monitoring purposes. + oldRateBumped := mathpkg.IncreaseIntByPercent(b.AvgFeeRate, decentFeeBumpPercent) + if b.CCTXRate < oldRateBumped { + b.Logger.Warn(). + Int64("old_fee_rate", b.AvgFeeRate). + Int64("new_fee_rate", b.CCTXRate). + Msg("new fee rate is not much higher than the old fee rate") } - // the live rate may continue increasing during network congestion, we should wait until it stabilizes a bit. - // this is to ensure the live rate is not 20%+ higher than the cctx rate, otherwise, the replacement tx may - // also get stuck and need another replacement. - bumpedRate := mathpkg.IncreaseIntByPercent(b.CCTXRate, minCPFPFeeBumpPercent) - if b.LiveRate > bumpedRate { - return nil, 0, 0, fmt.Errorf( - "hold on RBF: live rate %d is much higher than the cctx rate %d", - b.LiveRate, - b.CCTXRate, - ) + // the live rate may continue increasing during network congestion, and the new fee rate is still not high enough. + // but we should still continue with the tx replacement because zetacore had already bumped the fee rate. + newRateBumped := mathpkg.IncreaseIntByPercent(b.CCTXRate, decentFeeBumpPercent) + if b.LiveRate > newRateBumped { + b.Logger.Warn(). + Int64("new_fee_rate", b.CCTXRate). + Int64("live_fee_rate", b.LiveRate). + Msg("live fee rate is still much higher than the new fee rate") } // cap the fee rate to avoid excessive fees - feeRateNew := b.CCTXRate - if b.CCTXRate > feeRateCap { - feeRateNew = feeRateCap - } + feeRateNew := min(b.CCTXRate, feeRateCap) // calculate minmimum relay fees of the new replacement tx // the new tx will have almost same size as the old one because the tx body stays the same @@ -165,7 +162,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, int64, error) { } // fetchFeeBumpInfo fetches all necessary information needed to bump the stuck tx -func (b *CPFPFeeBumper) FetchFeeBumpInfo(logger zerolog.Logger) error { +func (b *CPFPFeeBumper) FetchFeeBumpInfo() error { // query live fee rate liveRate, err := b.RPC.GetEstimatedFeeRate(b.Ctx, 1) if err != nil { @@ -191,8 +188,13 @@ func (b *CPFPFeeBumper) FetchFeeBumpInfo(logger zerolog.Logger) error { b.TotalFees = totalFeesSats b.TotalVSize = totalVSize b.AvgFeeRate = avgFeeRate - logger.Info(). - Msgf("totalTxs %d, totalFees %f, totalVSize %d, avgFeeRate %d", totalTxs, totalFees, totalVSize, avgFeeRate) + + b.Logger.Info(). + Int64("total_txs", totalTxs). + Int64("total_fees", totalFeesSats). + Int64("total_vsize", totalVSize). + Int64("avg_fee_rate", avgFeeRate). + Msg("fetched fee bump information") return nil } diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index c45b60a78b..80d2903a50 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -72,6 +72,7 @@ func Test_NewCPFPFeeBumper(t *testing.T) { TotalFees: 10000, TotalVSize: 1000, AvgFeeRate: 10, + Logger: log.Logger, }, }, { @@ -143,18 +144,19 @@ func Test_BumpTxFee(t *testing.T) { feeBumper: &signer.CPFPFeeBumper{ Tx: btcutil.NewTx(msgTx), MinRelayFee: 0.00001, - CCTXRate: 57, - LiveRate: 60, + CCTXRate: 55, + LiveRate: 67, TotalFees: 27213, TotalVSize: 579, AvgFeeRate: 47, + Logger: log.Logger, }, - additionalFees: 5790, - expectedNewRate: 57, + additionalFees: 4632, // 579*55 - 27213 + expectedNewRate: 55, expectedNewTx: func() *wire.MsgTx { // deduct additional fees newTx := signer.CopyMsgTxNoWitness(msgTx) - newTx.TxOut[2].Value -= 5790 + newTx.TxOut[2].Value -= 4632 return newTx }(), }, @@ -164,18 +166,19 @@ func Test_BumpTxFee(t *testing.T) { Tx: func() *btcutil.Tx { // modify reserved bump fees to barely cover bump fees newTx := msgTx.Copy() - newTx.TxOut[2].Value = 5790 + constant.BTCWithdrawalDustAmount - 1 + newTx.TxOut[2].Value = 57*579 - 27213 + constant.BTCWithdrawalDustAmount - 1 // 6789 return btcutil.NewTx(newTx) }(), MinRelayFee: 0.00001, CCTXRate: 57, - LiveRate: 60, + LiveRate: 67, TotalFees: 27213, TotalVSize: 579, AvgFeeRate: 47, + Logger: log.Logger, }, - additionalFees: 5790 + constant.BTCWithdrawalDustAmount - 1, // 6789 - expectedNewRate: 59, // (27213 + 6789) / 579 ≈ 59 + additionalFees: 6789, // same as the reserved value in 2nd output + expectedNewRate: 59, // (27213 + 6789) / 579 ≈ 59 expectedNewTx: func() *wire.MsgTx { // give up all reserved bump fees newTx := signer.CopyMsgTxNoWitness(msgTx) @@ -184,7 +187,7 @@ func Test_BumpTxFee(t *testing.T) { }(), }, { - name: "should cap new gas rate to 'gasRateCap'", + name: "should set new gas rate to 'gasRateCap'", feeBumper: &signer.CPFPFeeBumper{ Tx: btcutil.NewTx(msgTx), MinRelayFee: 0.00001, @@ -193,6 +196,7 @@ func Test_BumpTxFee(t *testing.T) { TotalFees: 27213, TotalVSize: 579, AvgFeeRate: 47, + Logger: log.Logger, }, additionalFees: 30687, // (100-47)*579 expectedNewRate: 100, @@ -215,25 +219,6 @@ func Test_BumpTxFee(t *testing.T) { }, errMsg: "original tx has no reserved bump fees", }, - { - name: "should hold on RBF if CCTX rate is lower than minimum bumpeed rate", - feeBumper: &signer.CPFPFeeBumper{ - Tx: btcutil.NewTx(msgTx), - CCTXRate: 55, // 56 < 47 * 120% - AvgFeeRate: 47, - }, - errMsg: "lower than the min bumped rate", - }, - { - name: "should hold on RBF if live rate is much higher than CCTX rate", - feeBumper: &signer.CPFPFeeBumper{ - Tx: btcutil.NewTx(msgTx), - CCTXRate: 57, - LiveRate: 70, // 70 > 57 * 120% - AvgFeeRate: 47, - }, - errMsg: "much higher than the cctx rate", - }, { name: "should hold on RBF if additional fees is lower than min relay fees", feeBumper: &signer.CPFPFeeBumper{ @@ -244,6 +229,7 @@ func Test_BumpTxFee(t *testing.T) { TotalFees: 2895, TotalVSize: 579, AvgFeeRate: 5, + Logger: log.Logger, }, errMsg: "lower than min relay fees", }, @@ -294,6 +280,7 @@ func Test_FetchFeeBumpInfo(t *testing.T) { TotalFees: 10000, TotalVSize: 1000, AvgFeeRate: 10, + Logger: log.Logger, }, }, { @@ -339,10 +326,11 @@ func Test_FetchFeeBumpInfo(t *testing.T) { // ACT bumper := &signer.CPFPFeeBumper{ - RPC: client, - Tx: tt.tx, + RPC: client, + Tx: tt.tx, + Logger: log.Logger, } - err := bumper.FetchFeeBumpInfo(log.Logger) + err := bumper.FetchFeeBumpInfo() // ASSERT if tt.errMsg != "" { diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index 0bfa6625e7..3e5ac48f64 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -119,9 +119,13 @@ func Test_SignRBFTx(t *testing.T) { errMsg: "NewCPFPFeeBumper failed", }, { - name: "should return error if live rate is too high", - chain: chains.BitcoinMainnet, - lastTx: btcutil.NewTx(msgTx.Copy()), + name: "should return error if unable to bump tx fee", + chain: chains.BitcoinMainnet, + lastTx: func() *btcutil.Tx { + txCopy := msgTx.Copy() + txCopy.TxOut = txCopy.TxOut[:2] // remove reserved bump fees to cause error + return btcutil.NewTx(txCopy) + }(), txData: mkTxData(t, 0.00001, "57"), // 57 sat/vB as cctx rate liveRate: 99, // 99 sat/vB is much higher than ccxt rate memplTxsInfo: newMempoolTxsInfo( From 1627f03c395df082a2f971f05000e7b7f18dcaa4 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 19 Mar 2025 12:21:15 -0500 Subject: [PATCH 52/74] adjust SignRBFTx to work with local E2E test --- zetaclient/chains/bitcoin/signer/sign_rbf.go | 10 ++++++---- zetaclient/chains/bitcoin/signer/sign_rbf_test.go | 7 +++++++ zetaclient/chains/bitcoin/signer/signer.go | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go index 8898dfbf6d..ab8b4604be 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -25,14 +25,16 @@ func (signer *Signer) SignRBFTx(ctx context.Context, txData *OutboundData, lastT } logger = signer.Logger().Std.With().Fields(lf).Logger() - // use latest fee rate specified by zetacore + isRegnet = signer.rpc.IsRegnet() cctxRate = txData.feeRateLatest ) - // use hardcoded value for regnet E2E test because it takes 40 - // minutes for zetacore to bump gas rate and we should not wait that long - if signer.rpc.IsRegnet() { + // 1. for E2E test in regnet, hardcoded fee rate is used as we can't wait 40 minutes for zetacore to bump the fee rate + // 2. for testnet and mainnet, we must wait for zetacore to bump the fee rate before signing the RBF tx + if isRegnet { cctxRate = client.FeeRateRegnetRBF + } else if !txData.feeRateBumped { + return nil, errors.New("fee rate is not bumped by zetacore yet, please hold on") } // create fee bumper diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index 3e5ac48f64..2f91489f2f 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -110,6 +110,13 @@ func Test_SignRBFTx(t *testing.T) { return newTx }(), }, + { + name: "should return error if fee rate is not bumped by zetacore yet", + chain: chains.BitcoinMainnet, + lastTx: btcutil.NewTx(msgTx.Copy()), + txData: mkTxData(t, 0.00001, ""), // empty gas priority fee, not bumped yet + errMsg: "fee rate is not bumped by zetacore yet", + }, { name: "should return error if unable to create fee bumper", chain: chains.BitcoinMainnet, diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 03e7016b8e..4181ef0833 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -129,7 +129,7 @@ func (signer *Signer) TryProcessOutbound( var ( signedTx *wire.MsgTx stuckTx = observer.GetLastStuckOutbound() - rbfTx = stuckTx != nil && stuckTx.Nonce == txData.nonce && txData.feeRateBumped + rbfTx = stuckTx != nil && stuckTx.Nonce == txData.nonce ) // sign outbound From 3776e30a4c4ad0cc8773135701865fe567d8955a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 19 Mar 2025 14:55:14 -0500 Subject: [PATCH 53/74] code clean up; improve comments --- zetaclient/chains/bitcoin/common/fee.go | 4 ++-- zetaclient/chains/bitcoin/observer/mempool.go | 4 ++-- zetaclient/chains/bitcoin/observer/observer.go | 1 + zetaclient/chains/bitcoin/signer/signer.go | 4 +++- zetaclient/common/constant.go | 3 --- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/zetaclient/chains/bitcoin/common/fee.go b/zetaclient/chains/bitcoin/common/fee.go index d4f9687609..ffa457f784 100644 --- a/zetaclient/chains/bitcoin/common/fee.go +++ b/zetaclient/chains/bitcoin/common/fee.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/pkg/errors" - "github.com/zeta-chain/node/zetaclient/common" + clientcommon "github.com/zeta-chain/node/zetaclient/common" ) const ( @@ -250,7 +250,7 @@ func CalcDepositorFee( // apply gas price multiplier // #nosec G115 always in range - feeRate = int64(float64(feeRate) * common.BTCOutboundGasPriceMultiplier) + feeRate = int64(float64(feeRate) * clientcommon.BTCOutboundGasPriceMultiplier) return DepositorFee(feeRate), nil } diff --git a/zetaclient/chains/bitcoin/observer/mempool.go b/zetaclient/chains/bitcoin/observer/mempool.go index 502b6b7b9e..c97a2f1ad8 100644 --- a/zetaclient/chains/bitcoin/observer/mempool.go +++ b/zetaclient/chains/bitcoin/observer/mempool.go @@ -95,7 +95,7 @@ func (ob *Observer) RefreshLastStuckOutbound( // step 3: update last outbound stuck tx information // // the key ideas to determine if Bitcoin outbound is stuck/unstuck: - // 1. outbound txs are a sequence of txs chained by nonce-mark UTXOs. + // 1. outbound txs are a sequence of txs chained by nonce-mark UTXOs. // 2. outbound tx with nonce N+1 MUST spend the nonce-mark UTXO produced by parent tx with nonce N. // 3. when the last descendant tx is stuck, none of its ancestor txs can go through, so the stuck flag is set. // 4. then RBF kicks in, it bumps the fee of the last descendant tx and aims to increase the average fee @@ -108,7 +108,7 @@ func (ob *Observer) RefreshLastStuckOutbound( // // Note: reserved RBF bumping fee might be not enough to clear the stuck txs during extreme traffic surges, two options: // 1. wait for the gas rate to drop. - // 2. manually clear the stuck txs by using offline accelerator services. + // 2. manually clear the stuck txs by using transaction accelerator services. if stuck { ob.SetLastStuckOutbound(NewLastStuckOutbound(lastNonce, lastTx, stuckFor)) } else { diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index abe25d35c2..cd5e1ef0db 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -214,6 +214,7 @@ func (ob *Observer) GetBlockByNumberCached(ctx context.Context, blockNumber int6 func (ob *Observer) GetLastStuckOutbound() *LastStuckOutbound { ob.Mu().Lock() defer ob.Mu().Unlock() + return ob.lastStuckTx } diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 4181ef0833..3c94b290a7 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -175,7 +175,9 @@ func (signer *Signer) BroadcastOutbound( Str(logs.FieldCctx, cctx.Index). Logger() - // double check to ensure the tx being replaced is still the last outbound + // double check to ensure the tx being replaced is still the last outbound. + // when CCTX gets stuck at nonce 'N', the pending nonce will stop incrementing + // and stay at 'N' or 'N+1' (at most). if rbfTx && ob.GetPendingNonce() > nonce+1 { logger.Warn().Msgf("RBF tx nonce is outdated, skipping broadcasting") return diff --git a/zetaclient/common/constant.go b/zetaclient/common/constant.go index 69eb7b5b01..0c5a4bf1b5 100644 --- a/zetaclient/common/constant.go +++ b/zetaclient/common/constant.go @@ -12,9 +12,6 @@ const ( // BTCOutboundGasPriceMultiplier is the default gas price multiplier for BTC outbond txs BTCOutboundGasPriceMultiplier = 2.0 - // BTCGasPriceMultiplierSendTx is the default gas price multiplier to send out BTC TSS txs - BTCGasPriceMultiplierSendTx = 1.5 - // RPCStatusCheckInterval is the interval to check RPC status, 1 minute RPCStatusCheckInterval = time.Minute From 592d93eed2845b67c8e7636d86b107da50ef0424 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 19 Mar 2025 15:09:08 -0500 Subject: [PATCH 54/74] handle invalid gas priority fee explicitly --- zetaclient/chains/bitcoin/signer/outbound_data.go | 8 ++++++-- .../chains/bitcoin/signer/outbound_data_test.go | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index 8688fc416c..40260b84a7 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -79,8 +79,12 @@ func NewOutboundData( feeRateBumped bool feeRateLatest int64 ) - gasPriorityFee, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) - if err == nil && gasPriorityFee > 0 { + if params.GasPriorityFee != "" { + gasPriorityFee, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) + if err != nil || gasPriorityFee <= 0 { + return nil, fmt.Errorf("invalid gas priority fee %s", params.GasPriorityFee) + } + feeRateBumped = true feeRateLatest = gasPriorityFee logger.Info().Str("latest_fee_rate", params.GasPriorityFee).Msg("fee rate is bumped by zetacore") diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index 3e635daaf1..40a9b9323e 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -115,6 +115,17 @@ func Test_NewOutboundData(t *testing.T) { expected: nil, errMsg: "invalid fee rate", }, + { + name: "invalid gas priority fee", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().GasPrice = "8" + cctx.GetCurrentOutboundParam().GasPriorityFee = "0" // invalid value + }, + expected: nil, + errMsg: "invalid gas priority fee", + }, { name: "zero fee rate", cctx: sample.CrossChainTx(t, "0x123"), From 0b4a632df79c7aa8e8a535e1cf8c7013ae3323fe Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 1 May 2025 11:48:12 -0500 Subject: [PATCH 55/74] fix withdraw failure and unit tests --- .../chains/bitcoin/signer/outbound_data.go | 12 +++++++----- .../chains/bitcoin/signer/outbound_data_test.go | 16 ++++++++-------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index f33b279edd..f1d310889d 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -74,20 +74,22 @@ func NewOutboundData( } // check if zetacore has bumped the fee rate - // 'GasPriorityFee' is always empty for Bitcoin unless zetacore bumps the fee rate + // 'GasPriorityFee' is always "0" for Bitcoin unless zetacore bumps the fee rate var ( feeRateBumped bool feeRateLatest int64 ) if params.GasPriorityFee != "" { gasPriorityFee, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) - if err != nil || gasPriorityFee <= 0 { + if err != nil { return nil, fmt.Errorf("invalid gas priority fee %s", params.GasPriorityFee) } - feeRateBumped = true - feeRateLatest = gasPriorityFee - logger.Info().Str("latest_fee_rate", params.GasPriorityFee).Msg("fee rate is bumped by zetacore") + if gasPriorityFee > 0 { + feeRateBumped = true + feeRateLatest = gasPriorityFee + logger.Info().Str("latest_fee_rate", params.GasPriorityFee).Msg("fee rate is bumped by zetacore") + } } // to avoid minRelayTxFee error, please do not use the minimum rate (1 sat/vB by default). diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index 40a9b9323e..2eae7cdaa9 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -106,7 +106,7 @@ func Test_NewOutboundData(t *testing.T) { errMsg: "invalid coin type", }, { - name: "invalid gas price", + name: "invalid fee rate", cctx: sample.CrossChainTx(t, "0x123"), cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { cctx.InboundParams.CoinType = coin.CoinType_Gas @@ -116,25 +116,25 @@ func Test_NewOutboundData(t *testing.T) { errMsg: "invalid fee rate", }, { - name: "invalid gas priority fee", + name: "zero fee rate", cctx: sample.CrossChainTx(t, "0x123"), cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { cctx.InboundParams.CoinType = coin.CoinType_Gas - cctx.GetCurrentOutboundParam().GasPrice = "8" - cctx.GetCurrentOutboundParam().GasPriorityFee = "0" // invalid value + cctx.GetCurrentOutboundParam().GasPrice = "0" }, expected: nil, - errMsg: "invalid gas priority fee", + errMsg: "invalid fee rate", }, { - name: "zero fee rate", + name: "invalid gas priority fee", cctx: sample.CrossChainTx(t, "0x123"), cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { cctx.InboundParams.CoinType = coin.CoinType_Gas - cctx.GetCurrentOutboundParam().GasPrice = "0" + cctx.GetCurrentOutboundParam().GasPrice = "8" + cctx.GetCurrentOutboundParam().GasPriorityFee = "invalid" }, expected: nil, - errMsg: "invalid fee rate", + errMsg: "invalid gas priority fee", }, { name: "invalid receiver address", From 7e4cd7828ddac1f51949c6845d9c97a1ab2f9593 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 1 May 2025 15:57:25 -0500 Subject: [PATCH 56/74] rename GetLastStuckOutbound -> LastStuckOutbound; unexport SetLastStuckOutbound --- zetaclient/chains/bitcoin/observer/db_test.go | 5 +- .../chains/bitcoin/observer/event_test.go | 39 +++++++------ .../chains/bitcoin/observer/inbound_test.go | 55 +++++++++--------- zetaclient/chains/bitcoin/observer/mempool.go | 4 +- .../chains/bitcoin/observer/mempool_test.go | 57 +++++++++---------- .../chains/bitcoin/observer/observer.go | 10 ++-- .../chains/bitcoin/observer/observer_test.go | 38 +++++++------ .../chains/bitcoin/observer/outbound_test.go | 3 - .../chains/bitcoin/observer/utxos_test.go | 2 +- .../chains/bitcoin/observer/witness_test.go | 39 +++++++------ zetaclient/chains/bitcoin/signer/signer.go | 6 +- 11 files changed, 128 insertions(+), 130 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/db_test.go b/zetaclient/chains/bitcoin/observer/db_test.go index 1a3ad0c206..578980ba80 100644 --- a/zetaclient/chains/bitcoin/observer/db_test.go +++ b/zetaclient/chains/bitcoin/observer/db_test.go @@ -1,4 +1,4 @@ -package observer_test +package observer import ( "context" @@ -11,7 +11,6 @@ import ( "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/testutil/sample" "github.com/zeta-chain/node/zetaclient/chains/base" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" ) func Test_SaveBroadcastedTx(t *testing.T) { @@ -114,7 +113,7 @@ func Test_LoadLastBlockScanned(t *testing.T) { // load last block scanned err := obRegnet.LoadLastBlockScanned(ctx) require.NoError(t, err) - require.EqualValues(t, observer.RegnetStartBlock, obRegnet.LastBlockScanned()) + require.EqualValues(t, RegnetStartBlock, obRegnet.LastBlockScanned()) }) } diff --git a/zetaclient/chains/bitcoin/observer/event_test.go b/zetaclient/chains/bitcoin/observer/event_test.go index 5db29d409f..cd8b80f63e 100644 --- a/zetaclient/chains/bitcoin/observer/event_test.go +++ b/zetaclient/chains/bitcoin/observer/event_test.go @@ -1,4 +1,4 @@ -package observer_test +package observer import ( "testing" @@ -13,7 +13,6 @@ import ( "github.com/zeta-chain/node/pkg/constant" "github.com/zeta-chain/node/pkg/memo" "github.com/zeta-chain/node/testutil/sample" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/node/zetaclient/config" "github.com/zeta-chain/node/zetaclient/testutils" clienttypes "github.com/zeta-chain/node/zetaclient/types" @@ -25,8 +24,8 @@ func createTestBtcEvent( net *chaincfg.Params, memo []byte, memoStd *memo.InboundMemo, -) observer.BTCInboundEvent { - return observer.BTCInboundEvent{ +) BTCInboundEvent { + return BTCInboundEvent{ FromAddress: sample.BTCAddressP2WPKH(t, sample.Rand(), net).String(), ToAddress: sample.EthAddress().Hex(), MemoBytes: memo, @@ -47,12 +46,12 @@ func Test_Category(t *testing.T) { // test cases tests := []struct { name string - event *observer.BTCInboundEvent + event *BTCInboundEvent expected clienttypes.InboundCategory }{ { name: "should return InboundCategoryProcessable for a processable inbound event", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ FromAddress: "tb1quhassyrlj43qar0mn0k5sufyp6mazmh2q85lr6ex8ehqfhxpzsksllwrsu", ToAddress: testutils.TSSAddressBTCAthens3, }, @@ -60,7 +59,7 @@ func Test_Category(t *testing.T) { }, { name: "should return InboundCategoryRestricted for a restricted sender address", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ FromAddress: sample.RestrictedBtcAddressTest, ToAddress: testutils.TSSAddressBTCAthens3, }, @@ -68,7 +67,7 @@ func Test_Category(t *testing.T) { }, { name: "should return InboundCategoryRestricted for a restricted receiver address in standard memo", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ FromAddress: "tb1quhassyrlj43qar0mn0k5sufyp6mazmh2q85lr6ex8ehqfhxpzsksllwrsu", ToAddress: testutils.TSSAddressBTCAthens3, MemoStd: &memo.InboundMemo{ @@ -81,7 +80,7 @@ func Test_Category(t *testing.T) { }, { name: "should return InboundCategoryRestricted for a restricted revert address in standard memo", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ FromAddress: "tb1quhassyrlj43qar0mn0k5sufyp6mazmh2q85lr6ex8ehqfhxpzsksllwrsu", ToAddress: testutils.TSSAddressBTCAthens3, MemoStd: &memo.InboundMemo{ @@ -96,7 +95,7 @@ func Test_Category(t *testing.T) { }, { name: "should return InboundCategoryDonation for a donation inbound event", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ FromAddress: "tb1quhassyrlj43qar0mn0k5sufyp6mazmh2q85lr6ex8ehqfhxpzsksllwrsu", ToAddress: testutils.TSSAddressBTCAthens3, MemoBytes: []byte(constant.DonationMessage), @@ -118,7 +117,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { tests := []struct { name string chainID int64 - event *observer.BTCInboundEvent + event *BTCInboundEvent expectedMemoStd *memo.InboundMemo expectedReceiver common.Address donation bool @@ -127,7 +126,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { { name: "should decode standard memo bytes successfully", chainID: chains.BitcoinTestnet.ChainId, - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ // a deposit and call MemoBytes: testutil.HexToBytes( t, @@ -150,7 +149,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { { name: "should fall back to legacy memo successfully", chainID: chains.BitcoinTestnet.ChainId, - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ // raw address + payload MemoBytes: testutil.HexToBytes(t, "2d07a9cbd57dcca3e2cf966c88bc874445b6e3b668656c6c6f207361746f736869"), }, @@ -159,7 +158,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { { name: "should disable standard memo for Bitcoin mainnet", chainID: chains.BitcoinMainnet.ChainId, - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ // a deposit and call MemoBytes: testutil.HexToBytes( t, @@ -171,7 +170,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { { name: "should return error if no memo is found", chainID: chains.BitcoinTestnet.ChainId, - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ MemoBytes: []byte("no memo found"), }, errMsg: "no memo found in inbound", @@ -179,7 +178,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { { name: "should do nothing for donation message", chainID: chains.BitcoinTestnet.ChainId, - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ MemoBytes: []byte(constant.DonationMessage), }, donation: true, @@ -187,7 +186,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { { name: "should return error if standard memo contains improper data", chainID: chains.BitcoinTestnet.ChainId, - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ // a deposit and call, receiver is empty ZEVM address MemoBytes: testutil.HexToBytes( t, @@ -199,7 +198,7 @@ func Test_DecodeEventMemoBytes(t *testing.T) { { name: "should return error if standard memo validation failed", chainID: chains.BitcoinTestnet.ChainId, - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ // a no asset call opCode passed, not supported at the moment MemoBytes: testutil.HexToBytes( t, @@ -297,7 +296,7 @@ func Test_ValidateStandardMemo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := observer.ValidateStandardMemo(tt.memo, chains.BitcoinTestnet.ChainId) + err := ValidateStandardMemo(tt.memo, chains.BitcoinTestnet.ChainId) if tt.errMsg != "" { require.Contains(t, err.Error(), tt.errMsg) return @@ -323,7 +322,7 @@ func Test_IsEventProcessable(t *testing.T) { // test cases tests := []struct { name string - event observer.BTCInboundEvent + event BTCInboundEvent result bool }{ { diff --git a/zetaclient/chains/bitcoin/observer/inbound_test.go b/zetaclient/chains/bitcoin/observer/inbound_test.go index e8a1b4e24e..e31932c91c 100644 --- a/zetaclient/chains/bitcoin/observer/inbound_test.go +++ b/zetaclient/chains/bitcoin/observer/inbound_test.go @@ -1,4 +1,4 @@ -package observer_test +package observer import ( "bytes" @@ -29,7 +29,6 @@ import ( "github.com/zeta-chain/node/testutil" "github.com/zeta-chain/node/testutil/sample" crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" clientcommon "github.com/zeta-chain/node/zetaclient/common" "github.com/zeta-chain/node/zetaclient/keys" "github.com/zeta-chain/node/zetaclient/testutils" @@ -169,13 +168,13 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { // test cases tests := []struct { name string - event *observer.BTCInboundEvent + event *BTCInboundEvent observationStatus crosschaintypes.InboundStatus nilVote bool }{ { name: "should return vote for standard memo", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ FromAddress: sample.BTCAddressP2WPKH(t, r, &chaincfg.MainNetParams).String(), // a deposit and call MemoBytes: testutil.HexToBytes( @@ -187,7 +186,7 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { }, { name: "should return vote for legacy memo", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ // raw address + payload MemoBytes: testutil.HexToBytes(t, "2d07a9cbd57dcca3e2cf966c88bc874445b6e3b668656c6c6f207361746f736869"), }, @@ -195,7 +194,7 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { }, { name: "should return vote for invalid memo", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ // standard memo that carries payload only, receiver address is empty MemoBytes: testutil.HexToBytes(t, "5a0110020d68656c6c6f207361746f736869"), }, @@ -203,14 +202,14 @@ func Test_GetInboundVoteFromBtcEvent(t *testing.T) { }, { name: "should return nil on donation message", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ MemoBytes: []byte(constant.DonationMessage), }, nilVote: true, }, { name: "should return nil on invalid deposit value", - event: &observer.BTCInboundEvent{ + event: &BTCInboundEvent{ Value: -1, // invalid value MemoBytes: testutil.HexToBytes(t, "2d07a9cbd57dcca3e2cf966c88bc874445b6e3b668656c6c6f207361746f736869"), }, @@ -247,7 +246,7 @@ func TestGetSenderAddressByVin(t *testing.T) { // get sender address txVin := btcjson.Vin{Txid: txHash, Vout: 2} - sender, err := observer.GetSenderAddressByVin(ctx, rpcClient, txVin, net) + sender, err := GetSenderAddressByVin(ctx, rpcClient, txVin, net) require.NoError(t, err) require.Equal(t, "bc1q68kxnq52ahz5vd6c8czevsawu0ux9nfrzzrh6e", sender) }) @@ -256,7 +255,7 @@ func TestGetSenderAddressByVin(t *testing.T) { rpcClient := mocks.NewBitcoinClient(t) // use invalid tx hash txVin := btcjson.Vin{Txid: "invalid tx hash", Vout: 2} - sender, err := observer.GetSenderAddressByVin(ctx, rpcClient, txVin, net) + sender, err := GetSenderAddressByVin(ctx, rpcClient, txVin, net) require.Error(t, err) require.Empty(t, sender) }) @@ -268,7 +267,7 @@ func TestGetSenderAddressByVin(t *testing.T) { // get sender address txVin := btcjson.Vin{Txid: txHash, Vout: 2} - sender, err := observer.GetSenderAddressByVin(ctx, rpcClient, txVin, net) + sender, err := GetSenderAddressByVin(ctx, rpcClient, txVin, net) require.ErrorContains(t, err, "error getting raw transaction") require.Empty(t, sender) }) @@ -279,7 +278,7 @@ func TestGetSenderAddressByVin(t *testing.T) { // invalid output index txVin := btcjson.Vin{Txid: txHash, Vout: 3} - sender, err := observer.GetSenderAddressByVin(ctx, rpcClient, txVin, net) + sender, err := GetSenderAddressByVin(ctx, rpcClient, txVin, net) require.ErrorContains(t, err, "out of range") require.Empty(t, sender) }) @@ -306,7 +305,7 @@ func TestGetBtcEvent(t *testing.T) { // expected result memo, err := hex.DecodeString(tx.Vout[1].ScriptPubKey.Hex[4:]) require.NoError(t, err) - eventExpected := &observer.BTCInboundEvent{ + eventExpected := &BTCInboundEvent{ FromAddress: "bc1q68kxnq52ahz5vd6c8czevsawu0ux9nfrzzrh6e", ToAddress: tssAddress, Value: tx.Vout[0].Value - depositorFee, // 6192 sataoshis @@ -326,7 +325,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -359,7 +358,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -384,7 +383,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -409,7 +408,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -434,7 +433,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -459,7 +458,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -492,7 +491,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -513,7 +512,7 @@ func TestGetBtcEvent(t *testing.T) { // modify the tx to have Vout[0] a P2SH output tx.Vout[0].ScriptPubKey.Hex = strings.Replace(tx.Vout[0].ScriptPubKey.Hex, "0014", "a914", 1) - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -528,7 +527,7 @@ func TestGetBtcEvent(t *testing.T) { // append 1 byte to script to make it longer than 22 bytes tx.Vout[0].ScriptPubKey.Hex = tx.Vout[0].ScriptPubKey.Hex + "00" - event, err = observer.GetBtcEvent( + event, err = GetBtcEvent( ctx, rpcClient, *tx, @@ -549,7 +548,7 @@ func TestGetBtcEvent(t *testing.T) { // get BTC event rpcClient := mocks.NewBitcoinClient(t) - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -569,7 +568,7 @@ func TestGetBtcEvent(t *testing.T) { // get BTC event rpcClient := mocks.NewBitcoinClient(t) - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -597,7 +596,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -625,7 +624,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -657,7 +656,7 @@ func TestGetBtcEvent(t *testing.T) { rpcClient.On("GetRawTransaction", mock.Anything, mock.Anything).Return(btcutil.NewTx(msgTx), nil) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, @@ -696,7 +695,7 @@ func TestGetBtcEventErrors(t *testing.T) { rpcClient.On("GetRawTransaction", mock.Anything, mock.Anything).Return(nil, errors.New("rpc error")) // get BTC event - event, err := observer.GetBtcEvent( + event, err := GetBtcEvent( ctx, rpcClient, *tx, diff --git a/zetaclient/chains/bitcoin/observer/mempool.go b/zetaclient/chains/bitcoin/observer/mempool.go index c97a2f1ad8..89fc133edb 100644 --- a/zetaclient/chains/bitcoin/observer/mempool.go +++ b/zetaclient/chains/bitcoin/observer/mempool.go @@ -110,9 +110,9 @@ func (ob *Observer) RefreshLastStuckOutbound( // 1. wait for the gas rate to drop. // 2. manually clear the stuck txs by using transaction accelerator services. if stuck { - ob.SetLastStuckOutbound(NewLastStuckOutbound(lastNonce, lastTx, stuckFor)) + ob.setLastStuckOutbound(NewLastStuckOutbound(lastNonce, lastTx, stuckFor)) } else { - ob.SetLastStuckOutbound(nil) + ob.setLastStuckOutbound(nil) } return nil diff --git a/zetaclient/chains/bitcoin/observer/mempool_test.go b/zetaclient/chains/bitcoin/observer/mempool_test.go index 66ff60f9ba..60211f44a1 100644 --- a/zetaclient/chains/bitcoin/observer/mempool_test.go +++ b/zetaclient/chains/bitcoin/observer/mempool_test.go @@ -1,4 +1,4 @@ -package observer_test +package observer import ( "context" @@ -15,7 +15,6 @@ import ( "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/testutil/sample" observertypes "github.com/zeta-chain/node/x/observer/types" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/node/zetaclient/testutils" ) @@ -23,7 +22,7 @@ func Test_NewLastStuckOutbound(t *testing.T) { nonce := uint64(1) tx := btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)) stuckFor := 30 * time.Minute - stuckOutbound := observer.NewLastStuckOutbound(nonce, tx, stuckFor) + stuckOutbound := NewLastStuckOutbound(nonce, tx, stuckFor) require.Equal(t, nonce, stuckOutbound.Nonce) require.Equal(t, tx, stuckOutbound.Tx) @@ -36,10 +35,10 @@ func Test_FefreshLastStuckOutbound(t *testing.T) { tests := []struct { name string - txFinder observer.PendingTxFinder - txChecker observer.StuckTxChecker - oldStuckTx *observer.LastStuckOutbound - expectedTx *observer.LastStuckOutbound + txFinder PendingTxFinder + txChecker StuckTxChecker + oldStuckTx *LastStuckOutbound + expectedTx *LastStuckOutbound errMsg string }{ { @@ -47,20 +46,20 @@ func Test_FefreshLastStuckOutbound(t *testing.T) { txFinder: makePendingTxFinder(sampleTx1, 1, ""), txChecker: makeStuckTxChecker(true, 30*time.Minute, ""), oldStuckTx: nil, - expectedTx: observer.NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), + expectedTx: NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), }, { name: "should update last stuck tx successfully", txFinder: makePendingTxFinder(sampleTx2, 2, ""), txChecker: makeStuckTxChecker(true, 40*time.Minute, ""), - oldStuckTx: observer.NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), - expectedTx: observer.NewLastStuckOutbound(2, sampleTx2, 40*time.Minute), + oldStuckTx: NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), + expectedTx: NewLastStuckOutbound(2, sampleTx2, 40*time.Minute), }, { name: "should clear last stuck tx successfully", txFinder: makePendingTxFinder(sampleTx1, 1, ""), txChecker: makeStuckTxChecker(false, 1*time.Minute, ""), - oldStuckTx: observer.NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), + oldStuckTx: NewLastStuckOutbound(1, sampleTx1, 30*time.Minute), expectedTx: nil, }, { @@ -84,7 +83,7 @@ func Test_FefreshLastStuckOutbound(t *testing.T) { // setup old stuck tx if tt.oldStuckTx != nil { - ob.SetLastStuckOutbound(tt.oldStuckTx) + ob.setLastStuckOutbound(tt.oldStuckTx) } // refresh @@ -98,7 +97,7 @@ func Test_FefreshLastStuckOutbound(t *testing.T) { } // check - stuckTx := ob.GetLastStuckOutbound() + stuckTx, _ := ob.LastStuckOutbound() require.Equal(t, tt.expectedTx, stuckTx) }) } @@ -313,7 +312,7 @@ func Test_GetLastPendingOutbound(t *testing.T) { } ctx := context.Background() - lastTx, lastNonce, err := observer.GetLastPendingOutbound(ctx, ob.Observer) + lastTx, lastNonce, err := GetLastPendingOutbound(ctx, ob.Observer) if tt.errMsg != "" { require.ErrorContains(t, err, tt.errMsg) @@ -333,33 +332,33 @@ func Test_GetStuckTxCheck(t *testing.T) { tests := []struct { name string chainID int64 - txChecker observer.StuckTxChecker + txChecker StuckTxChecker }{ { name: "should return 3 blocks for Bitcoin mainnet", chainID: chains.BitcoinMainnet.ChainId, - txChecker: observer.IsTxStuckInMempool, + txChecker: IsTxStuckInMempool, }, { name: "should return 3 blocks for Bitcoin testnet4", chainID: chains.BitcoinTestnet.ChainId, - txChecker: observer.IsTxStuckInMempool, + txChecker: IsTxStuckInMempool, }, { name: "should return 3 blocks for Bitcoin Signet", chainID: chains.BitcoinSignetTestnet.ChainId, - txChecker: observer.IsTxStuckInMempool, + txChecker: IsTxStuckInMempool, }, { name: "should return 10 blocks for Bitcoin regtest", chainID: chains.BitcoinRegtest.ChainId, - txChecker: observer.IsTxStuckInMempoolRegnet, + txChecker: IsTxStuckInMempoolRegnet, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - txChecker := observer.GetStuckTxChecker(tt.chainID) + txChecker := GetStuckTxChecker(tt.chainID) require.Equal(t, reflect.ValueOf(tt.txChecker).Pointer(), reflect.ValueOf(txChecker).Pointer()) }) } @@ -374,51 +373,51 @@ func Test_GetFeeBumpWaitBlocks(t *testing.T) { { name: "should return wait blocks for Bitcoin mainnet", chainID: chains.BitcoinMainnet.ChainId, - expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocks, + expectedWaitBlocks: PendingTxFeeBumpWaitBlocks, }, { name: "should return wait blocks for Bitcoin testnet4", chainID: chains.BitcoinTestnet.ChainId, - expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocks, + expectedWaitBlocks: PendingTxFeeBumpWaitBlocks, }, { name: "should return wait blocks for Bitcoin signet", chainID: chains.BitcoinSignetTestnet.ChainId, - expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocks, + expectedWaitBlocks: PendingTxFeeBumpWaitBlocks, }, { name: "should return wait blocks for Bitcoin regtest", chainID: chains.BitcoinRegtest.ChainId, - expectedWaitBlocks: observer.PendingTxFeeBumpWaitBlocksRegnet, + expectedWaitBlocks: PendingTxFeeBumpWaitBlocksRegnet, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - blocks := observer.GetFeeBumpWaitBlocks(tt.chainID) + blocks := GetFeeBumpWaitBlocks(tt.chainID) require.Equal(t, tt.expectedWaitBlocks, blocks) }) } } // makePendingTxFinder is a helper function to create a mock pending tx finder -func makePendingTxFinder(tx *btcutil.Tx, nonce uint64, errMsg string) observer.PendingTxFinder { +func makePendingTxFinder(tx *btcutil.Tx, nonce uint64, errMsg string) PendingTxFinder { var err error if errMsg != "" { err = errors.New(errMsg) } - return func(_ context.Context, _ *observer.Observer) (*btcutil.Tx, uint64, error) { + return func(_ context.Context, _ *Observer) (*btcutil.Tx, uint64, error) { return tx, nonce, err } } // makeStuckTxChecker is a helper function to create a mock stuck tx checker -func makeStuckTxChecker(stuck bool, stuckFor time.Duration, errMsg string) observer.StuckTxChecker { +func makeStuckTxChecker(stuck bool, stuckFor time.Duration, errMsg string) StuckTxChecker { var err error if errMsg != "" { err = errors.New(errMsg) } - return func(_ context.Context, _ observer.RPC, _ string, _ int64) (bool, time.Duration, error) { + return func(_ context.Context, _ RPC, _ string, _ int64) (bool, time.Duration, error) { return stuck, stuckFor, err } } diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index cd5e1ef0db..175ac174c1 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -210,16 +210,16 @@ func (ob *Observer) GetBlockByNumberCached(ctx context.Context, blockNumber int6 return blockNheader, nil } -// GetLastStuckOutbound returns the last stuck outbound tx information -func (ob *Observer) GetLastStuckOutbound() *LastStuckOutbound { +// LastStuckOutbound returns the last stuck outbound tx information +func (ob *Observer) LastStuckOutbound() (tx *LastStuckOutbound, found bool) { ob.Mu().Lock() defer ob.Mu().Unlock() - return ob.lastStuckTx + return ob.lastStuckTx, ob.lastStuckTx != nil } -// SetLastStuckOutbound sets the information of last stuck outbound -func (ob *Observer) SetLastStuckOutbound(stuckTx *LastStuckOutbound) { +// setLastStuckOutbound sets the information of last stuck outbound +func (ob *Observer) setLastStuckOutbound(stuckTx *LastStuckOutbound) { ob.Mu().Lock() defer ob.Mu().Unlock() diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index cdbb4c52df..dc74c28645 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -1,4 +1,4 @@ -package observer_test +package observer import ( "context" @@ -22,7 +22,6 @@ import ( "github.com/zeta-chain/node/x/crosschain/types" observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/metrics" "github.com/zeta-chain/node/zetaclient/testutils/mocks" @@ -154,7 +153,7 @@ func Test_NewObserver(t *testing.T) { require.NoError(t, err) // create observer - ob, err := observer.New(tt.chain, baseObserver, tt.btcClient) + ob, err := New(tt.chain, baseObserver, tt.btcClient) if tt.errorMessage != "" { require.ErrorContains(t, err, tt.errorMessage) require.Nil(t, ob) @@ -210,34 +209,41 @@ func Test_BlockCache(t *testing.T) { func Test_SetLastStuckOutbound(t *testing.T) { // create observer and example stuck tx ob := newTestSuite(t, chains.BitcoinMainnet) - tx := btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)) + btcTx := btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)) // STEP 1 - // initial stuck outbound is nil - require.Nil(t, ob.GetLastStuckOutbound()) + // initial stuck outbound does not exist + _, found := ob.LastStuckOutbound() + require.False(t, found) // STEP 2 // set stuck outbound - stuckTx := observer.NewLastStuckOutbound(100, tx, 30*time.Minute) - ob.SetLastStuckOutbound(stuckTx) + stuckTx := NewLastStuckOutbound(100, btcTx, 30*time.Minute) + ob.setLastStuckOutbound(stuckTx) // retrieve stuck outbound - require.Equal(t, stuckTx, ob.GetLastStuckOutbound()) + tx, found := ob.LastStuckOutbound() + require.True(t, found) + require.Equal(t, stuckTx, tx) // STEP 3 // update stuck outbound - stuckTxUpdate := observer.NewLastStuckOutbound(101, tx, 40*time.Minute) - ob.SetLastStuckOutbound(stuckTxUpdate) + stuckTxUpdate := NewLastStuckOutbound(101, btcTx, 40*time.Minute) + ob.setLastStuckOutbound(stuckTxUpdate) // retrieve updated stuck outbound - require.Equal(t, stuckTxUpdate, ob.GetLastStuckOutbound()) + tx, found = ob.LastStuckOutbound() + require.True(t, found) + require.Equal(t, stuckTxUpdate, tx) // STEP 4 // clear stuck outbound - ob.SetLastStuckOutbound(nil) + ob.setLastStuckOutbound(nil) // stuck outbound should be nil - require.Nil(t, ob.GetLastStuckOutbound()) + tx, found = ob.LastStuckOutbound() + require.False(t, found) + require.Nil(t, tx) } func TestSubmittedTx(t *testing.T) { @@ -259,7 +265,7 @@ func TestSubmittedTx(t *testing.T) { } type testSuite struct { - *observer.Observer + *Observer ctx context.Context client *mocks.BitcoinClient @@ -327,7 +333,7 @@ func newTestSuite(t *testing.T, chain chains.Chain, opts ...opt) *testSuite { ) require.NoError(t, err) - ob, err := observer.New(chain, baseObserver, client) + ob, err := New(chain, baseObserver, client) require.NoError(t, err) ts := &testSuite{ diff --git a/zetaclient/chains/bitcoin/observer/outbound_test.go b/zetaclient/chains/bitcoin/observer/outbound_test.go index 3ec63b952f..d01fe2a757 100644 --- a/zetaclient/chains/bitcoin/observer/outbound_test.go +++ b/zetaclient/chains/bitcoin/observer/outbound_test.go @@ -17,9 +17,6 @@ import ( "github.com/zeta-chain/node/zetaclient/testutils/mocks" ) -// the relative path to the testdata directory -var TestDataDir = "../../../" - // MockBTCObserverMainnet creates a mock Bitcoin mainnet observer for testing func MockBTCObserverMainnet(t *testing.T, tss interfaces.TSSSigner) *Observer { // setup mock arguments diff --git a/zetaclient/chains/bitcoin/observer/utxos_test.go b/zetaclient/chains/bitcoin/observer/utxos_test.go index 57d32a43e7..0aa26015a9 100644 --- a/zetaclient/chains/bitcoin/observer/utxos_test.go +++ b/zetaclient/chains/bitcoin/observer/utxos_test.go @@ -1,4 +1,4 @@ -package observer_test +package observer import ( "context" diff --git a/zetaclient/chains/bitcoin/observer/witness_test.go b/zetaclient/chains/bitcoin/observer/witness_test.go index e84876dcfb..96328d6c11 100644 --- a/zetaclient/chains/bitcoin/observer/witness_test.go +++ b/zetaclient/chains/bitcoin/observer/witness_test.go @@ -1,4 +1,4 @@ -package observer_test +package observer import ( "context" @@ -15,7 +15,6 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" "github.com/zeta-chain/node/pkg/chains" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" clientcommon "github.com/zeta-chain/node/zetaclient/common" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/zetaclient/testutils/mocks" @@ -31,14 +30,14 @@ func TestParseScriptFromWitness(t *testing.T) { } expected := "20888269c4f0b7f6fe95d0cba364e2b1b879d9b00735d19cfab4b8d87096ce2b3cac00634d0802000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004c50000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000068" - script := observer.ParseScriptFromWitness(witness[:], log.Logger) + script := ParseScriptFromWitness(witness[:], log.Logger) require.NotNil(t, script) require.Equal(t, hex.EncodeToString(script), expected) }) t.Run("no witness", func(t *testing.T) { witness := [0]string{} - script := observer.ParseScriptFromWitness(witness[:], log.Logger) + script := ParseScriptFromWitness(witness[:], log.Logger) require.Nil(t, script) }) @@ -46,7 +45,7 @@ func TestParseScriptFromWitness(t *testing.T) { witness := [1]string{ "134896c42cd95680b048845847c8054756861ffab7d4abab72f6508d67d1ec0c590287ec2161dd7884983286e1cd56ce65c08a24ee0476ede92678a93b1b180c", } - script := observer.ParseScriptFromWitness(witness[:], log.Logger) + script := ParseScriptFromWitness(witness[:], log.Logger) require.Nil(t, script) }) } @@ -77,7 +76,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { tx.Vin[0].Vout = 2 memo, _ := hex.DecodeString(tx.Vout[1].ScriptPubKey.Hex[4:]) - eventExpected := &observer.BTCInboundEvent{ + eventExpected := &BTCInboundEvent{ FromAddress: "bc1q68kxnq52ahz5vd6c8czevsawu0ux9nfrzzrh6e", ToAddress: tssAddress, Value: tx.Vout[0].Value - depositorFee, @@ -91,7 +90,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -119,7 +118,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { tx.Vout[1] = tx.Vout[2] tx.Vout = tx.Vout[:2] - eventExpected := &observer.BTCInboundEvent{ + eventExpected := &BTCInboundEvent{ FromAddress: "bc1q68kxnq52ahz5vd6c8czevsawu0ux9nfrzzrh6e", ToAddress: tssAddress, Value: tx.Vout[0].Value - depositorFee, @@ -133,7 +132,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -159,7 +158,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { tx.Vin[0].Vout = 2 memo, _ := hex.DecodeString(tx.Vout[1].ScriptPubKey.Hex[4:]) - eventExpected := &observer.BTCInboundEvent{ + eventExpected := &BTCInboundEvent{ FromAddress: "bc1q68kxnq52ahz5vd6c8czevsawu0ux9nfrzzrh6e", ToAddress: tssAddress, Value: 0.0, @@ -174,7 +173,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -202,7 +201,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { rpcClient := testrpc.CreateBTCRPCAndLoadTx(t, TestDataDir, chain.ChainId, preHash) // get BTC event - eventExpected := &observer.BTCInboundEvent{ + eventExpected := &BTCInboundEvent{ FromAddress: "bc1q68kxnq52ahz5vd6c8czevsawu0ux9nfrzzrh6e", ToAddress: tssAddress, Value: tx.Vout[0].Value - depositorFee, @@ -213,7 +212,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { } // get BTC event - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -244,7 +243,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { memo, _ := hex.DecodeString( "72f080c854647755d0d9e6f6821f6931f855b9acffd53d87433395672756d58822fd143360762109ab898626556b1c3b8d3096d2361f1297df4a41c1b429471a9aa2fc9be5f27c13b3863d6ac269e4b587d8389f8fd9649859935b0d48dea88cdb40f20c", ) - eventExpected := &observer.BTCInboundEvent{ + eventExpected := &BTCInboundEvent{ FromAddress: "bc1q68kxnq52ahz5vd6c8czevsawu0ux9nfrzzrh6e", ToAddress: tssAddress, Value: tx.Vout[0].Value - depositorFee, @@ -255,7 +254,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { } // get BTC event - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -276,7 +275,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { // get BTC event rpcClient := mocks.NewBitcoinClient(t) - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -296,7 +295,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { // get BTC event rpcClient := mocks.NewBitcoinClient(t) - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -319,7 +318,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { rpcClient.On("GetRawTransaction", mock.Anything, mock.Anything).Return(nil, errors.New("rpc error")) // get BTC event - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -354,7 +353,7 @@ func TestGetBtcEventWithWitness(t *testing.T) { rpcClient.On("GetRawTransaction", mock.Anything, mock.Anything).Return(btcutil.NewTx(msgTx), nil) // get BTC event - event, err := observer.GetBtcEventWithWitness( + event, err := GetBtcEventWithWitness( ctx, rpcClient, *tx, @@ -400,7 +399,7 @@ func Test_DeductDepositorFee(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result, err := observer.DeductDepositorFee(tt.deposited, tt.depositorFee) + result, err := DeductDepositorFee(tt.deposited, tt.depositorFee) require.Equal(t, tt.expected, result) if tt.errMsg != "" { diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index f37ea0b4c7..980804d879 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -134,9 +134,9 @@ func (signer *Signer) TryProcessOutbound( } var ( - signedTx *wire.MsgTx - stuckTx = observer.GetLastStuckOutbound() - rbfTx = stuckTx != nil && stuckTx.Nonce == txData.nonce + signedTx *wire.MsgTx + stuckTx, found = observer.LastStuckOutbound() + rbfTx = found && stuckTx.Nonce == txData.nonce ) // sign outbound From ccd65338411eebbb3c94f9a2ccba00e28e697a6a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 00:17:39 -0500 Subject: [PATCH 57/74] remove IncreaseIntByPercent and use IncreaseUintByPercent instead --- pkg/math/integer.go | 38 --------------- pkg/math/integer_test.go | 31 ------------- .../chains/bitcoin/client/client_rbf_test.go | 4 +- zetaclient/chains/bitcoin/client/helpers.go | 17 +++++-- zetaclient/chains/bitcoin/client/mockgen.go | 4 +- zetaclient/chains/bitcoin/common/fee.go | 17 ++++--- zetaclient/chains/bitcoin/common/fee_test.go | 18 +++++++- .../chains/bitcoin/observer/gas_price.go | 4 +- .../chains/bitcoin/observer/observer.go | 2 +- .../chains/bitcoin/signer/fee_bumper.go | 46 +++++++++++-------- .../chains/bitcoin/signer/fee_bumper_test.go | 20 ++++---- .../chains/bitcoin/signer/outbound_data.go | 19 ++++---- .../bitcoin/signer/outbound_data_test.go | 21 +++++++-- zetaclient/chains/bitcoin/signer/sign.go | 8 ++-- zetaclient/chains/bitcoin/signer/sign_rbf.go | 4 +- .../chains/bitcoin/signer/sign_rbf_test.go | 4 +- zetaclient/chains/bitcoin/signer/signer.go | 4 +- zetaclient/testutils/mocks/bitcoin_client.go | 20 ++++---- 18 files changed, 132 insertions(+), 149 deletions(-) delete mode 100644 pkg/math/integer.go delete mode 100644 pkg/math/integer_test.go diff --git a/pkg/math/integer.go b/pkg/math/integer.go deleted file mode 100644 index dcd858f55d..0000000000 --- a/pkg/math/integer.go +++ /dev/null @@ -1,38 +0,0 @@ -// Package implements helper functions for integer math operations. -package math - -import ( - "math" - "math/big" -) - -// IncreaseIntByPercent is a function that increases integer by a percentage. -// Example1: IncreaseIntByPercent(10, 15) = 10 * 1.15 = 11 -// Example2: IncreaseIntByPercent(-10, 15) = -10 * 1.15 = -11 -func IncreaseIntByPercent(value int64, percent uint32) int64 { - if percent == 0 { - return value - } - - if value < 0 { - return -IncreaseIntByPercent(-value, percent) - } - - bigValue := big.NewInt(value) - bigPercent := big.NewInt(int64(percent)) - - // product = value * percent - product := new(big.Int).Mul(bigValue, bigPercent) - - // dividing product by 100 - product.Div(product, big.NewInt(100)) - - // result = original value + product - result := new(big.Int).Add(bigValue, product) - - // be mindful if result > MaxInt64 - if result.Cmp(big.NewInt(math.MaxInt64)) > 0 { - return math.MaxInt64 - } - return result.Int64() -} diff --git a/pkg/math/integer_test.go b/pkg/math/integer_test.go deleted file mode 100644 index daf6f966fc..0000000000 --- a/pkg/math/integer_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package math - -import ( - "fmt" - "math" - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_IncreaseIntByPercent(t *testing.T) { - for i, tt := range []struct { - value int64 - percent uint32 - expected int64 - }{ - {value: 10, percent: 0, expected: 10}, - {value: 10, percent: 15, expected: 11}, - {value: 10, percent: 225, expected: 32}, - {value: math.MaxInt64 / 2, percent: 101, expected: math.MaxInt64}, - {value: -10, percent: 0, expected: -10}, - {value: -10, percent: 15, expected: -11}, - {value: -10, percent: 225, expected: -32}, - {value: -math.MaxInt64 / 2, percent: 101, expected: -math.MaxInt64}, - } { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - result := IncreaseIntByPercent(tt.value, tt.percent) - assert.Equal(t, tt.expected, result) - }) - } -} diff --git a/zetaclient/chains/bitcoin/client/client_rbf_test.go b/zetaclient/chains/bitcoin/client/client_rbf_test.go index 91dd2dc49f..30f8bf9e93 100644 --- a/zetaclient/chains/bitcoin/client/client_rbf_test.go +++ b/zetaclient/chains/bitcoin/client/client_rbf_test.go @@ -131,7 +131,7 @@ func LiveTest_RBFTransaction(t *testing.T) { // two rules to satisfy: // - feeTx3 >= feeTx1 + feeTx2 // - additionalFees >= vSizeTx3 * minRelayFeeRate - // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/rbf.cpp#L166-L183 + // see: https://github.com/bitcoin/bitcoin/blob/5b8046a6e893b7fad5a93631e6d1e70db31878af/src/policy/rbf.cpp#L166-L183 minRelayFeeRate := int64(1) feeRateIncrease := minRelayFeeRate sizeTx3 := mempool.GetTxVirtualSize(rawTx1) @@ -241,7 +241,7 @@ func LiveTest_RBFTransaction_Chained_CPFP(t *testing.T) { // two rules to satisfy: // - feeTx4 >= feeTx3 // - additionalFees >= vSizeTx4 * minRelayFeeRate - // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/rbf.cpp#L166-L183 + // see: https://github.com/bitcoin/bitcoin/blob/5b8046a6e893b7fad5a93631e6d1e70db31878af/src/policy/rbf.cpp#L166-L183 minRelayFeeRate := int64(1) feeRateIncrease := minRelayFeeRate additionalFees := (mempool.GetTxVirtualSize(rawTx3) + 1) * feeRateIncrease diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 17dd4d8319..73fbaedb8a 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -125,7 +125,7 @@ func (c *Client) GetRawTransactionResult(ctx context.Context, } // GetEstimatedFeeRate gets estimated smart fee rate (sat/vB) targeting given block confirmation -func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (satsPerByte int64, err error) { +func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (satsPerByte uint64, err error) { // RPC 'EstimateSmartFee' is not available in regnet if c.isRegnet { return FeeRateRegnet, nil @@ -145,7 +145,13 @@ func (c *Client) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (sat if feeRate <= 0 || feeRate >= maxBTCSupply { return 0, fmt.Errorf("invalid fee rate: %f", feeRate) } - return common.FeeRateToSatPerByte(feeRate), nil + + feeRateUint, err := common.FeeRateToSatPerByte(feeRate) + if err != nil { + return 0, errors.Wrapf(err, "invalid fee rate: %f", feeRate) + } + + return feeRateUint, nil } // GetTransactionFeeAndRate gets the transaction fee and rate for a given tx result @@ -230,12 +236,12 @@ func (c *Client) GetTotalMempoolParentsSizeNFees( ctx context.Context, childHash string, timeout time.Duration, -) (int64, float64, int64, int64, error) { +) (int64, float64, int64, uint64, error) { var ( totalTxs int64 totalFees float64 totalVSize int64 - avgFeeRate int64 + avgFeeRate uint64 ) // loop through all parents @@ -280,7 +286,8 @@ func (c *Client) GetTotalMempoolParentsSizeNFees( } // calculate the average fee rate - avgFeeRate = int64(math.Ceil(totalFees / float64(totalVSize))) + // #nosec G115 always positive + avgFeeRate = uint64(math.Ceil(totalFees / float64(totalVSize))) return totalTxs, totalFees, totalVSize, avgFeeRate, nil } diff --git a/zetaclient/chains/bitcoin/client/mockgen.go b/zetaclient/chains/bitcoin/client/mockgen.go index 5579ba84c1..d22f077fe6 100644 --- a/zetaclient/chains/bitcoin/client/mockgen.go +++ b/zetaclient/chains/bitcoin/client/mockgen.go @@ -36,7 +36,7 @@ type client interface { ctx context.Context, childHash string, timeout time.Duration, - ) (int64, float64, int64, int64, error) + ) (int64, float64, int64, uint64, error) GetRawTransactionResult( ctx context.Context, @@ -46,7 +46,7 @@ type client interface { SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*hash.Hash, error) - GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64) (uint64, error) GetTransactionFeeAndRate(ctx context.Context, tx *types.TxRawResult) (int64, int64, error) EstimateSmartFee( ctx context.Context, diff --git a/zetaclient/chains/bitcoin/common/fee.go b/zetaclient/chains/bitcoin/common/fee.go index ffa457f784..80e6c36a89 100644 --- a/zetaclient/chains/bitcoin/common/fee.go +++ b/zetaclient/chains/bitcoin/common/fee.go @@ -69,10 +69,14 @@ type RPC interface { type DepositorFeeCalculator func(context.Context, RPC, *btcjson.TxRawResult, *chaincfg.Params) (float64, error) // FeeRateToSatPerByte converts a fee rate from BTC/KB to sat/vB. -func FeeRateToSatPerByte(rate float64) int64 { +func FeeRateToSatPerByte(rate float64) (uint64, error) { + if rate <= 0 { + return 0, fmt.Errorf("invalid fee rate %f", rate) + } satPerKB := rate * btcutil.SatoshiPerBitcoin - // #nosec G115 always in range - return int64(satPerKB / bytesPerKB) + + // #nosec G115 always positive + return uint64(satPerKB / bytesPerKB), nil } // WiredTxSize calculates the wired tx size in bytes @@ -257,7 +261,7 @@ func CalcDepositorFee( // GetRecentFeeRate gets the highest fee rate from recent blocks // Note: this method should be used for testnet ONLY -func GetRecentFeeRate(ctx context.Context, rpc RPC, netParams *chaincfg.Params) (int64, error) { +func GetRecentFeeRate(ctx context.Context, rpc RPC, netParams *chaincfg.Params) (uint64, error) { // should avoid using this method for mainnet if netParams.Name == chaincfg.MainNetParams.Name { return 0, errors.New("GetRecentFeeRate should not be used for mainnet") @@ -293,9 +297,10 @@ func GetRecentFeeRate(ctx context.Context, rpc RPC, netParams *chaincfg.Params) } // use 10 sat/byte as default estimation if recent fee rate drops to 0 - if highestRate == 0 { + if highestRate <= 0 { highestRate = defaultTestnetFeeRate } - return highestRate, nil + // #nosec G115 checked positive + return uint64(highestRate), nil } diff --git a/zetaclient/chains/bitcoin/common/fee_test.go b/zetaclient/chains/bitcoin/common/fee_test.go index 4465ed6645..56d4711791 100644 --- a/zetaclient/chains/bitcoin/common/fee_test.go +++ b/zetaclient/chains/bitcoin/common/fee_test.go @@ -182,7 +182,8 @@ func Test_FeeRateToSatPerByte(t *testing.T) { tests := []struct { name string rate float64 - expected int64 + expected uint64 + errMsg string }{ { name: "0 sat/vByte", @@ -204,11 +205,24 @@ func Test_FeeRateToSatPerByte(t *testing.T) { rate: 0.0001, expected: 10, }, + { + name: "invalid fee rate", + rate: 0, + expected: 0, + errMsg: "invalid fee rate", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rate := FeeRateToSatPerByte(tt.rate) + rate, err := FeeRateToSatPerByte(tt.rate) + if tt.errMsg != "" { + require.ErrorContains(t, err, tt.errMsg) + require.Zero(t, rate) + return + } + + require.NoError(t, err) require.Equal(t, tt.expected, rate) }) } diff --git a/zetaclient/chains/bitcoin/observer/gas_price.go b/zetaclient/chains/bitcoin/observer/gas_price.go index 8071702a2c..bfdaa136a2 100644 --- a/zetaclient/chains/bitcoin/observer/gas_price.go +++ b/zetaclient/chains/bitcoin/observer/gas_price.go @@ -15,7 +15,7 @@ import ( func (ob *Observer) PostGasPrice(ctx context.Context) error { var ( err error - feeRateEstimated int64 + feeRateEstimated uint64 ) // estimate fee rate according to network type @@ -49,7 +49,7 @@ func (ob *Observer) PostGasPrice(ctx context.Context) error { // #nosec G115 always positive _, err = ob.ZetacoreClient(). - PostVoteGasPrice(ctx, ob.Chain(), uint64(feeRateEstimated), priorityFee, uint64(blockNumber)) + PostVoteGasPrice(ctx, ob.Chain(), feeRateEstimated, priorityFee, uint64(blockNumber)) if err != nil { return errors.Wrap(err, "PostVoteGasPrice error") } diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 175ac174c1..f0987680ad 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -38,7 +38,7 @@ type RPC interface { ) (btcjson.TxRawResult, error) GetMempoolEntry(ctx context.Context, txHash string) (*btcjson.GetMempoolEntryResult, error) - GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64) (uint64, error) GetTransactionFeeAndRate(ctx context.Context, tx *btcjson.TxRawResult) (int64, int64, error) EstimateSmartFee( diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go index 8e828b4af6..1137a42a9b 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -6,6 +6,7 @@ import ( "math" "time" + sdkmath "cosmossdk.io/math" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/mempool" "github.com/btcsuite/btcd/wire" @@ -46,10 +47,10 @@ type CPFPFeeBumper struct { MinRelayFee float64 // CCTXRate is the most recent fee rate of the CCTX - CCTXRate int64 + CCTXRate uint64 // LiveRate is the most recent market fee rate - LiveRate int64 + LiveRate uint64 // TotalTxs is the total number of stuck TSS txs TotalTxs int64 @@ -61,7 +62,7 @@ type CPFPFeeBumper struct { TotalVSize int64 // AvgFeeRate is the average fee rate of all stuck TSS txs - AvgFeeRate int64 + AvgFeeRate uint64 Logger zerolog.Logger } @@ -72,7 +73,7 @@ func NewCPFPFeeBumper( rpc RPC, chain chains.Chain, tx *btcutil.Tx, - cctxRate int64, + cctxRate uint64, minRelayFee float64, logger zerolog.Logger, ) (*CPFPFeeBumper, error) { @@ -94,7 +95,7 @@ func NewCPFPFeeBumper( } // BumpTxFee bumps the fee of the stuck transaction using reserved bump fees -func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, int64, error) { +func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { // reuse old tx body newTx := CopyMsgTxNoWitness(b.Tx.MsgTx()) if len(newTx.TxOut) < 3 { @@ -103,21 +104,22 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, int64, error) { // the new fee rate is supposed to be much higher than current paid rate (old rate). // we print a warning message if it's not the case for monitoring purposes. - oldRateBumped := mathpkg.IncreaseIntByPercent(b.AvgFeeRate, decentFeeBumpPercent) - if b.CCTXRate < oldRateBumped { + // #nosec G115 always positive + oldRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.AvgFeeRate), decentFeeBumpPercent) + if sdkmath.NewUint(b.CCTXRate).LT(oldRateBumped) { b.Logger.Warn(). - Int64("old_fee_rate", b.AvgFeeRate). - Int64("new_fee_rate", b.CCTXRate). + Uint64("old_fee_rate", b.AvgFeeRate). + Uint64("new_fee_rate", b.CCTXRate). Msg("new fee rate is not much higher than the old fee rate") } // the live rate may continue increasing during network congestion, and the new fee rate is still not high enough. // but we should still continue with the tx replacement because zetacore had already bumped the fee rate. - newRateBumped := mathpkg.IncreaseIntByPercent(b.CCTXRate, decentFeeBumpPercent) - if b.LiveRate > newRateBumped { + newRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.CCTXRate), decentFeeBumpPercent) + if sdkmath.NewUint(b.LiveRate).GT(newRateBumped) { b.Logger.Warn(). - Int64("new_fee_rate", b.CCTXRate). - Int64("live_fee_rate", b.LiveRate). + Uint64("new_fee_rate", b.CCTXRate). + Uint64("live_fee_rate", b.LiveRate). Msg("live fee rate is still much higher than the new fee rate") } @@ -127,16 +129,21 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, int64, error) { // calculate minmimum relay fees of the new replacement tx // the new tx will have almost same size as the old one because the tx body stays the same txVSize := mempool.GetTxVirtualSize(b.Tx) - minRelayFeeRate := common.FeeRateToSatPerByte(b.MinRelayFee) - minRelayTxFees := txVSize * minRelayFeeRate + minRelayFeeRate, err := common.FeeRateToSatPerByte(b.MinRelayFee) + if err != nil { + return nil, 0, 0, errors.Wrapf(err, "unable to convert min relay fee rate") + } + // #nosec G115 always in range + minRelayTxFees := txVSize * int64(minRelayFeeRate) // calculate the RBF additional fees required by Bitcoin protocol // two conditions to satisfy: // 1. new txFees >= old txFees (already handled above) // 2. additionalFees >= minRelayTxFees // - // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/rbf.cpp#L166-L183 - additionalFees := b.TotalVSize*feeRateNew - b.TotalFees + // see: https://github.com/bitcoin/bitcoin/blob/5b8046a6e893b7fad5a93631e6d1e70db31878af/src/policy/rbf.cpp#L166-L183 + // #nosec G115 always in range + additionalFees := b.TotalVSize*int64(feeRateNew) - b.TotalFees if additionalFees < minRelayTxFees { return nil, 0, 0, fmt.Errorf( "hold on RBF: additional fees %d is lower than min relay fees %d", @@ -156,7 +163,8 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, int64, error) { } // effective fee rate - feeRateNew = int64(math.Ceil(float64(b.TotalFees+additionalFees) / float64(b.TotalVSize))) + // #nosec G115 always positive + feeRateNew = uint64(math.Ceil(float64(b.TotalFees+additionalFees) / float64(b.TotalVSize))) return newTx, additionalFees, feeRateNew, nil } @@ -193,7 +201,7 @@ func (b *CPFPFeeBumper) FetchFeeBumpInfo() error { Int64("total_txs", totalTxs). Int64("total_fees", totalFeesSats). Int64("total_vsize", totalVSize). - Int64("avg_fee_rate", avgFeeRate). + Uint64("avg_fee_rate", avgFeeRate). Msg("fetched fee bump information") return nil diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index 80d2903a50..6bf4f66fa4 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -22,10 +22,10 @@ type mempoolTxsInfo struct { totalTxs int64 totalFees float64 totalVSize int64 - avgFeeRate int64 + avgFeeRate uint64 } -func newMempoolTxsInfo(totalTxs int64, totalFees float64, totalVSize int64, avgFeeRate int64) *mempoolTxsInfo { +func newMempoolTxsInfo(totalTxs int64, totalFees float64, totalVSize int64, avgFeeRate uint64) *mempoolTxsInfo { return &mempoolTxsInfo{ totalTxs: totalTxs, totalFees: totalFees, @@ -40,8 +40,8 @@ func Test_NewCPFPFeeBumper(t *testing.T) { chain chains.Chain client *mocks.BitcoinClient tx *btcutil.Tx - cctxRate int64 - liveRate int64 + cctxRate uint64 + liveRate uint64 minRelayFee float64 memplTxsInfo *mempoolTxsInfo errMsg string @@ -98,7 +98,7 @@ func Test_NewCPFPFeeBumper(t *testing.T) { Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) } else { v := int64(0) - tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Return(v, 0.0, v, v, errors.New("rpc error")) + tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Return(v, 0.0, v, uint64(0), errors.New("rpc error")) } // ACT @@ -135,7 +135,7 @@ func Test_BumpTxFee(t *testing.T) { name string feeBumper *signer.CPFPFeeBumper additionalFees int64 - expectedNewRate int64 + expectedNewRate uint64 expectedNewTx *wire.MsgTx errMsg string }{ @@ -253,12 +253,12 @@ func Test_BumpTxFee(t *testing.T) { } func Test_FetchFeeBumpInfo(t *testing.T) { - liveRate := int64(12) + liveRate := uint64(12) tests := []struct { name string tx *btcutil.Tx - liveRate int64 + liveRate uint64 memplTxsInfo *mempoolTxsInfo expected *signer.CPFPFeeBumper errMsg string @@ -312,7 +312,7 @@ func Test_FetchFeeBumpInfo(t *testing.T) { if tt.liveRate > 0 { client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(liveRate, nil) } else { - client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), errors.New("rpc error")) + client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(uint64(0), errors.New("rpc error")) } // mock mempool txs information @@ -321,7 +321,7 @@ func Test_FetchFeeBumpInfo(t *testing.T) { Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) } else { v := int64(0) - client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(v, 0.0, v, v, errors.New("rpc error")) + client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(v, 0.0, v, uint64(0), errors.New("rpc error")) } // ACT diff --git a/zetaclient/chains/bitcoin/signer/outbound_data.go b/zetaclient/chains/bitcoin/signer/outbound_data.go index f1d310889d..49e5da3221 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data.go @@ -28,11 +28,11 @@ type OutboundData struct { amountSats int64 // feeRate is the fee rate in satoshis/vByte - feeRate int64 + feeRate uint64 // feeRateLatest is the latest median fee rate in satoshis/vByte // this value is fed by the zetacore when it bumps the gas price with gas stability pool - feeRateLatest int64 + feeRateLatest uint64 // feeRateBumpped is a flag to indicate if the fee rate in CCTX is bumped by zetacore feeRateBumped bool @@ -68,8 +68,8 @@ func NewOutboundData( } // parse fee rate - feeRate, err := strconv.ParseInt(params.GasPrice, 10, 64) - if err != nil || feeRate <= 0 { + feeRate, err := strconv.ParseUint(params.GasPrice, 10, 64) + if err != nil || feeRate == 0 { return nil, fmt.Errorf("invalid fee rate %s", params.GasPrice) } @@ -77,10 +77,10 @@ func NewOutboundData( // 'GasPriorityFee' is always "0" for Bitcoin unless zetacore bumps the fee rate var ( feeRateBumped bool - feeRateLatest int64 + feeRateLatest uint64 ) if params.GasPriorityFee != "" { - gasPriorityFee, err := strconv.ParseInt(params.GasPriorityFee, 10, 64) + gasPriorityFee, err := strconv.ParseUint(params.GasPriorityFee, 10, 64) if err != nil { return nil, fmt.Errorf("invalid gas priority fee %s", params.GasPriorityFee) } @@ -94,8 +94,11 @@ func NewOutboundData( // to avoid minRelayTxFee error, please do not use the minimum rate (1 sat/vB by default). // we simply add additional 1 sat/vB to 'minRate' to avoid tx rejection by Bitcoin core. - // see: https://github.com/bitcoin/bitcoin/blob/master/src/policy/policy.h#L35 - minRate := common.FeeRateToSatPerByte(minRelayFee) + // see: https://github.com/bitcoin/bitcoin/blob/5b8046a6e893b7fad5a93631e6d1e70db31878af/src/policy/policy.h#L42 + minRate, err := common.FeeRateToSatPerByte(minRelayFee) + if err != nil { + return nil, errors.Wrapf(err, "invalid min relay fee") + } if feeRate <= minRate { feeRate = minRate + 1 } diff --git a/zetaclient/chains/bitcoin/signer/outbound_data_test.go b/zetaclient/chains/bitcoin/signer/outbound_data_test.go index 2eae7cdaa9..a42ef70017 100644 --- a/zetaclient/chains/bitcoin/signer/outbound_data_test.go +++ b/zetaclient/chains/bitcoin/signer/outbound_data_test.go @@ -136,6 +136,17 @@ func Test_NewOutboundData(t *testing.T) { expected: nil, errMsg: "invalid gas priority fee", }, + { + name: "invalid min relay fee", + cctx: sample.CrossChainTx(t, "0x123"), + cctxModifier: func(cctx *crosschaintypes.CrossChainTx) { + cctx.InboundParams.CoinType = coin.CoinType_Gas + cctx.GetCurrentOutboundParam().GasPrice = "8" + }, + minRelayFee: 0, // invalid min relay fee + expected: nil, + errMsg: "invalid min relay fee", + }, { name: "invalid receiver address", cctx: sample.CrossChainTx(t, "0x123"), @@ -143,8 +154,9 @@ func Test_NewOutboundData(t *testing.T) { cctx.InboundParams.CoinType = coin.CoinType_Gas cctx.GetCurrentOutboundParam().Receiver = "invalid" }, - expected: nil, - errMsg: "cannot decode receiver address", + minRelayFee: 0.00001, // 1000 sat/KB + expected: nil, + errMsg: "cannot decode receiver address", }, { name: "unsupported receiver address", @@ -154,8 +166,9 @@ func Test_NewOutboundData(t *testing.T) { cctx.GetCurrentOutboundParam().Receiver = "035e4ae279bd416b5da724972c9061ec6298dac020d1e3ca3f06eae715135cdbec" cctx.GetCurrentOutboundParam().ReceiverChainId = chain.ChainId }, - expected: nil, - errMsg: "unsupported receiver address", + minRelayFee: 0.00001, // 1000 sat/KB + expected: nil, + errMsg: "unsupported receiver address", }, { name: "should cancel restricted CCTX", diff --git a/zetaclient/chains/bitcoin/signer/sign.go b/zetaclient/chains/bitcoin/signer/sign.go index f7d390c9c7..f55aa16969 100644 --- a/zetaclient/chains/bitcoin/signer/sign.go +++ b/zetaclient/chains/bitcoin/signer/sign.go @@ -49,7 +49,8 @@ func (signer *Signer) SignWithdrawTx( // we don't know how many UTXOs will be used beforehand, so we do // a conservative estimation using the maximum size of the outbound tx: // estimateFee = feeRate * maxTxSize - estimateFee := float64(txData.feeRate*common.OutboundBytesMax) / 1e8 + // #nosec G115 always in range + estimateFee := float64(int64(txData.feeRate)*common.OutboundBytesMax) / 1e8 totalAmount := txData.amount + estimateFee + reservedRBFFees + float64(nonceMark)*1e-8 // refreshing UTXO list before TSS keysign is important: @@ -97,7 +98,8 @@ func (signer *Signer) SignWithdrawTx( } // fee calculation - fees := txSize * txData.feeRate + // #nosec G115 always in range + fees := txSize * int64(txData.feeRate) // add tx outputs inputValue := selected.Value @@ -106,7 +108,7 @@ func (signer *Signer) SignWithdrawTx( } signer.Logger(). Std.Info(). - Int64("tx.rate", txData.feeRate). + Uint64("tx.rate", txData.feeRate). Int64("tx.fees", fees). Uint16("tx.consolidated_utxos", selected.ConsolidatedUTXOs). Float64("tx.consolidated_value", selected.ConsolidatedValue). diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go index ab8b4604be..b4f128bd79 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -57,8 +57,8 @@ func (signer *Signer) SignRBFTx(ctx context.Context, txData *OutboundData, lastT return nil, errors.Wrap(err, "BumpTxFee failed") } logger.Info(). - Int64("old_fee_rate", fb.AvgFeeRate). - Int64("new_fee_rate", newRate). + Uint64("old_fee_rate", fb.AvgFeeRate). + Uint64("new_fee_rate", newRate). Int64("additional_fees", additionalFees). Msg("BumpTxFee succeed") diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index 2f91489f2f..ebc1c13803 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -85,7 +85,7 @@ func Test_SignRBFTx(t *testing.T) { lastTx *btcutil.Tx preTxs []prevTx txData signer.OutboundData - liveRate int64 + liveRate uint64 memplTxsInfo *mempoolTxsInfo errMsg string expectedTx *wire.MsgTx @@ -173,7 +173,7 @@ func Test_SignRBFTx(t *testing.T) { if tt.liveRate > 0 { s.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(tt.liveRate, nil) } else { - s.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(int64(0), errors.New("rpc error")) + s.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(uint64(0), errors.New("rpc error")) } // mock mempool txs information diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 980804d879..0ce31e2100 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -35,13 +35,13 @@ type RPC interface { IsRegnet() bool GetNetworkInfo(ctx context.Context) (*btcjson.GetNetworkInfoResult, error) GetRawTransaction(ctx context.Context, hash *chainhash.Hash) (*btcutil.Tx, error) - GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) + GetEstimatedFeeRate(ctx context.Context, confTarget int64) (uint64, error) SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*chainhash.Hash, error) GetTotalMempoolParentsSizeNFees( ctx context.Context, childHash string, timeout time.Duration, - ) (int64, float64, int64, int64, error) + ) (int64, float64, int64, uint64, error) } // Signer deals with signing & broadcasting BTC transactions. diff --git a/zetaclient/testutils/mocks/bitcoin_client.go b/zetaclient/testutils/mocks/bitcoin_client.go index 2d8668ae3b..4f2a43414d 100644 --- a/zetaclient/testutils/mocks/bitcoin_client.go +++ b/zetaclient/testutils/mocks/bitcoin_client.go @@ -328,22 +328,22 @@ func (_m *BitcoinClient) GetBlockVerboseByStr(ctx context.Context, blockHash str } // GetEstimatedFeeRate provides a mock function with given fields: ctx, confTarget -func (_m *BitcoinClient) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (int64, error) { +func (_m *BitcoinClient) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (uint64, error) { ret := _m.Called(ctx, confTarget) if len(ret) == 0 { panic("no return value specified for GetEstimatedFeeRate") } - var r0 int64 + var r0 uint64 var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int64) (int64, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, int64) (uint64, error)); ok { return rf(ctx, confTarget) } - if rf, ok := ret.Get(0).(func(context.Context, int64) int64); ok { + if rf, ok := ret.Get(0).(func(context.Context, int64) uint64); ok { r0 = rf(ctx, confTarget) } else { - r0 = ret.Get(0).(int64) + r0 = ret.Get(0).(uint64) } if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { @@ -594,7 +594,7 @@ func (_m *BitcoinClient) GetRawTransactionVerbose(ctx context.Context, hash *cha } // GetTotalMempoolParentsSizeNFees provides a mock function with given fields: ctx, childHash, timeout -func (_m *BitcoinClient) GetTotalMempoolParentsSizeNFees(ctx context.Context, childHash string, timeout time.Duration) (int64, float64, int64, int64, error) { +func (_m *BitcoinClient) GetTotalMempoolParentsSizeNFees(ctx context.Context, childHash string, timeout time.Duration) (int64, float64, int64, uint64, error) { ret := _m.Called(ctx, childHash, timeout) if len(ret) == 0 { @@ -604,9 +604,9 @@ func (_m *BitcoinClient) GetTotalMempoolParentsSizeNFees(ctx context.Context, ch var r0 int64 var r1 float64 var r2 int64 - var r3 int64 + var r3 uint64 var r4 error - if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) (int64, float64, int64, int64, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) (int64, float64, int64, uint64, error)); ok { return rf(ctx, childHash, timeout) } if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) int64); ok { @@ -627,10 +627,10 @@ func (_m *BitcoinClient) GetTotalMempoolParentsSizeNFees(ctx context.Context, ch r2 = ret.Get(2).(int64) } - if rf, ok := ret.Get(3).(func(context.Context, string, time.Duration) int64); ok { + if rf, ok := ret.Get(3).(func(context.Context, string, time.Duration) uint64); ok { r3 = rf(ctx, childHash, timeout) } else { - r3 = ret.Get(3).(int64) + r3 = ret.Get(3).(uint64) } if rf, ok := ret.Get(4).(func(context.Context, string, time.Duration) error); ok { From f94d6aea89cdabaffced56e712d57d559c5b891a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 13:07:10 -0500 Subject: [PATCH 58/74] use same signer package for unit tests --- .../chains/bitcoin/observer/outbound.go | 3 +- .../chains/bitcoin/signer/fee_bumper_test.go | 37 +++++++++---------- .../chains/bitcoin/signer/sign_rbf_test.go | 11 +++--- zetaclient/chains/bitcoin/signer/sign_test.go | 15 ++++---- .../chains/bitcoin/signer/signer_test.go | 7 ++-- 5 files changed, 35 insertions(+), 38 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/outbound.go b/zetaclient/chains/bitcoin/observer/outbound.go index 10783ae528..d4eb41e268 100644 --- a/zetaclient/chains/bitcoin/observer/outbound.go +++ b/zetaclient/chains/bitcoin/observer/outbound.go @@ -348,7 +348,8 @@ func (ob *Observer) SetIncludedTx(nonce uint64, getTxResult *btcjson.GetTransact } else { // for other hash: // got multiple hashes for same nonce. RBF tx replacement happened. - ob.logger.Outbound.Info().Fields(lf).Msgf("replaced bitcoin outbound %s", res.TxID) + lf["prior_tx"] = res.TxID + ob.logger.Outbound.Info().Fields(lf).Msgf("replaced bitcoin outbound") // remove prior txHash and txResult delete(ob.tssOutboundHashes, res.TxID) diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index 6bf4f66fa4..475815fc07 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -1,4 +1,4 @@ -package signer_test +package signer import ( "context" @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/pkg/constant" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/zetaclient/testutils/mocks" ) @@ -45,7 +44,7 @@ func Test_NewCPFPFeeBumper(t *testing.T) { minRelayFee float64 memplTxsInfo *mempoolTxsInfo errMsg string - expected *signer.CPFPFeeBumper + expected *CPFPFeeBumper }{ { chain: chains.BitcoinMainnet, @@ -61,7 +60,7 @@ func Test_NewCPFPFeeBumper(t *testing.T) { 1000, // total vsize 1000 10, // average fee rate 10 sat/vB ), - expected: &signer.CPFPFeeBumper{ + expected: &CPFPFeeBumper{ Ctx: context.Background(), Chain: chains.BitcoinMainnet, Tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), @@ -102,7 +101,7 @@ func Test_NewCPFPFeeBumper(t *testing.T) { } // ACT - bumper, err := signer.NewCPFPFeeBumper( + bumper, err := NewCPFPFeeBumper( context.Background(), tt.client, tt.chain, @@ -133,7 +132,7 @@ func Test_BumpTxFee(t *testing.T) { tests := []struct { name string - feeBumper *signer.CPFPFeeBumper + feeBumper *CPFPFeeBumper additionalFees int64 expectedNewRate uint64 expectedNewTx *wire.MsgTx @@ -141,7 +140,7 @@ func Test_BumpTxFee(t *testing.T) { }{ { name: "should bump tx fee successfully", - feeBumper: &signer.CPFPFeeBumper{ + feeBumper: &CPFPFeeBumper{ Tx: btcutil.NewTx(msgTx), MinRelayFee: 0.00001, CCTXRate: 55, @@ -155,14 +154,14 @@ func Test_BumpTxFee(t *testing.T) { expectedNewRate: 55, expectedNewTx: func() *wire.MsgTx { // deduct additional fees - newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx := CopyMsgTxNoWitness(msgTx) newTx.TxOut[2].Value -= 4632 return newTx }(), }, { name: "should give up all reserved bump fees", - feeBumper: &signer.CPFPFeeBumper{ + feeBumper: &CPFPFeeBumper{ Tx: func() *btcutil.Tx { // modify reserved bump fees to barely cover bump fees newTx := msgTx.Copy() @@ -181,14 +180,14 @@ func Test_BumpTxFee(t *testing.T) { expectedNewRate: 59, // (27213 + 6789) / 579 ≈ 59 expectedNewTx: func() *wire.MsgTx { // give up all reserved bump fees - newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx := CopyMsgTxNoWitness(msgTx) newTx.TxOut = newTx.TxOut[:2] return newTx }(), }, { name: "should set new gas rate to 'gasRateCap'", - feeBumper: &signer.CPFPFeeBumper{ + feeBumper: &CPFPFeeBumper{ Tx: btcutil.NewTx(msgTx), MinRelayFee: 0.00001, CCTXRate: 101, // > 100 @@ -202,14 +201,14 @@ func Test_BumpTxFee(t *testing.T) { expectedNewRate: 100, expectedNewTx: func() *wire.MsgTx { // deduct additional fees - newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx := CopyMsgTxNoWitness(msgTx) newTx.TxOut[2].Value -= 30687 return newTx }(), }, { name: "should fail if original tx has no reserved bump fees", - feeBumper: &signer.CPFPFeeBumper{ + feeBumper: &CPFPFeeBumper{ Tx: func() *btcutil.Tx { // remove the change output newTx := msgTx.Copy() @@ -221,7 +220,7 @@ func Test_BumpTxFee(t *testing.T) { }, { name: "should hold on RBF if additional fees is lower than min relay fees", - feeBumper: &signer.CPFPFeeBumper{ + feeBumper: &CPFPFeeBumper{ Tx: btcutil.NewTx(msgTx), MinRelayFee: 0.00002, // min relay fee will be 579vB * 2 = 1158 sats CCTXRate: 6, @@ -260,7 +259,7 @@ func Test_FetchFeeBumpInfo(t *testing.T) { tx *btcutil.Tx liveRate uint64 memplTxsInfo *mempoolTxsInfo - expected *signer.CPFPFeeBumper + expected *CPFPFeeBumper errMsg string }{ { @@ -273,7 +272,7 @@ func Test_FetchFeeBumpInfo(t *testing.T) { 1000, // total vsize 1000 10, // average fee rate 10 sat/vB ), - expected: &signer.CPFPFeeBumper{ + expected: &CPFPFeeBumper{ Tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), LiveRate: 12, TotalTxs: 2, @@ -325,7 +324,7 @@ func Test_FetchFeeBumpInfo(t *testing.T) { } // ACT - bumper := &signer.CPFPFeeBumper{ + bumper := &CPFPFeeBumper{ RPC: client, Tx: tt.tx, Logger: log.Logger, @@ -351,7 +350,7 @@ func Test_CopyMsgTxNoWitness(t *testing.T) { msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chain.ChainId, txid) // make a non-witness copy - copyTx := signer.CopyMsgTxNoWitness(msgTx) + copyTx := CopyMsgTxNoWitness(msgTx) // make another copy and clear witness data manually newTx := msgTx.Copy() @@ -365,7 +364,7 @@ func Test_CopyMsgTxNoWitness(t *testing.T) { t.Run("should handle nil input", func(t *testing.T) { require.Panics(t, func() { - signer.CopyMsgTxNoWitness(nil) + CopyMsgTxNoWitness(nil) }, "should panic on nil input") }) } diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index ebc1c13803..335ce63542 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -1,4 +1,4 @@ -package signer_test +package signer import ( "context" @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/testutil/sample" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/pkg/chains" @@ -84,7 +83,7 @@ func Test_SignRBFTx(t *testing.T) { chain chains.Chain lastTx *btcutil.Tx preTxs []prevTx - txData signer.OutboundData + txData OutboundData liveRate uint64 memplTxsInfo *mempoolTxsInfo errMsg string @@ -105,7 +104,7 @@ func Test_SignRBFTx(t *testing.T) { ), expectedTx: func() *wire.MsgTx { // deduct additional fees - newTx := signer.CopyMsgTxNoWitness(msgTx) + newTx := CopyMsgTxNoWitness(msgTx) newTx.TxOut[2].Value -= 5790 return newTx }(), @@ -231,7 +230,7 @@ func Test_SignRBFTx(t *testing.T) { } // mkTxData creates a new outbound data for testing -func mkTxData(t *testing.T, minRelayFee float64, latestFeeRate string) signer.OutboundData { +func mkTxData(t *testing.T, minRelayFee float64, latestFeeRate string) OutboundData { net := &chaincfg.MainNetParams cctx := sample.CrossChainTx(t, "0x123") cctx.InboundParams.CoinType = coin.CoinType_Gas @@ -241,7 +240,7 @@ func mkTxData(t *testing.T, minRelayFee float64, latestFeeRate string) signer.Ou cctx.GetCurrentOutboundParam().ReceiverChainId = chains.BitcoinMainnet.ChainId cctx.GetCurrentOutboundParam().Amount = sdkmath.NewUint(1e7) // 0.1 BTC - txData, err := signer.NewOutboundData(cctx, 1, minRelayFee, zerolog.Nop(), zerolog.Nop()) + txData, err := NewOutboundData(cctx, 1, minRelayFee, zerolog.Nop(), zerolog.Nop()) require.NoError(t, err) return *txData } diff --git a/zetaclient/chains/bitcoin/signer/sign_test.go b/zetaclient/chains/bitcoin/signer/sign_test.go index fe68de3c9c..149df97318 100644 --- a/zetaclient/chains/bitcoin/signer/sign_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_test.go @@ -1,4 +1,4 @@ -package signer_test +package signer import ( "context" @@ -24,7 +24,6 @@ import ( crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" "github.com/zeta-chain/node/zetaclient/testutils/mocks" ) @@ -45,9 +44,9 @@ func Test_SignWithdrawTx(t *testing.T) { } // helper function to create tx data - mkTxData := func(height uint64, minRelayFee float64) signer.OutboundData { + mkTxData := func(height uint64, minRelayFee float64) OutboundData { cctx := mkCCTX(t) - txData, err := signer.NewOutboundData(cctx, height, minRelayFee, zerolog.Nop(), zerolog.Nop()) + txData, err := NewOutboundData(cctx, height, minRelayFee, zerolog.Nop(), zerolog.Nop()) require.NoError(t, err) return *txData } @@ -55,7 +54,7 @@ func Test_SignWithdrawTx(t *testing.T) { tests := []struct { name string chain chains.Chain - txData signer.OutboundData + txData OutboundData failFetchUTXOs bool failSignTx bool fail bool @@ -185,7 +184,7 @@ func Test_AddTxInputs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // create tx msg and add inputs tx := wire.NewMsgTx(wire.TxVersion) - inAmounts, err := signer.AddTxInputs(tx, tt.utxos) + inAmounts, err := AddTxInputs(tx, tt.utxos) // assert if tt.fail { @@ -206,7 +205,7 @@ func Test_AddWithdrawTxOutputs(t *testing.T) { mocks.NewTSS(t).FakePubKey(testutils.TSSPubKeyMainnet), base.DefaultLogger(), ) - signer := signer.New( + signer := New( baseSigner, mocks.NewBitcoinClient(t), ) @@ -414,7 +413,7 @@ func Test_SignTx(t *testing.T) { Amount: amount, }) } - inAmounts, err := signer.AddTxInputs(tx, utxos) + inAmounts, err := AddTxInputs(tx, utxos) require.NoError(t, err) require.Len(t, inAmounts, len(tt.inputs)) diff --git a/zetaclient/chains/bitcoin/signer/signer_test.go b/zetaclient/chains/bitcoin/signer/signer_test.go index 641d23636d..dfbf65e4a4 100644 --- a/zetaclient/chains/bitcoin/signer/signer_test.go +++ b/zetaclient/chains/bitcoin/signer/signer_test.go @@ -1,4 +1,4 @@ -package signer_test +package signer import ( "context" @@ -22,7 +22,6 @@ import ( observertypes "github.com/zeta-chain/node/x/observer/types" "github.com/zeta-chain/node/zetaclient/chains/base" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" - "github.com/zeta-chain/node/zetaclient/chains/bitcoin/signer" "github.com/zeta-chain/node/zetaclient/config" zctx "github.com/zeta-chain/node/zetaclient/context" "github.com/zeta-chain/node/zetaclient/db" @@ -37,7 +36,7 @@ import ( var TestDataDir = "../../../" type testSuite struct { - *signer.Signer + *Signer observer *observer.Observer tss *mocks.TSS client *mocks.BitcoinClient @@ -68,7 +67,7 @@ func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { // create signer baseSigner := base.NewSigner(chain, tss, baseLogger) - signer := signer.New(baseSigner, rpcClient) + signer := New(baseSigner, rpcClient) // create test suite and observer suite := &testSuite{ From 6f57fb54c5c6d37201384d3b1ad048ea7a41176d Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 13:39:42 -0500 Subject: [PATCH 59/74] unexport FetchFeeBumpInfo --- .../chains/bitcoin/signer/fee_bumper.go | 4 +- .../chains/bitcoin/signer/fee_bumper_test.go | 122 ++++-------------- 2 files changed, 28 insertions(+), 98 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go index 1137a42a9b..07fd6da323 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -87,7 +87,7 @@ func NewCPFPFeeBumper( Logger: logger, } - err := fb.FetchFeeBumpInfo() + err := fb.fetchFeeBumpInfo() if err != nil { return nil, err } @@ -170,7 +170,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { } // fetchFeeBumpInfo fetches all necessary information needed to bump the stuck tx -func (b *CPFPFeeBumper) FetchFeeBumpInfo() error { +func (b *CPFPFeeBumper) fetchFeeBumpInfo() error { // query live fee rate liveRate, err := b.RPC.GetEstimatedFeeRate(b.Ctx, 1) if err != nil { diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index 475815fc07..e3b581b9c0 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -74,30 +74,52 @@ func Test_NewCPFPFeeBumper(t *testing.T) { Logger: log.Logger, }, }, + { + chain: chains.BitcoinMainnet, + name: "should fail if unable to estimate smart fee", + client: mocks.NewBitcoinClient(t), + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + liveRate: 0, + errMsg: "GetEstimatedFeeRate failed", + }, { chain: chains.BitcoinMainnet, - name: "should fail when mempool txs info fetcher returns error", + name: "should fail if unable to fetch mempool txs info", client: mocks.NewBitcoinClient(t), tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), liveRate: 12, memplTxsInfo: nil, errMsg: "unable to fetch mempool txs info", }, + { + chain: chains.BitcoinMainnet, + name: "should fail if unable to convert total fees", + client: mocks.NewBitcoinClient(t), + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + liveRate: 12, + memplTxsInfo: newMempoolTxsInfo(2, 21000000.1, 1000, 10), // fee exceeds max BTC supply + errMsg: "cannot convert total fees", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // ARRANGE // mock RPC fee rate - tt.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(tt.liveRate, nil) + if tt.liveRate > 0 { + tt.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything).Return(tt.liveRate, nil) + } else { + tt.client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything).Return(uint64(0), errors.New("rpc error")) + } // mock mempool txs information if tt.memplTxsInfo != nil { - tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything). + tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) } else { v := int64(0) - tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Return(v, 0.0, v, uint64(0), errors.New("rpc error")) + tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). + Return(v, 0.0, v, uint64(0), errors.New("rpc error")) } // ACT @@ -251,98 +273,6 @@ func Test_BumpTxFee(t *testing.T) { } } -func Test_FetchFeeBumpInfo(t *testing.T) { - liveRate := uint64(12) - - tests := []struct { - name string - tx *btcutil.Tx - liveRate uint64 - memplTxsInfo *mempoolTxsInfo - expected *CPFPFeeBumper - errMsg string - }{ - { - name: "should fetch fee bump info successfully", - tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), - liveRate: 12, - memplTxsInfo: newMempoolTxsInfo( - 2, // 2 stuck TSS txs - 0.0001, // total fees 0.0001 BTC - 1000, // total vsize 1000 - 10, // average fee rate 10 sat/vB - ), - expected: &CPFPFeeBumper{ - Tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), - LiveRate: 12, - TotalTxs: 2, - TotalFees: 10000, - TotalVSize: 1000, - AvgFeeRate: 10, - Logger: log.Logger, - }, - }, - { - name: "should fail if unable to estimate smart fee", - liveRate: 0, - errMsg: "GetEstimatedFeeRate failed", - }, - { - name: "should fail if unable to fetch mempool txs info", - liveRate: 12, - tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), - memplTxsInfo: nil, - errMsg: "unable to fetch mempool txs info", - }, - { - name: "should fail on invalid total fees", - liveRate: 12, - tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), - memplTxsInfo: newMempoolTxsInfo(2, 21000000.1, 1000, 10), // fee exceeds max BTC supply - errMsg: "cannot convert total fees", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // ARRANGE - // mock RPC fee rate - client := mocks.NewBitcoinClient(t) - if tt.liveRate > 0 { - client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(liveRate, nil) - } else { - client.On("GetEstimatedFeeRate", mock.Anything, mock.Anything, mock.Anything).Return(uint64(0), errors.New("rpc error")) - } - - // mock mempool txs information - if tt.memplTxsInfo != nil { - client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything). - Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) - } else { - v := int64(0) - client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(v, 0.0, v, uint64(0), errors.New("rpc error")) - } - - // ACT - bumper := &CPFPFeeBumper{ - RPC: client, - Tx: tt.tx, - Logger: log.Logger, - } - err := bumper.FetchFeeBumpInfo() - - // ASSERT - if tt.errMsg != "" { - require.ErrorContains(t, err, tt.errMsg) - } else { - bumper.RPC = nil // ignore the RPC client - require.NoError(t, err) - require.Equal(t, tt.expected, bumper) - } - }) - } -} - func Test_CopyMsgTxNoWitness(t *testing.T) { t.Run("should copy tx msg without witness", func(t *testing.T) { chain := chains.BitcoinMainnet From 851a53e851145f08bb17b533a512ee5258fa1858 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 15:13:23 -0500 Subject: [PATCH 60/74] have GetTotalMempoolParentsSizeNFees return a struct; add function IsTxNotInMempoolErr to wrap reusable logic --- zetaclient/chains/bitcoin/client/commands.go | 2 +- zetaclient/chains/bitcoin/client/helpers.go | 60 ++++---- zetaclient/chains/bitcoin/client/mockgen.go | 6 +- zetaclient/chains/bitcoin/observer/mempool.go | 6 +- .../chains/bitcoin/signer/fee_bumper.go | 46 ++---- .../chains/bitcoin/signer/fee_bumper_test.go | 136 ++++++++---------- zetaclient/chains/bitcoin/signer/sign_rbf.go | 2 +- .../chains/bitcoin/signer/sign_rbf_test.go | 76 +++++----- zetaclient/chains/bitcoin/signer/signer.go | 7 +- zetaclient/testutils/mocks/bitcoin_client.go | 79 ++++------ 10 files changed, 186 insertions(+), 234 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/commands.go b/zetaclient/chains/bitcoin/client/commands.go index b0de9955e8..4ff19a374a 100644 --- a/zetaclient/chains/bitcoin/client/commands.go +++ b/zetaclient/chains/bitcoin/client/commands.go @@ -82,7 +82,7 @@ func (c *Client) GetRawMempool(ctx context.Context) ([]*chainhash.Hash, error) { out, err := c.sendCommand(ctx, cmd) if err != nil { - return nil, errors.Wrap(err, "unable to list unspent") + return nil, errors.Wrap(err, "unable to get raw mempool") } txHashStrs, err := unmarshal[[]string](out) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 73fbaedb8a..811d775b79 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -26,6 +26,14 @@ const ( maxBTCSupply = 21000000.0 ) +// MempoolTxsAndFees contains the information of pending mempool txs and fees +type MempoolTxsAndFees struct { + TotalTxs int64 + TotalFees int64 + TotalVSize int64 + AvgFeeRate uint64 +} + // IsRegnet returns true if the chain is regnet func (c *Client) IsRegnet() bool { return c.isRegnet @@ -225,24 +233,17 @@ func (c *Client) Healthcheck(ctx context.Context) (time.Time, error) { return header.Timestamp, nil } -// GetTotalMempoolParentsSizeNFees returns the information of all pending parent txs of a given tx (inclusive) +// GetMempoolTxsAndFees returns the information of all pending parent txs and fees of a given tx (inclusive) // // A parent tx is defined as: // - a tx that is also pending in the mempool // - a tx that has its first output spent by the child as first input -// -// Returns: (totalTxs, totalFees, totalVSize, error) -func (c *Client) GetTotalMempoolParentsSizeNFees( +func (c *Client) GetMempoolTxsAndFees( ctx context.Context, childHash string, timeout time.Duration, -) (int64, float64, int64, uint64, error) { - var ( - totalTxs int64 - totalFees float64 - totalVSize int64 - avgFeeRate uint64 - ) +) (txsAndFees MempoolTxsAndFees, err error) { + totalFeesFloat := float64(0) // loop through all parents startTime := time.Now() @@ -250,46 +251,57 @@ func (c *Client) GetTotalMempoolParentsSizeNFees( for { memplEntry, err := c.GetMempoolEntry(ctx, parentHash) if err != nil { - if strings.Contains(err.Error(), "Transaction not in mempool") { + if IsTxNotInMempoolError(err) { // not a mempool tx, stop looking for parents break } - return 0, 0, 0, 0, errors.Wrapf(err, "unable to get mempool entry for tx %s", parentHash) + return txsAndFees, errors.Wrapf(err, "unable to get mempool entry for tx %s", parentHash) } // accumulate fees and vsize - totalTxs++ - totalFees += memplEntry.Fee - totalVSize += int64(memplEntry.VSize) + txsAndFees.TotalTxs++ + totalFeesFloat += memplEntry.Fee + txsAndFees.TotalVSize += int64(memplEntry.VSize) // find the parent tx tx, err := c.GetRawTransactionByStr(ctx, parentHash) if err != nil { - return 0, 0, 0, 0, errors.Wrapf(err, "unable to get tx %s", parentHash) + return txsAndFees, errors.Wrapf(err, "unable to get tx %s", parentHash) } parentHash = tx.MsgTx().TxIn[0].PreviousOutPoint.Hash.String() // check timeout to avoid infinite loop if time.Since(startTime) > timeout { - return 0, 0, 0, 0, errors.Errorf("timeout reached on %dth tx: %s", totalTxs, parentHash) + return txsAndFees, errors.Errorf("timeout reached on %dth tx: %s", txsAndFees.TotalTxs, parentHash) } } // no pending tx found - if totalTxs == 0 { - return 0, 0, 0, 0, errors.Errorf("given tx is not pending: %s", childHash) + if txsAndFees.TotalTxs == 0 { + return txsAndFees, errors.Errorf("given tx is not pending: %s", childHash) + } + + // convert total fees to satoshis + txsAndFees.TotalFees, err = common.GetSatoshis(totalFeesFloat) + if err != nil { + return txsAndFees, errors.Wrapf(err, "invalid total fees: %f", totalFeesFloat) } // sanity check, should never happen - if totalFees < 0 || totalVSize <= 0 { - return 0, 0, 0, 0, errors.Errorf("invalid result: totalFees %f, totalVSize %d", totalFees, totalVSize) + if txsAndFees.TotalVSize <= 0 { + return txsAndFees, errors.Errorf("invalid totalVSize %d", txsAndFees.TotalVSize) } // calculate the average fee rate // #nosec G115 always positive - avgFeeRate = uint64(math.Ceil(totalFees / float64(totalVSize))) + txsAndFees.AvgFeeRate = uint64(math.Ceil(totalFeesFloat / float64(txsAndFees.TotalVSize))) + + return txsAndFees, nil +} - return totalTxs, totalFees, totalVSize, avgFeeRate, nil +// IsTxNotInMempoolError checks if the given error is due to the transaction not being in the mempool. +func IsTxNotInMempoolError(err error) bool { + return strings.Contains(err.Error(), "Transaction not in mempool") } func strToHash(s string) (*chainhash.Hash, error) { diff --git a/zetaclient/chains/bitcoin/client/mockgen.go b/zetaclient/chains/bitcoin/client/mockgen.go index d22f077fe6..cf07658838 100644 --- a/zetaclient/chains/bitcoin/client/mockgen.go +++ b/zetaclient/chains/bitcoin/client/mockgen.go @@ -32,11 +32,7 @@ type client interface { GetRawTransactionVerbose(ctx context.Context, hash *hash.Hash) (*types.TxRawResult, error) GetMempoolEntry(ctx context.Context, txHash string) (*types.GetMempoolEntryResult, error) GetRawMempool(ctx context.Context) ([]*hash.Hash, error) - GetTotalMempoolParentsSizeNFees( - ctx context.Context, - childHash string, - timeout time.Duration, - ) (int64, float64, int64, uint64, error) + GetMempoolTxsAndFees(ctx context.Context, childHash string, timeout time.Duration) (MempoolTxsAndFees, error) GetRawTransactionResult( ctx context.Context, diff --git a/zetaclient/chains/bitcoin/observer/mempool.go b/zetaclient/chains/bitcoin/observer/mempool.go index 89fc133edb..dc5064c903 100644 --- a/zetaclient/chains/bitcoin/observer/mempool.go +++ b/zetaclient/chains/bitcoin/observer/mempool.go @@ -2,13 +2,13 @@ package observer import ( "context" - "strings" "time" "github.com/btcsuite/btcd/btcutil" "github.com/pkg/errors" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" "github.com/zeta-chain/node/zetaclient/logs" ) @@ -210,7 +210,7 @@ func IsTxStuckInMempool( memplEntry, err := rpc.GetMempoolEntry(ctx, txHash) if err != nil { - if strings.Contains(err.Error(), "Transaction not in mempool") { + if client.IsTxNotInMempoolError(err) { return false, 0, nil // not a mempool tx, of course not stuck } return false, 0, errors.Wrap(err, "GetMempoolEntry failed") @@ -242,7 +242,7 @@ func IsTxStuckInMempoolRegnet( memplEntry, err := rpc.GetMempoolEntry(ctx, txHash) if err != nil { - if strings.Contains(err.Error(), "Transaction not in mempool") { + if client.IsTxNotInMempoolError(err) { return false, 0, nil // not a mempool tx, of course not stuck } return false, 0, errors.Wrap(err, "GetMempoolEntry failed") diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go index 07fd6da323..86ec3519b8 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -16,6 +16,7 @@ import ( "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/pkg/constant" mathpkg "github.com/zeta-chain/node/pkg/math" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/common" ) @@ -52,17 +53,8 @@ type CPFPFeeBumper struct { // LiveRate is the most recent market fee rate LiveRate uint64 - // TotalTxs is the total number of stuck TSS txs - TotalTxs int64 - - // TotalFees is the total fees of all stuck TSS txs - TotalFees int64 - - // TotalVSize is the total vsize of all stuck TSS txs - TotalVSize int64 - - // AvgFeeRate is the average fee rate of all stuck TSS txs - AvgFeeRate uint64 + // TxsAndFees contains the information of all pending txs and fees + TxsAndFees client.MempoolTxsAndFees Logger zerolog.Logger } @@ -105,10 +97,10 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { // the new fee rate is supposed to be much higher than current paid rate (old rate). // we print a warning message if it's not the case for monitoring purposes. // #nosec G115 always positive - oldRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.AvgFeeRate), decentFeeBumpPercent) + oldRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.TxsAndFees.AvgFeeRate), decentFeeBumpPercent) if sdkmath.NewUint(b.CCTXRate).LT(oldRateBumped) { b.Logger.Warn(). - Uint64("old_fee_rate", b.AvgFeeRate). + Uint64("old_fee_rate", b.TxsAndFees.AvgFeeRate). Uint64("new_fee_rate", b.CCTXRate). Msg("new fee rate is not much higher than the old fee rate") } @@ -143,7 +135,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { // // see: https://github.com/bitcoin/bitcoin/blob/5b8046a6e893b7fad5a93631e6d1e70db31878af/src/policy/rbf.cpp#L166-L183 // #nosec G115 always in range - additionalFees := b.TotalVSize*int64(feeRateNew) - b.TotalFees + additionalFees := b.TxsAndFees.TotalVSize*int64(feeRateNew) - b.TxsAndFees.TotalFees if additionalFees < minRelayTxFees { return nil, 0, 0, fmt.Errorf( "hold on RBF: additional fees %d is lower than min relay fees %d", @@ -164,7 +156,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { // effective fee rate // #nosec G115 always positive - feeRateNew = uint64(math.Ceil(float64(b.TotalFees+additionalFees) / float64(b.TotalVSize))) + feeRateNew = uint64(math.Ceil(float64(b.TxsAndFees.TotalFees+additionalFees) / float64(b.TxsAndFees.TotalVSize))) return newTx, additionalFees, feeRateNew, nil } @@ -179,29 +171,17 @@ func (b *CPFPFeeBumper) fetchFeeBumpInfo() error { b.LiveRate = liveRate // query total fees and sizes of all pending parent TSS txs - totalTxs, totalFees, totalVSize, avgFeeRate, err := b.RPC.GetTotalMempoolParentsSizeNFees( - b.Ctx, - b.Tx.MsgTx().TxID(), - time.Minute, - ) + txsAndFees, err := b.RPC.GetMempoolTxsAndFees(b.Ctx, b.Tx.MsgTx().TxID(), time.Minute) if err != nil { return errors.Wrap(err, "unable to fetch mempool txs info") } - totalFeesSats, err := common.GetSatoshis(totalFees) - if err != nil { - return errors.Wrapf(err, "cannot convert total fees %f", totalFees) - } - - b.TotalTxs = totalTxs - b.TotalFees = totalFeesSats - b.TotalVSize = totalVSize - b.AvgFeeRate = avgFeeRate + b.TxsAndFees = txsAndFees b.Logger.Info(). - Int64("total_txs", totalTxs). - Int64("total_fees", totalFeesSats). - Int64("total_vsize", totalVSize). - Uint64("avg_fee_rate", avgFeeRate). + Int64("total_txs", b.TxsAndFees.TotalTxs). + Int64("total_fees", b.TxsAndFees.TotalFees). + Int64("total_vsize", b.TxsAndFees.TotalVSize). + Uint64("avg_fee_rate", b.TxsAndFees.AvgFeeRate). Msg("fetched fee bump information") return nil diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index e3b581b9c0..5bf8d7bce6 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -12,39 +12,23 @@ import ( "github.com/stretchr/testify/require" "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/pkg/constant" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/zetaclient/testutils/mocks" ) -// mempoolTxsInfo is a helper struct to contain mempool txs information -type mempoolTxsInfo struct { - totalTxs int64 - totalFees float64 - totalVSize int64 - avgFeeRate uint64 -} - -func newMempoolTxsInfo(totalTxs int64, totalFees float64, totalVSize int64, avgFeeRate uint64) *mempoolTxsInfo { - return &mempoolTxsInfo{ - totalTxs: totalTxs, - totalFees: totalFees, - totalVSize: totalVSize, - avgFeeRate: avgFeeRate, - } -} - func Test_NewCPFPFeeBumper(t *testing.T) { tests := []struct { - name string - chain chains.Chain - client *mocks.BitcoinClient - tx *btcutil.Tx - cctxRate uint64 - liveRate uint64 - minRelayFee float64 - memplTxsInfo *mempoolTxsInfo - errMsg string - expected *CPFPFeeBumper + name string + chain chains.Chain + client *mocks.BitcoinClient + tx *btcutil.Tx + cctxRate uint64 + liveRate uint64 + minRelayFee float64 + txsAndFees *client.MempoolTxsAndFees + errMsg string + expected *CPFPFeeBumper }{ { chain: chains.BitcoinMainnet, @@ -54,12 +38,12 @@ func Test_NewCPFPFeeBumper(t *testing.T) { cctxRate: 10, liveRate: 12, minRelayFee: 0.00001, - memplTxsInfo: newMempoolTxsInfo( - 2, // 2 stuck TSS txs - 0.0001, // total fees 0.0001 BTC - 1000, // total vsize 1000 - 10, // average fee rate 10 sat/vB - ), + txsAndFees: &client.MempoolTxsAndFees{ + TotalTxs: 2, // 2 stuck TSS txs + TotalFees: 10000, // total fees 0.0001 BTC + TotalVSize: 1000, // total vsize 1000 + AvgFeeRate: 10, // average fee rate 10 sat/vB + }, expected: &CPFPFeeBumper{ Ctx: context.Background(), Chain: chains.BitcoinMainnet, @@ -67,11 +51,13 @@ func Test_NewCPFPFeeBumper(t *testing.T) { MinRelayFee: 0.00001, CCTXRate: 10, LiveRate: 12, - TotalTxs: 2, - TotalFees: 10000, - TotalVSize: 1000, - AvgFeeRate: 10, - Logger: log.Logger, + TxsAndFees: client.MempoolTxsAndFees{ + TotalTxs: 2, + TotalFees: 10000, + TotalVSize: 1000, + AvgFeeRate: 10, + }, + Logger: log.Logger, }, }, { @@ -83,22 +69,13 @@ func Test_NewCPFPFeeBumper(t *testing.T) { errMsg: "GetEstimatedFeeRate failed", }, { - chain: chains.BitcoinMainnet, - name: "should fail if unable to fetch mempool txs info", - client: mocks.NewBitcoinClient(t), - tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), - liveRate: 12, - memplTxsInfo: nil, - errMsg: "unable to fetch mempool txs info", - }, - { - chain: chains.BitcoinMainnet, - name: "should fail if unable to convert total fees", - client: mocks.NewBitcoinClient(t), - tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), - liveRate: 12, - memplTxsInfo: newMempoolTxsInfo(2, 21000000.1, 1000, 10), // fee exceeds max BTC supply - errMsg: "cannot convert total fees", + chain: chains.BitcoinMainnet, + name: "should fail if unable to fetch mempool txs info", + client: mocks.NewBitcoinClient(t), + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + liveRate: 12, + txsAndFees: nil, + errMsg: "unable to fetch mempool txs info", }, } @@ -113,13 +90,12 @@ func Test_NewCPFPFeeBumper(t *testing.T) { } // mock mempool txs information - if tt.memplTxsInfo != nil { - tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) + if tt.txsAndFees != nil { + tt.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). + Return(*tt.txsAndFees, nil) } else { - v := int64(0) - tt.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return(v, 0.0, v, uint64(0), errors.New("rpc error")) + tt.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). + Return(client.MempoolTxsAndFees{}, errors.New("rpc error")) } // ACT @@ -167,10 +143,12 @@ func Test_BumpTxFee(t *testing.T) { MinRelayFee: 0.00001, CCTXRate: 55, LiveRate: 67, - TotalFees: 27213, - TotalVSize: 579, - AvgFeeRate: 47, - Logger: log.Logger, + TxsAndFees: client.MempoolTxsAndFees{ + TotalFees: 27213, + TotalVSize: 579, + AvgFeeRate: 47, + }, + Logger: log.Logger, }, additionalFees: 4632, // 579*55 - 27213 expectedNewRate: 55, @@ -193,10 +171,12 @@ func Test_BumpTxFee(t *testing.T) { MinRelayFee: 0.00001, CCTXRate: 57, LiveRate: 67, - TotalFees: 27213, - TotalVSize: 579, - AvgFeeRate: 47, - Logger: log.Logger, + TxsAndFees: client.MempoolTxsAndFees{ + TotalFees: 27213, + TotalVSize: 579, + AvgFeeRate: 47, + }, + Logger: log.Logger, }, additionalFees: 6789, // same as the reserved value in 2nd output expectedNewRate: 59, // (27213 + 6789) / 579 ≈ 59 @@ -214,10 +194,12 @@ func Test_BumpTxFee(t *testing.T) { MinRelayFee: 0.00001, CCTXRate: 101, // > 100 LiveRate: 120, - TotalFees: 27213, - TotalVSize: 579, - AvgFeeRate: 47, - Logger: log.Logger, + TxsAndFees: client.MempoolTxsAndFees{ + TotalFees: 27213, + TotalVSize: 579, + AvgFeeRate: 47, + }, + Logger: log.Logger, }, additionalFees: 30687, // (100-47)*579 expectedNewRate: 100, @@ -247,10 +229,12 @@ func Test_BumpTxFee(t *testing.T) { MinRelayFee: 0.00002, // min relay fee will be 579vB * 2 = 1158 sats CCTXRate: 6, LiveRate: 7, - TotalFees: 2895, - TotalVSize: 579, - AvgFeeRate: 5, - Logger: log.Logger, + TxsAndFees: client.MempoolTxsAndFees{ + TotalFees: 2895, + TotalVSize: 579, + AvgFeeRate: 5, + }, + Logger: log.Logger, }, errMsg: "lower than min relay fees", }, diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go index b4f128bd79..4004b18030 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -57,7 +57,7 @@ func (signer *Signer) SignRBFTx(ctx context.Context, txData *OutboundData, lastT return nil, errors.Wrap(err, "BumpTxFee failed") } logger.Info(). - Uint64("old_fee_rate", fb.AvgFeeRate). + Uint64("old_fee_rate", fb.TxsAndFees.AvgFeeRate). Uint64("new_fee_rate", newRate). Int64("additional_fees", additionalFees). Msg("BumpTxFee succeed") diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index 335ce63542..3b590b7343 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/testutil/sample" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" "github.com/zeta-chain/node/zetaclient/testutils" "github.com/zeta-chain/node/pkg/chains" @@ -79,15 +80,15 @@ func Test_SignRBFTx(t *testing.T) { // test cases tests := []struct { - name string - chain chains.Chain - lastTx *btcutil.Tx - preTxs []prevTx - txData OutboundData - liveRate uint64 - memplTxsInfo *mempoolTxsInfo - errMsg string - expectedTx *wire.MsgTx + name string + chain chains.Chain + lastTx *btcutil.Tx + preTxs []prevTx + txData OutboundData + liveRate uint64 + txsAndFees *client.MempoolTxsAndFees + errMsg string + expectedTx *wire.MsgTx }{ { name: "should sign RBF tx successfully", @@ -96,12 +97,12 @@ func Test_SignRBFTx(t *testing.T) { preTxs: preTxs, txData: mkTxData(t, 0.00001, "57"), // 57 sat/vB as cctx rate liveRate: 59, // 59 sat/vB - memplTxsInfo: newMempoolTxsInfo( - 1, // 1 stuck tx - 0.00027213, // fees: 0.00027213 BTC - 579, // size: 579 vByte - 47, // rate: 47 sat/vB - ), + txsAndFees: &client.MempoolTxsAndFees{ + TotalTxs: 1, // 1 stuck tx + TotalFees: 27213, // fees: 0.00027213 BTC + TotalVSize: 579, // size: 579 vByte + AvgFeeRate: 47, // rate: 47 sat/vB + }, expectedTx: func() *wire.MsgTx { // deduct additional fees newTx := CopyMsgTxNoWitness(msgTx) @@ -117,12 +118,12 @@ func Test_SignRBFTx(t *testing.T) { errMsg: "fee rate is not bumped by zetacore yet", }, { - name: "should return error if unable to create fee bumper", - chain: chains.BitcoinMainnet, - lastTx: btcutil.NewTx(msgTx.Copy()), - txData: mkTxData(t, 0.00001, "57"), - memplTxsInfo: nil, // no mempool txs info provided - errMsg: "NewCPFPFeeBumper failed", + name: "should return error if unable to create fee bumper", + chain: chains.BitcoinMainnet, + lastTx: btcutil.NewTx(msgTx.Copy()), + txData: mkTxData(t, 0.00001, "57"), + txsAndFees: nil, // no mempool txs info provided + errMsg: "NewCPFPFeeBumper failed", }, { name: "should return error if unable to bump tx fee", @@ -134,12 +135,12 @@ func Test_SignRBFTx(t *testing.T) { }(), txData: mkTxData(t, 0.00001, "57"), // 57 sat/vB as cctx rate liveRate: 99, // 99 sat/vB is much higher than ccxt rate - memplTxsInfo: newMempoolTxsInfo( - 1, // 1 stuck tx - 0.00027213, // fees: 0.00027213 BTC - 579, // size: 579 vByte - 47, // rate: 47 sat/vB - ), + txsAndFees: &client.MempoolTxsAndFees{ + TotalTxs: 1, // 1 stuck tx + TotalFees: 27213, // fees: 0.00027213 BTC + TotalVSize: 579, // size: 579 vByte + AvgFeeRate: 47, // rate: 47 sat/vB + }, errMsg: "BumpTxFee failed", }, { @@ -149,12 +150,12 @@ func Test_SignRBFTx(t *testing.T) { txData: mkTxData(t, 0.00001, "57"), // 57 sat/vB as cctx rate preTxs: nil, // no previous info provided liveRate: 59, // 59 sat/vB - memplTxsInfo: newMempoolTxsInfo( - 1, // 1 stuck tx - 0.00027213, // fees: 0.00027213 BTC - 579, // size: 579 vByte - 47, // rate: 47 sat/vB - ), + txsAndFees: &client.MempoolTxsAndFees{ + TotalTxs: 1, // 1 stuck tx + TotalFees: 27213, // fees: 0.00027213 BTC + TotalVSize: 579, // size: 579 vByte + AvgFeeRate: 47, // rate: 47 sat/vB + }, errMsg: "unable to get previous tx", }, } @@ -176,11 +177,12 @@ func Test_SignRBFTx(t *testing.T) { } // mock mempool txs information - if tt.memplTxsInfo != nil { - s.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything). - Return(tt.memplTxsInfo.totalTxs, tt.memplTxsInfo.totalFees, tt.memplTxsInfo.totalVSize, tt.memplTxsInfo.avgFeeRate, nil) + if tt.txsAndFees != nil { + s.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything). + Return(*tt.txsAndFees, nil) } else { - s.client.On("GetTotalMempoolParentsSizeNFees", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(0, 0.0, 0, 0, "rpc error") + s.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). + Return(client.MempoolTxsAndFees{}, errors.New("rpc error")) } // mock RPC transactions diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 0ce31e2100..97717ae6ec 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -18,6 +18,7 @@ import ( "github.com/zeta-chain/node/pkg/retry" "github.com/zeta-chain/node/x/crosschain/types" "github.com/zeta-chain/node/zetaclient/chains/base" + "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" "github.com/zeta-chain/node/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/logs" @@ -37,11 +38,7 @@ type RPC interface { GetRawTransaction(ctx context.Context, hash *chainhash.Hash) (*btcutil.Tx, error) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (uint64, error) SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*chainhash.Hash, error) - GetTotalMempoolParentsSizeNFees( - ctx context.Context, - childHash string, - timeout time.Duration, - ) (int64, float64, int64, uint64, error) + GetMempoolTxsAndFees(ctx context.Context, childHash string, timeout time.Duration) (client.MempoolTxsAndFees, error) } // Signer deals with signing & broadcasting BTC transactions. diff --git a/zetaclient/testutils/mocks/bitcoin_client.go b/zetaclient/testutils/mocks/bitcoin_client.go index 4f2a43414d..2415c2e972 100644 --- a/zetaclient/testutils/mocks/bitcoin_client.go +++ b/zetaclient/testutils/mocks/bitcoin_client.go @@ -8,6 +8,8 @@ import ( chainhash "github.com/btcsuite/btcd/chaincfg/chainhash" + client "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" + context "context" json "encoding/json" @@ -385,6 +387,34 @@ func (_m *BitcoinClient) GetMempoolEntry(ctx context.Context, txHash string) (*b return r0, r1 } +// GetMempoolTxsAndFees provides a mock function with given fields: ctx, childHash, timeout +func (_m *BitcoinClient) GetMempoolTxsAndFees(ctx context.Context, childHash string, timeout time.Duration) (client.MempoolTxsAndFees, error) { + ret := _m.Called(ctx, childHash, timeout) + + if len(ret) == 0 { + panic("no return value specified for GetMempoolTxsAndFees") + } + + var r0 client.MempoolTxsAndFees + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) (client.MempoolTxsAndFees, error)); ok { + return rf(ctx, childHash, timeout) + } + if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) client.MempoolTxsAndFees); ok { + r0 = rf(ctx, childHash, timeout) + } else { + r0 = ret.Get(0).(client.MempoolTxsAndFees) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, time.Duration) error); ok { + r1 = rf(ctx, childHash, timeout) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetNetworkInfo provides a mock function with given fields: ctx func (_m *BitcoinClient) GetNetworkInfo(ctx context.Context) (*btcjson.GetNetworkInfoResult, error) { ret := _m.Called(ctx) @@ -593,55 +623,6 @@ func (_m *BitcoinClient) GetRawTransactionVerbose(ctx context.Context, hash *cha return r0, r1 } -// GetTotalMempoolParentsSizeNFees provides a mock function with given fields: ctx, childHash, timeout -func (_m *BitcoinClient) GetTotalMempoolParentsSizeNFees(ctx context.Context, childHash string, timeout time.Duration) (int64, float64, int64, uint64, error) { - ret := _m.Called(ctx, childHash, timeout) - - if len(ret) == 0 { - panic("no return value specified for GetTotalMempoolParentsSizeNFees") - } - - var r0 int64 - var r1 float64 - var r2 int64 - var r3 uint64 - var r4 error - if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) (int64, float64, int64, uint64, error)); ok { - return rf(ctx, childHash, timeout) - } - if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) int64); ok { - r0 = rf(ctx, childHash, timeout) - } else { - r0 = ret.Get(0).(int64) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, time.Duration) float64); ok { - r1 = rf(ctx, childHash, timeout) - } else { - r1 = ret.Get(1).(float64) - } - - if rf, ok := ret.Get(2).(func(context.Context, string, time.Duration) int64); ok { - r2 = rf(ctx, childHash, timeout) - } else { - r2 = ret.Get(2).(int64) - } - - if rf, ok := ret.Get(3).(func(context.Context, string, time.Duration) uint64); ok { - r3 = rf(ctx, childHash, timeout) - } else { - r3 = ret.Get(3).(uint64) - } - - if rf, ok := ret.Get(4).(func(context.Context, string, time.Duration) error); ok { - r4 = rf(ctx, childHash, timeout) - } else { - r4 = ret.Error(4) - } - - return r0, r1, r2, r3, r4 -} - // GetTransaction provides a mock function with given fields: ctx, hash func (_m *BitcoinClient) GetTransaction(ctx context.Context, hash *chainhash.Hash) (*btcjson.GetTransactionResult, error) { ret := _m.Called(ctx, hash) From da87ab58a0e567b7cbbf211fc1746da21f173ff1 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 16:31:32 -0500 Subject: [PATCH 61/74] move timeout handling into context --- zetaclient/chains/bitcoin/client/helpers.go | 8 ++++---- zetaclient/chains/bitcoin/client/mockgen.go | 2 +- zetaclient/chains/bitcoin/signer/fee_bumper.go | 6 +++++- .../chains/bitcoin/signer/fee_bumper_test.go | 6 ++---- .../chains/bitcoin/signer/sign_rbf_test.go | 6 ++---- zetaclient/chains/bitcoin/signer/signer.go | 2 +- zetaclient/testutils/mocks/bitcoin_client.go | 18 +++++++++--------- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/helpers.go b/zetaclient/chains/bitcoin/client/helpers.go index 811d775b79..400d6e8bbe 100644 --- a/zetaclient/chains/bitcoin/client/helpers.go +++ b/zetaclient/chains/bitcoin/client/helpers.go @@ -241,12 +241,10 @@ func (c *Client) Healthcheck(ctx context.Context) (time.Time, error) { func (c *Client) GetMempoolTxsAndFees( ctx context.Context, childHash string, - timeout time.Duration, ) (txsAndFees MempoolTxsAndFees, err error) { totalFeesFloat := float64(0) // loop through all parents - startTime := time.Now() parentHash := childHash for { memplEntry, err := c.GetMempoolEntry(ctx, parentHash) @@ -271,8 +269,10 @@ func (c *Client) GetMempoolTxsAndFees( parentHash = tx.MsgTx().TxIn[0].PreviousOutPoint.Hash.String() // check timeout to avoid infinite loop - if time.Since(startTime) > timeout { - return txsAndFees, errors.Errorf("timeout reached on %dth tx: %s", txsAndFees.TotalTxs, parentHash) + if deadline, ok := ctx.Deadline(); ok { + if time.Now().After(deadline) { + return txsAndFees, errors.Errorf("timeout reached on %dth tx: %s", txsAndFees.TotalTxs, parentHash) + } } } diff --git a/zetaclient/chains/bitcoin/client/mockgen.go b/zetaclient/chains/bitcoin/client/mockgen.go index cf07658838..3f6a59f096 100644 --- a/zetaclient/chains/bitcoin/client/mockgen.go +++ b/zetaclient/chains/bitcoin/client/mockgen.go @@ -32,7 +32,7 @@ type client interface { GetRawTransactionVerbose(ctx context.Context, hash *hash.Hash) (*types.TxRawResult, error) GetMempoolEntry(ctx context.Context, txHash string) (*types.GetMempoolEntryResult, error) GetRawMempool(ctx context.Context) ([]*hash.Hash, error) - GetMempoolTxsAndFees(ctx context.Context, childHash string, timeout time.Duration) (MempoolTxsAndFees, error) + GetMempoolTxsAndFees(ctx context.Context, childHash string) (MempoolTxsAndFees, error) GetRawTransactionResult( ctx context.Context, diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go index 86ec3519b8..9e5c3078e3 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -170,8 +170,12 @@ func (b *CPFPFeeBumper) fetchFeeBumpInfo() error { } b.LiveRate = liveRate + // create a new context with timeout + ctx, cancel := context.WithTimeout(b.Ctx, time.Minute) + defer cancel() + // query total fees and sizes of all pending parent TSS txs - txsAndFees, err := b.RPC.GetMempoolTxsAndFees(b.Ctx, b.Tx.MsgTx().TxID(), time.Minute) + txsAndFees, err := b.RPC.GetMempoolTxsAndFees(ctx, b.Tx.MsgTx().TxID()) if err != nil { return errors.Wrap(err, "unable to fetch mempool txs info") } diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index 5bf8d7bce6..0f3cc4b369 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -91,11 +91,9 @@ func Test_NewCPFPFeeBumper(t *testing.T) { // mock mempool txs information if tt.txsAndFees != nil { - tt.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return(*tt.txsAndFees, nil) + tt.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything).Maybe().Return(*tt.txsAndFees, nil) } else { - tt.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return(client.MempoolTxsAndFees{}, errors.New("rpc error")) + tt.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything).Maybe().Return(client.MempoolTxsAndFees{}, errors.New("rpc error")) } // ACT diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go index 3b590b7343..6fc0e5a05d 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf_test.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf_test.go @@ -178,11 +178,9 @@ func Test_SignRBFTx(t *testing.T) { // mock mempool txs information if tt.txsAndFees != nil { - s.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything). - Return(*tt.txsAndFees, nil) + s.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything).Return(*tt.txsAndFees, nil) } else { - s.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return(client.MempoolTxsAndFees{}, errors.New("rpc error")) + s.client.On("GetMempoolTxsAndFees", mock.Anything, mock.Anything).Maybe().Return(client.MempoolTxsAndFees{}, nil) } // mock RPC transactions diff --git a/zetaclient/chains/bitcoin/signer/signer.go b/zetaclient/chains/bitcoin/signer/signer.go index 97717ae6ec..4b26f0baf9 100644 --- a/zetaclient/chains/bitcoin/signer/signer.go +++ b/zetaclient/chains/bitcoin/signer/signer.go @@ -38,7 +38,7 @@ type RPC interface { GetRawTransaction(ctx context.Context, hash *chainhash.Hash) (*btcutil.Tx, error) GetEstimatedFeeRate(ctx context.Context, confTarget int64) (uint64, error) SendRawTransaction(ctx context.Context, tx *wire.MsgTx, allowHighFees bool) (*chainhash.Hash, error) - GetMempoolTxsAndFees(ctx context.Context, childHash string, timeout time.Duration) (client.MempoolTxsAndFees, error) + GetMempoolTxsAndFees(ctx context.Context, childHash string) (client.MempoolTxsAndFees, error) } // Signer deals with signing & broadcasting BTC transactions. diff --git a/zetaclient/testutils/mocks/bitcoin_client.go b/zetaclient/testutils/mocks/bitcoin_client.go index 2415c2e972..708f281c95 100644 --- a/zetaclient/testutils/mocks/bitcoin_client.go +++ b/zetaclient/testutils/mocks/bitcoin_client.go @@ -387,9 +387,9 @@ func (_m *BitcoinClient) GetMempoolEntry(ctx context.Context, txHash string) (*b return r0, r1 } -// GetMempoolTxsAndFees provides a mock function with given fields: ctx, childHash, timeout -func (_m *BitcoinClient) GetMempoolTxsAndFees(ctx context.Context, childHash string, timeout time.Duration) (client.MempoolTxsAndFees, error) { - ret := _m.Called(ctx, childHash, timeout) +// GetMempoolTxsAndFees provides a mock function with given fields: ctx, childHash +func (_m *BitcoinClient) GetMempoolTxsAndFees(ctx context.Context, childHash string) (client.MempoolTxsAndFees, error) { + ret := _m.Called(ctx, childHash) if len(ret) == 0 { panic("no return value specified for GetMempoolTxsAndFees") @@ -397,17 +397,17 @@ func (_m *BitcoinClient) GetMempoolTxsAndFees(ctx context.Context, childHash str var r0 client.MempoolTxsAndFees var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) (client.MempoolTxsAndFees, error)); ok { - return rf(ctx, childHash, timeout) + if rf, ok := ret.Get(0).(func(context.Context, string) (client.MempoolTxsAndFees, error)); ok { + return rf(ctx, childHash) } - if rf, ok := ret.Get(0).(func(context.Context, string, time.Duration) client.MempoolTxsAndFees); ok { - r0 = rf(ctx, childHash, timeout) + if rf, ok := ret.Get(0).(func(context.Context, string) client.MempoolTxsAndFees); ok { + r0 = rf(ctx, childHash) } else { r0 = ret.Get(0).(client.MempoolTxsAndFees) } - if rf, ok := ret.Get(1).(func(context.Context, string, time.Duration) error); ok { - r1 = rf(ctx, childHash, timeout) + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, childHash) } else { r1 = ret.Error(1) } From 3843eb8fc6bd9accbb7871844c40108cdd74219c Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 17:08:29 -0500 Subject: [PATCH 62/74] have BumpTxFee return a result struct BumpResult --- .../chains/bitcoin/signer/fee_bumper.go | 21 ++++-- .../chains/bitcoin/signer/fee_bumper_test.go | 73 ++++++++++--------- zetaclient/chains/bitcoin/signer/sign_rbf.go | 14 ++-- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go index 9e5c3078e3..66b10482b4 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -59,6 +59,13 @@ type CPFPFeeBumper struct { Logger zerolog.Logger } +// BumpResult contains the result of the fee bump +type BumpResult struct { + NewTx *wire.MsgTx + AdditionalFees int64 + NewFeeRate uint64 +} + // NewCPFPFeeBumper creates a new CPFPFeeBumper func NewCPFPFeeBumper( ctx context.Context, @@ -87,11 +94,11 @@ func NewCPFPFeeBumper( } // BumpTxFee bumps the fee of the stuck transaction using reserved bump fees -func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { +func (b *CPFPFeeBumper) BumpTxFee() (result BumpResult, err error) { // reuse old tx body newTx := CopyMsgTxNoWitness(b.Tx.MsgTx()) if len(newTx.TxOut) < 3 { - return nil, 0, 0, errors.New("original tx has no reserved bump fees") + return result, errors.New("original tx has no reserved bump fees") } // the new fee rate is supposed to be much higher than current paid rate (old rate). @@ -123,7 +130,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { txVSize := mempool.GetTxVirtualSize(b.Tx) minRelayFeeRate, err := common.FeeRateToSatPerByte(b.MinRelayFee) if err != nil { - return nil, 0, 0, errors.Wrapf(err, "unable to convert min relay fee rate") + return result, errors.Wrapf(err, "unable to convert min relay fee rate") } // #nosec G115 always in range minRelayTxFees := txVSize * int64(minRelayFeeRate) @@ -137,7 +144,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { // #nosec G115 always in range additionalFees := b.TxsAndFees.TotalVSize*int64(feeRateNew) - b.TxsAndFees.TotalFees if additionalFees < minRelayTxFees { - return nil, 0, 0, fmt.Errorf( + return result, fmt.Errorf( "hold on RBF: additional fees %d is lower than min relay fees %d", additionalFees, minRelayTxFees, @@ -158,7 +165,11 @@ func (b *CPFPFeeBumper) BumpTxFee() (*wire.MsgTx, int64, uint64, error) { // #nosec G115 always positive feeRateNew = uint64(math.Ceil(float64(b.TxsAndFees.TotalFees+additionalFees) / float64(b.TxsAndFees.TotalVSize))) - return newTx, additionalFees, feeRateNew, nil + return BumpResult{ + NewTx: newTx, + AdditionalFees: additionalFees, + NewFeeRate: feeRateNew, + }, nil } // fetchFeeBumpInfo fetches all necessary information needed to bump the stuck tx diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index 0f3cc4b369..4a2fbbc1ba 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -127,12 +127,10 @@ func Test_BumpTxFee(t *testing.T) { msgTx := testutils.LoadBTCMsgTx(t, TestDataDir, chain.ChainId, txid) tests := []struct { - name string - feeBumper *CPFPFeeBumper - additionalFees int64 - expectedNewRate uint64 - expectedNewTx *wire.MsgTx - errMsg string + name string + feeBumper *CPFPFeeBumper + expected BumpResult + errMsg string }{ { name: "should bump tx fee successfully", @@ -148,14 +146,16 @@ func Test_BumpTxFee(t *testing.T) { }, Logger: log.Logger, }, - additionalFees: 4632, // 579*55 - 27213 - expectedNewRate: 55, - expectedNewTx: func() *wire.MsgTx { - // deduct additional fees - newTx := CopyMsgTxNoWitness(msgTx) - newTx.TxOut[2].Value -= 4632 - return newTx - }(), + expected: BumpResult{ + NewTx: func() *wire.MsgTx { + // deduct additional fees + newTx := CopyMsgTxNoWitness(msgTx) + newTx.TxOut[2].Value -= 4632 + return newTx + }(), + AdditionalFees: 4632, // 579*55 - 27213 + NewFeeRate: 55, + }, }, { name: "should give up all reserved bump fees", @@ -176,14 +176,16 @@ func Test_BumpTxFee(t *testing.T) { }, Logger: log.Logger, }, - additionalFees: 6789, // same as the reserved value in 2nd output - expectedNewRate: 59, // (27213 + 6789) / 579 ≈ 59 - expectedNewTx: func() *wire.MsgTx { - // give up all reserved bump fees - newTx := CopyMsgTxNoWitness(msgTx) - newTx.TxOut = newTx.TxOut[:2] - return newTx - }(), + expected: BumpResult{ + NewTx: func() *wire.MsgTx { + // give up all reserved bump fees + newTx := CopyMsgTxNoWitness(msgTx) + newTx.TxOut = newTx.TxOut[:2] + return newTx + }(), + AdditionalFees: 6789, // same as the reserved value in 2nd output + NewFeeRate: 59, // (27213 + 6789) / 579 ≈ 59 + }, }, { name: "should set new gas rate to 'gasRateCap'", @@ -199,14 +201,16 @@ func Test_BumpTxFee(t *testing.T) { }, Logger: log.Logger, }, - additionalFees: 30687, // (100-47)*579 - expectedNewRate: 100, - expectedNewTx: func() *wire.MsgTx { - // deduct additional fees - newTx := CopyMsgTxNoWitness(msgTx) - newTx.TxOut[2].Value -= 30687 - return newTx - }(), + expected: BumpResult{ + NewTx: func() *wire.MsgTx { + // deduct additional fees + newTx := CopyMsgTxNoWitness(msgTx) + newTx.TxOut[2].Value -= 30687 + return newTx + }(), + AdditionalFees: 30687, // (100-47)*579 + NewFeeRate: 100, + }, }, { name: "should fail if original tx has no reserved bump fees", @@ -240,16 +244,13 @@ func Test_BumpTxFee(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - newTx, additionalFees, newRate, err := tt.feeBumper.BumpTxFee() + result, err := tt.feeBumper.BumpTxFee() if tt.errMsg != "" { - require.Nil(t, newTx) - require.Zero(t, additionalFees) + require.Nil(t, result.NewTx) require.ErrorContains(t, err, tt.errMsg) } else { require.NoError(t, err) - require.Equal(t, tt.expectedNewTx, newTx) - require.Equal(t, tt.additionalFees, additionalFees) - require.Equal(t, tt.expectedNewRate, newRate) + require.Equal(t, tt.expected, result) } }) } diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go index 4004b18030..a735bd1d96 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -52,19 +52,19 @@ func (signer *Signer) SignRBFTx(ctx context.Context, txData *OutboundData, lastT } // bump tx fees - newTx, additionalFees, newRate, err := fb.BumpTxFee() + result, err := fb.BumpTxFee() if err != nil { return nil, errors.Wrap(err, "BumpTxFee failed") } logger.Info(). Uint64("old_fee_rate", fb.TxsAndFees.AvgFeeRate). - Uint64("new_fee_rate", newRate). - Int64("additional_fees", additionalFees). + Uint64("new_fee_rate", result.NewFeeRate). + Int64("additional_fees", result.AdditionalFees). Msg("BumpTxFee succeed") // collect input amounts for signing - inAmounts := make([]int64, len(newTx.TxIn)) - for i, input := range newTx.TxIn { + inAmounts := make([]int64, len(result.NewTx.TxIn)) + for i, input := range result.NewTx.TxIn { preOut := input.PreviousOutPoint preTx, err := signer.rpc.GetRawTransaction(ctx, &preOut.Hash) if err != nil { @@ -74,10 +74,10 @@ func (signer *Signer) SignRBFTx(ctx context.Context, txData *OutboundData, lastT } // sign the RBF tx - err = signer.SignTx(ctx, newTx, inAmounts, txData.height, txData.nonce) + err = signer.SignTx(ctx, result.NewTx, inAmounts, txData.height, txData.nonce) if err != nil { return nil, errors.Wrap(err, "SignTx failed") } - return newTx, nil + return result.NewTx, nil } From 1b77a0df5987bdb42cb44d2b11f7d725802680ad Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 17:12:23 -0500 Subject: [PATCH 63/74] unexport fee bumper fields --- .../chains/bitcoin/signer/fee_bumper.go | 96 +++++++++---------- .../chains/bitcoin/signer/fee_bumper_test.go | 68 ++++++------- zetaclient/chains/bitcoin/signer/sign_rbf.go | 2 +- 3 files changed, 83 insertions(+), 83 deletions(-) diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper.go b/zetaclient/chains/bitcoin/signer/fee_bumper.go index 66b10482b4..d8304574f4 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper.go @@ -34,29 +34,29 @@ const ( // CPFPFeeBumper is a helper struct to contain CPFP (child-pays-for-parent) fee bumping logic type CPFPFeeBumper struct { - Ctx context.Context + ctx context.Context - Chain chains.Chain + chain chains.Chain - // RPC is the interface to interact with the Bitcoin chain - RPC RPC + // rpc is the interface to interact with the Bitcoin chain + rpc RPC - // Tx is the stuck transaction to bump - Tx *btcutil.Tx + // tx is the stuck transaction to bump + tx *btcutil.Tx - // MinRelayFee is the minimum relay fee in BTC - MinRelayFee float64 + // minRelayFee is the minimum relay fee in BTC + minRelayFee float64 - // CCTXRate is the most recent fee rate of the CCTX - CCTXRate uint64 + // cctxRate is the most recent fee rate of the CCTX + cctxRate uint64 - // LiveRate is the most recent market fee rate - LiveRate uint64 + // liveRate is the most recent market fee rate + liveRate uint64 - // TxsAndFees contains the information of all pending txs and fees - TxsAndFees client.MempoolTxsAndFees + // txsAndFees contains the information of all pending txs and fees + txsAndFees client.MempoolTxsAndFees - Logger zerolog.Logger + logger zerolog.Logger } // BumpResult contains the result of the fee bump @@ -77,13 +77,13 @@ func NewCPFPFeeBumper( logger zerolog.Logger, ) (*CPFPFeeBumper, error) { fb := &CPFPFeeBumper{ - Ctx: ctx, - Chain: chain, - RPC: rpc, - Tx: tx, - MinRelayFee: minRelayFee, - CCTXRate: cctxRate, - Logger: logger, + ctx: ctx, + chain: chain, + rpc: rpc, + tx: tx, + minRelayFee: minRelayFee, + cctxRate: cctxRate, + logger: logger, } err := fb.fetchFeeBumpInfo() @@ -96,7 +96,7 @@ func NewCPFPFeeBumper( // BumpTxFee bumps the fee of the stuck transaction using reserved bump fees func (b *CPFPFeeBumper) BumpTxFee() (result BumpResult, err error) { // reuse old tx body - newTx := CopyMsgTxNoWitness(b.Tx.MsgTx()) + newTx := CopyMsgTxNoWitness(b.tx.MsgTx()) if len(newTx.TxOut) < 3 { return result, errors.New("original tx has no reserved bump fees") } @@ -104,31 +104,31 @@ func (b *CPFPFeeBumper) BumpTxFee() (result BumpResult, err error) { // the new fee rate is supposed to be much higher than current paid rate (old rate). // we print a warning message if it's not the case for monitoring purposes. // #nosec G115 always positive - oldRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.TxsAndFees.AvgFeeRate), decentFeeBumpPercent) - if sdkmath.NewUint(b.CCTXRate).LT(oldRateBumped) { - b.Logger.Warn(). - Uint64("old_fee_rate", b.TxsAndFees.AvgFeeRate). - Uint64("new_fee_rate", b.CCTXRate). + oldRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.txsAndFees.AvgFeeRate), decentFeeBumpPercent) + if sdkmath.NewUint(b.cctxRate).LT(oldRateBumped) { + b.logger.Warn(). + Uint64("old_fee_rate", b.txsAndFees.AvgFeeRate). + Uint64("new_fee_rate", b.cctxRate). Msg("new fee rate is not much higher than the old fee rate") } // the live rate may continue increasing during network congestion, and the new fee rate is still not high enough. // but we should still continue with the tx replacement because zetacore had already bumped the fee rate. - newRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.CCTXRate), decentFeeBumpPercent) - if sdkmath.NewUint(b.LiveRate).GT(newRateBumped) { - b.Logger.Warn(). - Uint64("new_fee_rate", b.CCTXRate). - Uint64("live_fee_rate", b.LiveRate). + newRateBumped, _ := mathpkg.IncreaseUintByPercent(sdkmath.NewUint(b.cctxRate), decentFeeBumpPercent) + if sdkmath.NewUint(b.liveRate).GT(newRateBumped) { + b.logger.Warn(). + Uint64("new_fee_rate", b.cctxRate). + Uint64("live_fee_rate", b.liveRate). Msg("live fee rate is still much higher than the new fee rate") } // cap the fee rate to avoid excessive fees - feeRateNew := min(b.CCTXRate, feeRateCap) + feeRateNew := min(b.cctxRate, feeRateCap) // calculate minmimum relay fees of the new replacement tx // the new tx will have almost same size as the old one because the tx body stays the same - txVSize := mempool.GetTxVirtualSize(b.Tx) - minRelayFeeRate, err := common.FeeRateToSatPerByte(b.MinRelayFee) + txVSize := mempool.GetTxVirtualSize(b.tx) + minRelayFeeRate, err := common.FeeRateToSatPerByte(b.minRelayFee) if err != nil { return result, errors.Wrapf(err, "unable to convert min relay fee rate") } @@ -142,7 +142,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (result BumpResult, err error) { // // see: https://github.com/bitcoin/bitcoin/blob/5b8046a6e893b7fad5a93631e6d1e70db31878af/src/policy/rbf.cpp#L166-L183 // #nosec G115 always in range - additionalFees := b.TxsAndFees.TotalVSize*int64(feeRateNew) - b.TxsAndFees.TotalFees + additionalFees := b.txsAndFees.TotalVSize*int64(feeRateNew) - b.txsAndFees.TotalFees if additionalFees < minRelayTxFees { return result, fmt.Errorf( "hold on RBF: additional fees %d is lower than min relay fees %d", @@ -163,7 +163,7 @@ func (b *CPFPFeeBumper) BumpTxFee() (result BumpResult, err error) { // effective fee rate // #nosec G115 always positive - feeRateNew = uint64(math.Ceil(float64(b.TxsAndFees.TotalFees+additionalFees) / float64(b.TxsAndFees.TotalVSize))) + feeRateNew = uint64(math.Ceil(float64(b.txsAndFees.TotalFees+additionalFees) / float64(b.txsAndFees.TotalVSize))) return BumpResult{ NewTx: newTx, @@ -175,28 +175,28 @@ func (b *CPFPFeeBumper) BumpTxFee() (result BumpResult, err error) { // fetchFeeBumpInfo fetches all necessary information needed to bump the stuck tx func (b *CPFPFeeBumper) fetchFeeBumpInfo() error { // query live fee rate - liveRate, err := b.RPC.GetEstimatedFeeRate(b.Ctx, 1) + liveRate, err := b.rpc.GetEstimatedFeeRate(b.ctx, 1) if err != nil { return errors.Wrap(err, "GetEstimatedFeeRate failed") } - b.LiveRate = liveRate + b.liveRate = liveRate // create a new context with timeout - ctx, cancel := context.WithTimeout(b.Ctx, time.Minute) + ctx, cancel := context.WithTimeout(b.ctx, time.Minute) defer cancel() // query total fees and sizes of all pending parent TSS txs - txsAndFees, err := b.RPC.GetMempoolTxsAndFees(ctx, b.Tx.MsgTx().TxID()) + txsAndFees, err := b.rpc.GetMempoolTxsAndFees(ctx, b.tx.MsgTx().TxID()) if err != nil { return errors.Wrap(err, "unable to fetch mempool txs info") } - b.TxsAndFees = txsAndFees + b.txsAndFees = txsAndFees - b.Logger.Info(). - Int64("total_txs", b.TxsAndFees.TotalTxs). - Int64("total_fees", b.TxsAndFees.TotalFees). - Int64("total_vsize", b.TxsAndFees.TotalVSize). - Uint64("avg_fee_rate", b.TxsAndFees.AvgFeeRate). + b.logger.Info(). + Int64("total_txs", b.txsAndFees.TotalTxs). + Int64("total_fees", b.txsAndFees.TotalFees). + Int64("total_vsize", b.txsAndFees.TotalVSize). + Uint64("avg_fee_rate", b.txsAndFees.AvgFeeRate). Msg("fetched fee bump information") return nil diff --git a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go index 4a2fbbc1ba..e9220b5a5a 100644 --- a/zetaclient/chains/bitcoin/signer/fee_bumper_test.go +++ b/zetaclient/chains/bitcoin/signer/fee_bumper_test.go @@ -45,19 +45,19 @@ func Test_NewCPFPFeeBumper(t *testing.T) { AvgFeeRate: 10, // average fee rate 10 sat/vB }, expected: &CPFPFeeBumper{ - Ctx: context.Background(), - Chain: chains.BitcoinMainnet, - Tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), - MinRelayFee: 0.00001, - CCTXRate: 10, - LiveRate: 12, - TxsAndFees: client.MempoolTxsAndFees{ + ctx: context.Background(), + chain: chains.BitcoinMainnet, + tx: btcutil.NewTx(wire.NewMsgTx(wire.TxVersion)), + minRelayFee: 0.00001, + cctxRate: 10, + liveRate: 12, + txsAndFees: client.MempoolTxsAndFees{ TotalTxs: 2, TotalFees: 10000, TotalVSize: 1000, AvgFeeRate: 10, }, - Logger: log.Logger, + logger: log.Logger, }, }, { @@ -112,7 +112,7 @@ func Test_NewCPFPFeeBumper(t *testing.T) { require.Nil(t, bumper) require.ErrorContains(t, err, tt.errMsg) } else { - bumper.RPC = nil // ignore the RPC + bumper.rpc = nil // ignore the RPC require.NoError(t, err) require.Equal(t, tt.expected, bumper) } @@ -135,16 +135,16 @@ func Test_BumpTxFee(t *testing.T) { { name: "should bump tx fee successfully", feeBumper: &CPFPFeeBumper{ - Tx: btcutil.NewTx(msgTx), - MinRelayFee: 0.00001, - CCTXRate: 55, - LiveRate: 67, - TxsAndFees: client.MempoolTxsAndFees{ + tx: btcutil.NewTx(msgTx), + minRelayFee: 0.00001, + cctxRate: 55, + liveRate: 67, + txsAndFees: client.MempoolTxsAndFees{ TotalFees: 27213, TotalVSize: 579, AvgFeeRate: 47, }, - Logger: log.Logger, + logger: log.Logger, }, expected: BumpResult{ NewTx: func() *wire.MsgTx { @@ -160,21 +160,21 @@ func Test_BumpTxFee(t *testing.T) { { name: "should give up all reserved bump fees", feeBumper: &CPFPFeeBumper{ - Tx: func() *btcutil.Tx { + tx: func() *btcutil.Tx { // modify reserved bump fees to barely cover bump fees newTx := msgTx.Copy() newTx.TxOut[2].Value = 57*579 - 27213 + constant.BTCWithdrawalDustAmount - 1 // 6789 return btcutil.NewTx(newTx) }(), - MinRelayFee: 0.00001, - CCTXRate: 57, - LiveRate: 67, - TxsAndFees: client.MempoolTxsAndFees{ + minRelayFee: 0.00001, + cctxRate: 57, + liveRate: 67, + txsAndFees: client.MempoolTxsAndFees{ TotalFees: 27213, TotalVSize: 579, AvgFeeRate: 47, }, - Logger: log.Logger, + logger: log.Logger, }, expected: BumpResult{ NewTx: func() *wire.MsgTx { @@ -190,16 +190,16 @@ func Test_BumpTxFee(t *testing.T) { { name: "should set new gas rate to 'gasRateCap'", feeBumper: &CPFPFeeBumper{ - Tx: btcutil.NewTx(msgTx), - MinRelayFee: 0.00001, - CCTXRate: 101, // > 100 - LiveRate: 120, - TxsAndFees: client.MempoolTxsAndFees{ + tx: btcutil.NewTx(msgTx), + minRelayFee: 0.00001, + cctxRate: 101, // > 100 + liveRate: 120, + txsAndFees: client.MempoolTxsAndFees{ TotalFees: 27213, TotalVSize: 579, AvgFeeRate: 47, }, - Logger: log.Logger, + logger: log.Logger, }, expected: BumpResult{ NewTx: func() *wire.MsgTx { @@ -215,7 +215,7 @@ func Test_BumpTxFee(t *testing.T) { { name: "should fail if original tx has no reserved bump fees", feeBumper: &CPFPFeeBumper{ - Tx: func() *btcutil.Tx { + tx: func() *btcutil.Tx { // remove the change output newTx := msgTx.Copy() newTx.TxOut = newTx.TxOut[:2] @@ -227,16 +227,16 @@ func Test_BumpTxFee(t *testing.T) { { name: "should hold on RBF if additional fees is lower than min relay fees", feeBumper: &CPFPFeeBumper{ - Tx: btcutil.NewTx(msgTx), - MinRelayFee: 0.00002, // min relay fee will be 579vB * 2 = 1158 sats - CCTXRate: 6, - LiveRate: 7, - TxsAndFees: client.MempoolTxsAndFees{ + tx: btcutil.NewTx(msgTx), + minRelayFee: 0.00002, // min relay fee will be 579vB * 2 = 1158 sats + cctxRate: 6, + liveRate: 7, + txsAndFees: client.MempoolTxsAndFees{ TotalFees: 2895, TotalVSize: 579, AvgFeeRate: 5, }, - Logger: log.Logger, + logger: log.Logger, }, errMsg: "lower than min relay fees", }, diff --git a/zetaclient/chains/bitcoin/signer/sign_rbf.go b/zetaclient/chains/bitcoin/signer/sign_rbf.go index a735bd1d96..1b98cf3fcd 100644 --- a/zetaclient/chains/bitcoin/signer/sign_rbf.go +++ b/zetaclient/chains/bitcoin/signer/sign_rbf.go @@ -57,7 +57,7 @@ func (signer *Signer) SignRBFTx(ctx context.Context, txData *OutboundData, lastT return nil, errors.Wrap(err, "BumpTxFee failed") } logger.Info(). - Uint64("old_fee_rate", fb.TxsAndFees.AvgFeeRate). + Uint64("old_fee_rate", fb.txsAndFees.AvgFeeRate). Uint64("new_fee_rate", result.NewFeeRate). Int64("additional_fees", result.AdditionalFees). Msg("BumpTxFee succeed") From b6a4426263e2d00469fba23ad7ea65a16a4d7e2a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 17:16:52 -0500 Subject: [PATCH 64/74] remove Live prefix from live tests as we already have the env flag to disable these tests --- .../chains/bitcoin/client/client_rbf_test.go | 12 +++++------ zetaclient/chains/solana/rpc/rpc_live_test.go | 20 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/client_rbf_test.go b/zetaclient/chains/bitcoin/client/client_rbf_test.go index 30f8bf9e93..871b87f732 100644 --- a/zetaclient/chains/bitcoin/client/client_rbf_test.go +++ b/zetaclient/chains/bitcoin/client/client_rbf_test.go @@ -32,9 +32,9 @@ func Test_BitcoinRBFLive(t *testing.T) { t.Skip("skipping live test") } - LiveTest_RBFTransaction(t) - LiveTest_RBFTransaction_Chained_CPFP(t) - LiveTest_PendingMempoolTx(t) + Test_RBFTransaction(t) + Test_RBFTransaction_Chained_CPFP(t) + Test_PendingMempoolTx(t) } // setupRBFTest initializes the test suite, privateKey, sender, receiver @@ -71,7 +71,7 @@ func setupRBFTest(t *testing.T) (*testSuite, *secp256k1.PrivateKey, btcutil.Addr return ts, privKey, sender, to } -func LiveTest_RBFTransaction(t *testing.T) { +func Test_RBFTransaction(t *testing.T) { // setup test ts, privKey, sender, to := setupRBFTest(t) @@ -163,7 +163,7 @@ func LiveTest_RBFTransaction(t *testing.T) { } // Test_RBFTransactionChained_CPFP tests Child-Pays-For-Parent (CPFP) fee bumping strategy for chained RBF transactions -func LiveTest_RBFTransaction_Chained_CPFP(t *testing.T) { +func Test_RBFTransaction_Chained_CPFP(t *testing.T) { // setup test ts, privKey, sender, to := setupRBFTest(t) @@ -269,7 +269,7 @@ func LiveTest_RBFTransaction_Chained_CPFP(t *testing.T) { fmt.Println("tx1 dropped") } -func LiveTest_PendingMempoolTx(t *testing.T) { +func Test_PendingMempoolTx(t *testing.T) { // network to use config := config.BTCConfig{ RPCHost: os.Getenv(common.EnvBtcRPCMainnet), diff --git a/zetaclient/chains/solana/rpc/rpc_live_test.go b/zetaclient/chains/solana/rpc/rpc_live_test.go index 138a184758..5e07505e69 100644 --- a/zetaclient/chains/solana/rpc/rpc_live_test.go +++ b/zetaclient/chains/solana/rpc/rpc_live_test.go @@ -19,14 +19,14 @@ func Test_SolanaRPCLive(t *testing.T) { return } - LiveTest_GetTransactionWithVersion(t) - LiveTest_GetFirstSignatureForAddress(t) - LiveTest_GetSignaturesForAddressUntil(t) - LiveTest_GetSignaturesForAddressUntil_Version0(t) - LiveTest_HealthCheck(t) + Test_GetTransactionWithVersion(t) + Test_GetFirstSignatureForAddress(t) + Test_GetSignaturesForAddressUntil(t) + Test_GetSignaturesForAddressUntil_Version0(t) + Test_HealthCheck(t) } -func LiveTest_GetTransactionWithVersion(t *testing.T) { +func Test_GetTransactionWithVersion(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -44,7 +44,7 @@ func LiveTest_GetTransactionWithVersion(t *testing.T) { }) } -func LiveTest_GetFirstSignatureForAddress(t *testing.T) { +func Test_GetFirstSignatureForAddress(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -60,7 +60,7 @@ func LiveTest_GetFirstSignatureForAddress(t *testing.T) { require.Equal(t, actualSig, sig.String()) } -func LiveTest_GetSignaturesForAddressUntil(t *testing.T) { +func Test_GetSignaturesForAddressUntil(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -83,7 +83,7 @@ func LiveTest_GetSignaturesForAddressUntil(t *testing.T) { } } -func LiveTest_GetSignaturesForAddressUntil_Version0(t *testing.T) { +func Test_GetSignaturesForAddressUntil_Version0(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -99,7 +99,7 @@ func LiveTest_GetSignaturesForAddressUntil_Version0(t *testing.T) { require.NoError(t, err) } -func LiveTest_HealthCheck(t *testing.T) { +func Test_HealthCheck(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) From bbea857fb28597fccc7e447a5bb83102e9f6b1e7 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 2 May 2025 17:27:19 -0500 Subject: [PATCH 65/74] wrap live tests function into test runners so IDE can identify the tests --- .../chains/bitcoin/client/client_rbf_test.go | 20 +++++++---- zetaclient/chains/solana/rpc/rpc_live_test.go | 34 +++++++++++++------ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/zetaclient/chains/bitcoin/client/client_rbf_test.go b/zetaclient/chains/bitcoin/client/client_rbf_test.go index 871b87f732..af9ef5600d 100644 --- a/zetaclient/chains/bitcoin/client/client_rbf_test.go +++ b/zetaclient/chains/bitcoin/client/client_rbf_test.go @@ -32,9 +32,17 @@ func Test_BitcoinRBFLive(t *testing.T) { t.Skip("skipping live test") } - Test_RBFTransaction(t) - Test_RBFTransaction_Chained_CPFP(t) - Test_PendingMempoolTx(t) + t.Run("RBFTransaction", func(t *testing.T) { + Run_RBFTransaction(t) + }) + + t.Run("RBFTransaction_Chained_CPFP", func(t *testing.T) { + Run_RBFTransaction_Chained_CPFP(t) + }) + + t.Run("PendingMempoolTx", func(t *testing.T) { + Run_PendingMempoolTx(t) + }) } // setupRBFTest initializes the test suite, privateKey, sender, receiver @@ -71,7 +79,7 @@ func setupRBFTest(t *testing.T) (*testSuite, *secp256k1.PrivateKey, btcutil.Addr return ts, privKey, sender, to } -func Test_RBFTransaction(t *testing.T) { +func Run_RBFTransaction(t *testing.T) { // setup test ts, privKey, sender, to := setupRBFTest(t) @@ -163,7 +171,7 @@ func Test_RBFTransaction(t *testing.T) { } // Test_RBFTransactionChained_CPFP tests Child-Pays-For-Parent (CPFP) fee bumping strategy for chained RBF transactions -func Test_RBFTransaction_Chained_CPFP(t *testing.T) { +func Run_RBFTransaction_Chained_CPFP(t *testing.T) { // setup test ts, privKey, sender, to := setupRBFTest(t) @@ -269,7 +277,7 @@ func Test_RBFTransaction_Chained_CPFP(t *testing.T) { fmt.Println("tx1 dropped") } -func Test_PendingMempoolTx(t *testing.T) { +func Run_PendingMempoolTx(t *testing.T) { // network to use config := config.BTCConfig{ RPCHost: os.Getenv(common.EnvBtcRPCMainnet), diff --git a/zetaclient/chains/solana/rpc/rpc_live_test.go b/zetaclient/chains/solana/rpc/rpc_live_test.go index 5e07505e69..9a193bab48 100644 --- a/zetaclient/chains/solana/rpc/rpc_live_test.go +++ b/zetaclient/chains/solana/rpc/rpc_live_test.go @@ -19,14 +19,28 @@ func Test_SolanaRPCLive(t *testing.T) { return } - Test_GetTransactionWithVersion(t) - Test_GetFirstSignatureForAddress(t) - Test_GetSignaturesForAddressUntil(t) - Test_GetSignaturesForAddressUntil_Version0(t) - Test_HealthCheck(t) + t.Run("GetTransactionWithVersion", func(t *testing.T) { + Run_GetTransactionWithVersion(t) + }) + + t.Run("GetFirstSignatureForAddress", func(t *testing.T) { + Run_GetFirstSignatureForAddress(t) + }) + + t.Run("GetSignaturesForAddressUntil", func(t *testing.T) { + Run_GetSignaturesForAddressUntil(t) + }) + + t.Run("GetSignaturesForAddressUntil_Version0", func(t *testing.T) { + Run_GetSignaturesForAddressUntil_Version0(t) + }) + + t.Run("HealthCheck", func(t *testing.T) { + Run_HealthCheck(t) + }) } -func Test_GetTransactionWithVersion(t *testing.T) { +func Run_GetTransactionWithVersion(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -44,7 +58,7 @@ func Test_GetTransactionWithVersion(t *testing.T) { }) } -func Test_GetFirstSignatureForAddress(t *testing.T) { +func Run_GetFirstSignatureForAddress(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -60,7 +74,7 @@ func Test_GetFirstSignatureForAddress(t *testing.T) { require.Equal(t, actualSig, sig.String()) } -func Test_GetSignaturesForAddressUntil(t *testing.T) { +func Run_GetSignaturesForAddressUntil(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -83,7 +97,7 @@ func Test_GetSignaturesForAddressUntil(t *testing.T) { } } -func Test_GetSignaturesForAddressUntil_Version0(t *testing.T) { +func Run_GetSignaturesForAddressUntil_Version0(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) @@ -99,7 +113,7 @@ func Test_GetSignaturesForAddressUntil_Version0(t *testing.T) { require.NoError(t, err) } -func Test_HealthCheck(t *testing.T) { +func Run_HealthCheck(t *testing.T) { // create a Solana devnet RPC client client := solanarpc.New(solanarpc.DevNet_RPC) From a1b77610b19b0752d556210c8027c0096bcb7042 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Sun, 4 May 2025 23:24:48 -0500 Subject: [PATCH 66/74] add comment to the disabled Bitcoin RBF E2E test for clarity --- cmd/zetae2e/local/bitcoin.go | 4 +++- e2e/e2etests/test_bitcoin_withdraw_rbf.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/zetae2e/local/bitcoin.go b/cmd/zetae2e/local/bitcoin.go index 875211acd5..0664c4c0c4 100644 --- a/cmd/zetae2e/local/bitcoin.go +++ b/cmd/zetae2e/local/bitcoin.go @@ -59,6 +59,8 @@ func startBitcoinTests( e2etests.TestBitcoinWithdrawP2WSHName, e2etests.TestBitcoinWithdrawMultipleName, e2etests.TestBitcoinWithdrawRestrictedName, + // to run RBF test, change the constant 'minTxConfirmations' to 1 in the Bitcoin observer + // https://github.com/zeta-chain/node/blob/5c2a8ffbc702130fd9538b1cd7640d0e04d3e4f6/zetaclient/chains/bitcoin/observer/outbound.go#L27 //e2etests.TestBitcoinWithdrawRBFName, } @@ -204,7 +206,7 @@ func createBitcoinTestRoutine( for _, test := range testsToRun { // RBF test needs to wait for all deposit tests to complete if test.Name == e2etests.TestBitcoinWithdrawRBFName && wgDependency != nil { - r.Logger.Print("⏳waiting - %s", test.Description) + r.Logger.Print("⏳waiting for test %s to complete", test.Name) wgDependency.Wait() } if err := r.RunE2ETest(test, true); err != nil { diff --git a/e2e/e2etests/test_bitcoin_withdraw_rbf.go b/e2e/e2etests/test_bitcoin_withdraw_rbf.go index 57c764ddbc..7be1a3fb9e 100644 --- a/e2e/e2etests/test_bitcoin_withdraw_rbf.go +++ b/e2e/e2etests/test_bitcoin_withdraw_rbf.go @@ -17,7 +17,7 @@ import ( // // IMPORTANT: the test requires to simulate a stuck tx in the Bitcoin regnet. // Changing the 'minTxConfirmations' to 1 to not include Bitcoin pending txs. -// https://github.com/zeta-chain/node/blob/feat-bitcoin-Replace-By-Fee/zetaclient/chains/bitcoin/observer/outbound.go#L30 +// https://github.com/zeta-chain/node/blob/5c2a8ffbc702130fd9538b1cd7640d0e04d3e4f6/zetaclient/chains/bitcoin/observer/outbound.go#L27 func TestBitcoinWithdrawRBF(r *runner.E2ERunner, args []string) { require.Len(r, args, 2) From 1441ee2244346371e49e590e33185a17764edf6c Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 7 May 2025 17:51:14 -0500 Subject: [PATCH 67/74] disable Bitcoin rbf test --- cmd/zetae2e/local/bitcoin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zetae2e/local/bitcoin.go b/cmd/zetae2e/local/bitcoin.go index 5e71bf824d..9b79ec4bb7 100644 --- a/cmd/zetae2e/local/bitcoin.go +++ b/cmd/zetae2e/local/bitcoin.go @@ -61,7 +61,7 @@ func startBitcoinTests( e2etests.TestBitcoinWithdrawRestrictedName, // to run RBF test, change the constant 'minTxConfirmations' to 1 in the Bitcoin observer // https://github.com/zeta-chain/node/blob/5c2a8ffbc702130fd9538b1cd7640d0e04d3e4f6/zetaclient/chains/bitcoin/observer/outbound.go#L27 - e2etests.TestBitcoinWithdrawRBFName, + //e2etests.TestBitcoinWithdrawRBFName, } if !light { From fc88b12f64b52ad4969027c27a5011b7cc81f2e9 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 7 May 2025 17:52:33 -0500 Subject: [PATCH 68/74] keep the new line for better indentation --- zetaclient/chains/bitcoin/observer/mempool.go | 1 + 1 file changed, 1 insertion(+) diff --git a/zetaclient/chains/bitcoin/observer/mempool.go b/zetaclient/chains/bitcoin/observer/mempool.go index 3271873381..750e1e080b 100644 --- a/zetaclient/chains/bitcoin/observer/mempool.go +++ b/zetaclient/chains/bitcoin/observer/mempool.go @@ -128,6 +128,7 @@ func (ob *Observer) getLastPendingOutbound(ctx context.Context) (tx *btcutil.Tx, if err != nil { return nil, 0, errors.Wrap(err, "GetPendingNoncesByChain failed") } + // #nosec G115 always in range for nonce := uint64(p.NonceLow); nonce < uint64(p.NonceHigh); nonce++ { if nonce > lastNonce { From c5497703ea1234c0ceab7f6540a6519e339bb567 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 7 May 2025 18:03:32 -0500 Subject: [PATCH 69/74] sync changelog file --- changelog.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index e738f9c65f..a176bae4d2 100644 --- a/changelog.md +++ b/changelog.md @@ -23,13 +23,9 @@ * [3807](https://github.com/zeta-chain/node/pull/3807) - integrate ZEVM to Solana call * [3826](https://github.com/zeta-chain/node/pull/3826) - add global tss signature rate-limiter to zetaclient * [3793](https://github.com/zeta-chain/node/pull/3793) - support Sui withdrawAndCall using the PTB transaction -* [3396](https://github.com/zeta-chain/node/pull/3396) - add support for Bitcoin RBF (Replace-By-Fee) in zetaclient - -### Tests - * [3839](https://github.com/zeta-chain/node/pull/3839) - parse Solana inbounds from inner instructions * [3837](https://github.com/zeta-chain/node/pull/3837) - cancel Sui withdrawAndCall if tx cannot go through, e.g. on_call fails due to invalid data -* [3417](https://github.com/zeta-chain/node/pull/3417) - add e2e test for the Bitcoin RBF (Replace-By-Fee) feature +* [3396](https://github.com/zeta-chain/node/pull/3396) - add support for Bitcoin RBF (Replace-By-Fee) in zetaclient ### Refactor @@ -52,6 +48,7 @@ * [3692](https://github.com/zeta-chain/node/pull/3692) - e2e staking test for `MsgUndelegate` tx, to test observer staking hooks * [3831](https://github.com/zeta-chain/node/pull/3831) - e2e tests for sui fungible token withdraw and call +* [3417](https://github.com/zeta-chain/node/pull/3417) - add e2e test for the Bitcoin RBF (Replace-By-Fee) feature ### Refactor From 63eea64b7cbe0fda36787e3082516425c8cad68c Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 8 May 2025 10:28:22 -0500 Subject: [PATCH 70/74] add explanation why RBF test is disabled by default --- e2e/e2etests/test_bitcoin_withdraw_rbf.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/e2etests/test_bitcoin_withdraw_rbf.go b/e2e/e2etests/test_bitcoin_withdraw_rbf.go index 7be1a3fb9e..48856bd39d 100644 --- a/e2e/e2etests/test_bitcoin_withdraw_rbf.go +++ b/e2e/e2etests/test_bitcoin_withdraw_rbf.go @@ -16,8 +16,12 @@ import ( // It needs block mining to be stopped and runs as the last test in the suite. // // IMPORTANT: the test requires to simulate a stuck tx in the Bitcoin regnet. -// Changing the 'minTxConfirmations' to 1 to not include Bitcoin pending txs. -// https://github.com/zeta-chain/node/blob/5c2a8ffbc702130fd9538b1cd7640d0e04d3e4f6/zetaclient/chains/bitcoin/observer/outbound.go#L27 +// The challenge to simulate a stuck tx is to create overwhelming traffic in the local mempool. +// +// To work around this: +// 1. change the 'minTxConfirmations' to 1 to not include outbound tx right away (production should use 0) +// here: https://github.com/zeta-chain/node/blob/5c2a8ffbc702130fd9538b1cd7640d0e04d3e4f6/zetaclient/chains/bitcoin/observer/outbound.go#L27 +// 2. stop block mining to let the pending tx sit in the mempool for longer time func TestBitcoinWithdrawRBF(r *runner.E2ERunner, args []string) { require.Len(r, args, 2) From f1a1ca5059b0c9abc06bdf2ad0f846ba4630115d Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 8 May 2025 11:26:20 -0500 Subject: [PATCH 71/74] improve naming; add more description to the RBF e2e test; make 'approve' optional --- e2e/e2etests/helpers.go | 30 ++++++++++++----------- e2e/e2etests/test_bitcoin_withdraw_rbf.go | 6 ++--- e2e/utils/bitcoin.go | 8 +++--- e2e/utils/zetacore.go | 4 ++- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/e2e/e2etests/helpers.go b/e2e/e2etests/helpers.go index aab153ab41..22822ebd4f 100644 --- a/e2e/e2etests/helpers.go +++ b/e2e/e2etests/helpers.go @@ -30,7 +30,7 @@ func randomPayload(r *runner.E2ERunner) string { func withdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) *btcjson.TxRawResult { // approve and withdraw on ZRC20 contract - receipt := approveAndWithdrawBTCZRC20(r, to, amount) + receipt := BTCWithdraw(r, to, amount, true) // mine blocks if testing on regnet stop := r.MineBlocksIfLocalBitcoin() @@ -65,8 +65,8 @@ func withdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) return rawTx } -// approveAndWithdrawBTCZRC20 is a helper function to call 'approve' and 'withdraw' on BTCZRC20 contract -func approveAndWithdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) *ethtypes.Receipt { +// BTCWithdraw is a helper function to call 'withdraw' on BTCZRC20 contract with optional 'approve' +func BTCWithdraw(r *runner.E2ERunner, to btcutil.Address, amount *big.Int, approve bool) *ethtypes.Receipt { // ensure enough balance to cover the withdrawal _, gasFee, err := r.BTCZRC20.WithdrawGasFee(&bind.CallOpts{}) require.NoError(r, err) @@ -81,21 +81,23 @@ func approveAndWithdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount ) // approve more to cover withdraw fee - tx, err := r.BTCZRC20.Approve( - r.ZEVMAuth, - r.BTCZRC20Addr, - big.NewInt(amount.Int64()*2), - ) - require.NoError(r, err) - - receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt) + if approve { + tx, err := r.BTCZRC20.Approve( + r.ZEVMAuth, + r.BTCZRC20Addr, + big.NewInt(amount.Int64()*2), + ) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + } // withdraw 'amount' of BTC from ZRC20 to BTC address - tx, err = r.BTCZRC20.Withdraw(r.ZEVMAuth, []byte(to.EncodeAddress()), amount) + tx, err := r.BTCZRC20.Withdraw(r.ZEVMAuth, []byte(to.EncodeAddress()), amount) require.NoError(r, err) - receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) utils.RequireTxSuccessful(r, receipt) return receipt diff --git a/e2e/e2etests/test_bitcoin_withdraw_rbf.go b/e2e/e2etests/test_bitcoin_withdraw_rbf.go index 48856bd39d..aa7c6a9294 100644 --- a/e2e/e2etests/test_bitcoin_withdraw_rbf.go +++ b/e2e/e2etests/test_bitcoin_withdraw_rbf.go @@ -30,7 +30,7 @@ func TestBitcoinWithdrawRBF(r *runner.E2ERunner, args []string) { to, amount := utils.ParseBitcoinWithdrawArgs(r, args, defaultReceiver, r.GetBitcoinChainID()) // initiate a withdraw CCTX - receipt := approveAndWithdrawBTCZRC20(r, to, amount) + receipt := BTCWithdraw(r, to, amount, true) cctx := utils.GetCCTXByInboundHash(r.Ctx, r.CctxClient, receipt.TxHash.Hex()) // wait for the 1st outbound tracker hash to come in @@ -60,10 +60,10 @@ func TestBitcoinWithdrawRBF(r *runner.E2ERunner, args []string) { utils.RequireCCTXStatus(r, cctx, crosschaintypes.CctxStatus_OutboundMined) // ensure the original tx is dropped - utils.MustHaveDroppedTx(r.Ctx, r.BtcRPCClient, txHash) + utils.MustHaveDroppedBitcoinTx(r.Ctx, r.BtcRPCClient, txHash) // ensure the RBF tx is mined - rawResult := utils.MustHaveMinedTx(r.Ctx, r.BtcRPCClient, txHashRBF) + rawResult := utils.MustHaveMinedBitcoinTx(r.Ctx, r.BtcRPCClient, txHashRBF) // ensure RBF fee rate > old rate params := cctx.GetCurrentOutboundParam() diff --git a/e2e/utils/bitcoin.go b/e2e/utils/bitcoin.go index 782e727b03..ea63676299 100644 --- a/e2e/utils/bitcoin.go +++ b/e2e/utils/bitcoin.go @@ -10,8 +10,8 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/bitcoin/client" ) -// MustHaveDroppedTx ensures the given tx has been dropped -func MustHaveDroppedTx(ctx context.Context, client *client.Client, txHash *chainhash.Hash) { +// MustHaveDroppedBitcoinTx ensures the given Bitcoin tx has been dropped +func MustHaveDroppedBitcoinTx(ctx context.Context, client *client.Client, txHash *chainhash.Hash) { t := TestingFromContext(ctx) // dropped tx has negative confirmations @@ -32,8 +32,8 @@ func MustHaveDroppedTx(ctx context.Context, client *client.Client, txHash *chain require.Nil(t, rawTx) } -// MustHaveMinedTx ensures the given tx has been mined -func MustHaveMinedTx(ctx context.Context, client *client.Client, txHash *chainhash.Hash) *btcjson.TxRawResult { +// MustHaveMinedBitcoinTx ensures the given Bitcoin tx has been mined +func MustHaveMinedBitcoinTx(ctx context.Context, client *client.Client, txHash *chainhash.Hash) *btcjson.TxRawResult { t := TestingFromContext(ctx) // positive confirmations diff --git a/e2e/utils/zetacore.go b/e2e/utils/zetacore.go index 59c5980e64..7a6a15ab88 100644 --- a/e2e/utils/zetacore.go +++ b/e2e/utils/zetacore.go @@ -235,7 +235,9 @@ func WaitOutboundTracker( time.Since(startTime) > timeout, fmt.Sprintf("waiting outbound tracker timeout, chainID: %d, nonce: %d", chainID, nonce), ) - time.Sleep(5 * time.Second) + + // wait for a Zeta block before querying outbound tracker + time.Sleep(constant.ZetaBlockTime) outboundTracker, err := client.OutboundTrackerAllByChain(ctx, in) require.NoError(t, err) From 4f312365260dcaf3c8952785b6bd25244f1be17a Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Thu, 8 May 2025 14:00:38 -0500 Subject: [PATCH 72/74] add dependency to E2E test structure; inject dependency for Bitcoin RBF test --- cmd/zetae2e/local/bitcoin.go | 36 ++++++++++++++----------------- e2e/e2etests/e2etests.go | 7 ++++++ e2e/runner/e2etest.go | 41 +++++++++++++++++++++++++++++++++++- e2e/runner/run.go | 9 ++++++++ 4 files changed, 72 insertions(+), 21 deletions(-) diff --git a/cmd/zetae2e/local/bitcoin.go b/cmd/zetae2e/local/bitcoin.go index 9b79ec4bb7..efdcab655c 100644 --- a/cmd/zetae2e/local/bitcoin.go +++ b/cmd/zetae2e/local/bitcoin.go @@ -2,7 +2,6 @@ package local import ( "fmt" - "sync" "time" "github.com/fatih/color" @@ -15,6 +14,11 @@ import ( "github.com/zeta-chain/node/testutil" ) +const ( + groupDeposit = "btc_deposit" + groupWithdraw = "btc_withdraw" +) + // startBitcoinTests starts Bitcoin related tests func startBitcoinTests( eg *errgroup.Group, @@ -93,7 +97,7 @@ func bitcoinTestRoutines( // initialize runner for deposit tests account := conf.AdditionalAccounts.UserBitcoinDeposit runnerDeposit := initBitcoinRunner( - "btc_deposit", + groupDeposit, account, conf, deployerRunner, @@ -105,7 +109,7 @@ func bitcoinTestRoutines( // initialize runner for withdraw tests account = conf.AdditionalAccounts.UserBitcoinWithdraw runnerWithdraw := initBitcoinRunner( - "btc_withdraw", + groupWithdraw, account, conf, deployerRunner, @@ -127,8 +131,8 @@ func bitcoinTestRoutines( } // create test routines - routineDeposit, wgDeposit := createBitcoinTestRoutine(runnerDeposit, depositTests, nil) - routineWithdraw, _ := createBitcoinTestRoutine(runnerWithdraw, withdrawTests, wgDeposit) + routineDeposit := createBitcoinTestRoutine(runnerDeposit, depositTests, groupDeposit) + routineWithdraw := createBitcoinTestRoutine(runnerWithdraw, withdrawTests, groupWithdraw) return routineDeposit, routineWithdraw } @@ -182,14 +186,7 @@ func initBitcoinRunner( // createBitcoinTestRoutine creates a test routine for given test names // The 'wgDependency' argument is used to wait for dependent routine to complete -func createBitcoinTestRoutine( - r *runner.E2ERunner, - testNames []string, - wgDependency *sync.WaitGroup, -) (func() error, *sync.WaitGroup) { - var thisRoutine sync.WaitGroup - thisRoutine.Add(1) - +func createBitcoinTestRoutine(r *runner.E2ERunner, testNames []string, name string) func() error { return func() (err error) { r.Logger.Print("🏃 starting bitcoin tests") startTime := time.Now() @@ -204,19 +201,18 @@ func createBitcoinTestRoutine( } for _, test := range testsToRun { - // RBF test needs to wait for all deposit tests to complete - if test.Name == e2etests.TestBitcoinWithdrawRBFName && wgDependency != nil { - r.Logger.Print("⏳ waiting for deposit runner to complete before running RBF test") - wgDependency.Wait() - } if err := r.RunE2ETest(test, true); err != nil { return fmt.Errorf("bitcoin tests failed: %v", err) } } - thisRoutine.Done() r.Logger.Print("🍾 bitcoin tests completed in %s", time.Since(startTime).String()) + // mark deposit test group as done + if name == groupDeposit { + e2etests.DepdencyAllBitcoinDeposits.Done() + } + return err - }, &thisRoutine + } } diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 92f8bf61af..23e823101a 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -262,6 +262,12 @@ const ( CountArgDescription = "count" ) +// Here are all the dependencies for the e2e tests, add more dependencies here if needed +var ( + // DepdencyAllBitcoinDeposits is a dependency to wait for all bitcoin deposit tests to complete + DepdencyAllBitcoinDeposits = runner.NewE2EDependency("all_bitcoin_deposits") +) + // AllE2ETests is an ordered list of all e2e tests var AllE2ETests = []runner.E2ETest{ /* @@ -1147,6 +1153,7 @@ var AllE2ETests = []runner.E2ETest{ {Description: "amount in btc", DefaultValue: "0.001"}, }, TestBitcoinWithdrawRBF, + runner.WithDependencies(DepdencyAllBitcoinDeposits), runner.WithMinimumVersion("v31.0.0"), ), /* diff --git a/e2e/runner/e2etest.go b/e2e/runner/e2etest.go index 8de3217800..5e02f1d62f 100644 --- a/e2e/runner/e2etest.go +++ b/e2e/runner/e2etest.go @@ -1,6 +1,9 @@ package runner -import "fmt" +import ( + "fmt" + "sync" +) // E2ETestFunc is a function representing a E2E test // It takes a E2ERunner as an argument @@ -16,12 +19,20 @@ func WithMinimumVersion(version string) E2ETestOpt { } } +// WithDependencies sets dependencies to the E2ETest to wait for completion +func WithDependencies(dependencies ...E2EDependency) E2ETestOpt { + return func(t *E2ETest) { + t.Dependencies = dependencies + } +} + // E2ETest represents a E2E test with a name, args, description and test func type E2ETest struct { Name string Description string Args []string ArgsDefinition []ArgDefinition + Dependencies []E2EDependency E2ETest E2ETestFunc MinimumVersion string } @@ -46,6 +57,33 @@ func NewE2ETest( return test } +// E2EDependency defines a structure that holds a E2E test dependency +type E2EDependency struct { + name string + waitGroup *sync.WaitGroup +} + +// NewE2EDependency creates a new instance of E2Edependency with specified parameters. +func NewE2EDependency(name string) E2EDependency { + var wg sync.WaitGroup + wg.Add(1) + + return E2EDependency{ + name: name, + waitGroup: &wg, + } +} + +// Wait waits for the E2EDependency to complete +func (d *E2EDependency) Wait() { + d.waitGroup.Wait() +} + +// Done marks the E2EDependency as done +func (d *E2EDependency) Done() { + d.waitGroup.Done() +} + // ArgDefinition defines a structure for holding an argument's description along with it's default value. type ArgDefinition struct { Description string @@ -110,6 +148,7 @@ func (r *E2ERunner) GetE2ETestsToRunByConfig( Name: e2eTest.Name, Description: e2eTest.Description, ArgsDefinition: e2eTest.ArgsDefinition, + Dependencies: e2eTest.Dependencies, E2ETest: e2eTest.E2ETest, MinimumVersion: e2eTest.MinimumVersion, } diff --git a/e2e/runner/run.go b/e2e/runner/run.go index 66f6cc6474..2d4481904f 100644 --- a/e2e/runner/run.go +++ b/e2e/runner/run.go @@ -28,6 +28,15 @@ func (r *E2ERunner) RunE2ETests(e2eTests []E2ETest) (err error) { // RunE2ETest runs a e2e test func (r *E2ERunner) RunE2ETest(e2eTest E2ETest, checkAccounting bool) error { + // wait for all dependencies to complete + // this is only used by Bitcoin RBF test at the moment + if len(e2eTest.Dependencies) > 0 { + r.Logger.Print("⏳ waiting - %s", e2eTest.Name) + for _, dependency := range e2eTest.Dependencies { + dependency.Wait() + } + } + startTime := time.Now() // note: spacing is padded to width of completed message r.Logger.Print("⏳ running - %s", e2eTest.Name) From 142e0a459eb2f993a014328f95c5cc43a7f385ab Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Tue, 13 May 2025 10:32:08 -0500 Subject: [PATCH 73/74] move helper func BTCWithdraw to a bitcoin runner method --- cmd/zetae2e/local/bitcoin.go | 14 ++++---- e2e/e2etests/helpers.go | 42 +---------------------- e2e/e2etests/test_bitcoin_withdraw_rbf.go | 2 +- e2e/runner/bitcoin.go | 40 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 49 deletions(-) diff --git a/cmd/zetae2e/local/bitcoin.go b/cmd/zetae2e/local/bitcoin.go index efdcab655c..fd0bc1c3cc 100644 --- a/cmd/zetae2e/local/bitcoin.go +++ b/cmd/zetae2e/local/bitcoin.go @@ -15,8 +15,8 @@ import ( ) const ( - groupDeposit = "btc_deposit" - groupWithdraw = "btc_withdraw" + testGroupDepositName = "btc_deposit" + testGroupWithdrawName = "btc_withdraw" ) // startBitcoinTests starts Bitcoin related tests @@ -97,7 +97,7 @@ func bitcoinTestRoutines( // initialize runner for deposit tests account := conf.AdditionalAccounts.UserBitcoinDeposit runnerDeposit := initBitcoinRunner( - groupDeposit, + testGroupDepositName, account, conf, deployerRunner, @@ -109,7 +109,7 @@ func bitcoinTestRoutines( // initialize runner for withdraw tests account = conf.AdditionalAccounts.UserBitcoinWithdraw runnerWithdraw := initBitcoinRunner( - groupWithdraw, + testGroupWithdrawName, account, conf, deployerRunner, @@ -131,8 +131,8 @@ func bitcoinTestRoutines( } // create test routines - routineDeposit := createBitcoinTestRoutine(runnerDeposit, depositTests, groupDeposit) - routineWithdraw := createBitcoinTestRoutine(runnerWithdraw, withdrawTests, groupWithdraw) + routineDeposit := createBitcoinTestRoutine(runnerDeposit, depositTests, testGroupDepositName) + routineWithdraw := createBitcoinTestRoutine(runnerWithdraw, withdrawTests, testGroupWithdrawName) return routineDeposit, routineWithdraw } @@ -209,7 +209,7 @@ func createBitcoinTestRoutine(r *runner.E2ERunner, testNames []string, name stri r.Logger.Print("🍾 bitcoin tests completed in %s", time.Since(startTime).String()) // mark deposit test group as done - if name == groupDeposit { + if name == testGroupDepositName { e2etests.DepdencyAllBitcoinDeposits.Done() } diff --git a/e2e/e2etests/helpers.go b/e2e/e2etests/helpers.go index 22822ebd4f..ed4de62bc4 100644 --- a/e2e/e2etests/helpers.go +++ b/e2e/e2etests/helpers.go @@ -10,8 +10,6 @@ import ( "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/require" "github.com/zeta-chain/node/e2e/runner" @@ -30,7 +28,7 @@ func randomPayload(r *runner.E2ERunner) string { func withdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) *btcjson.TxRawResult { // approve and withdraw on ZRC20 contract - receipt := BTCWithdraw(r, to, amount, true) + receipt := r.WithdrawBTC(to, amount, true) // mine blocks if testing on regnet stop := r.MineBlocksIfLocalBitcoin() @@ -65,44 +63,6 @@ func withdrawBTCZRC20(r *runner.E2ERunner, to btcutil.Address, amount *big.Int) return rawTx } -// BTCWithdraw is a helper function to call 'withdraw' on BTCZRC20 contract with optional 'approve' -func BTCWithdraw(r *runner.E2ERunner, to btcutil.Address, amount *big.Int, approve bool) *ethtypes.Receipt { - // ensure enough balance to cover the withdrawal - _, gasFee, err := r.BTCZRC20.WithdrawGasFee(&bind.CallOpts{}) - require.NoError(r, err) - minimumAmount := new(big.Int).Add(amount, gasFee) - currentBalance, err := r.BTCZRC20.BalanceOf(&bind.CallOpts{}, r.ZEVMAuth.From) - require.NoError(r, err) - require.Greater( - r, - currentBalance.Int64(), - minimumAmount.Int64(), - "current balance must be greater than amount + gasFee", - ) - - // approve more to cover withdraw fee - if approve { - tx, err := r.BTCZRC20.Approve( - r.ZEVMAuth, - r.BTCZRC20Addr, - big.NewInt(amount.Int64()*2), - ) - require.NoError(r, err) - - receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt) - } - - // withdraw 'amount' of BTC from ZRC20 to BTC address - tx, err := r.BTCZRC20.Withdraw(r.ZEVMAuth, []byte(to.EncodeAddress()), amount) - require.NoError(r, err) - - receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt) - - return receipt -} - // bigAdd is shorthand for new(big.Int).Add(x, y) func bigAdd(x *big.Int, y *big.Int) *big.Int { return new(big.Int).Add(x, y) diff --git a/e2e/e2etests/test_bitcoin_withdraw_rbf.go b/e2e/e2etests/test_bitcoin_withdraw_rbf.go index aa7c6a9294..906cf87b79 100644 --- a/e2e/e2etests/test_bitcoin_withdraw_rbf.go +++ b/e2e/e2etests/test_bitcoin_withdraw_rbf.go @@ -30,7 +30,7 @@ func TestBitcoinWithdrawRBF(r *runner.E2ERunner, args []string) { to, amount := utils.ParseBitcoinWithdrawArgs(r, args, defaultReceiver, r.GetBitcoinChainID()) // initiate a withdraw CCTX - receipt := BTCWithdraw(r, to, amount, true) + receipt := r.WithdrawBTC(to, amount, true) cctx := utils.GetCCTXByInboundHash(r.Ctx, r.CctxClient, receipt.TxHash.Hex()) // wait for the 1st outbound tracker hash to come in diff --git a/e2e/runner/bitcoin.go b/e2e/runner/bitcoin.go index 6b70e1b093..6eb70739bd 100644 --- a/e2e/runner/bitcoin.go +++ b/e2e/runner/bitcoin.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "math/big" "sort" "time" @@ -15,6 +16,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/rs/zerolog/log" "github.com/stretchr/testify/require" @@ -182,6 +184,44 @@ func (r *E2ERunner) DepositBTC(receiver common.Address) { require.Equal(r, 1, balance.Sign(), "balance should be positive") } +// WithdrawBTC is a helper function to call 'withdraw' on BTCZRC20 contract with optional 'approve' +func (r *E2ERunner) WithdrawBTC(to btcutil.Address, amount *big.Int, approve bool) *ethtypes.Receipt { + // ensure enough balance to cover the withdrawal + _, gasFee, err := r.BTCZRC20.WithdrawGasFee(&bind.CallOpts{}) + require.NoError(r, err) + minimumAmount := new(big.Int).Add(amount, gasFee) + currentBalance, err := r.BTCZRC20.BalanceOf(&bind.CallOpts{}, r.ZEVMAuth.From) + require.NoError(r, err) + require.Greater( + r, + currentBalance.Int64(), + minimumAmount.Int64(), + "current balance must be greater than amount + gasFee", + ) + + // approve more to cover withdraw fee + if approve { + tx, err := r.BTCZRC20.Approve( + r.ZEVMAuth, + r.BTCZRC20Addr, + big.NewInt(amount.Int64()*2), + ) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + } + + // withdraw 'amount' of BTC from ZRC20 to BTC address + tx, err := r.BTCZRC20.Withdraw(r.ZEVMAuth, []byte(to.EncodeAddress()), amount) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + return receipt +} + func (r *E2ERunner) SendToTSSWithMemo( amount float64, memo []byte, From 8de2a68431b13b124290334ec621de72ccd371c8 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 14 May 2025 15:29:33 -0500 Subject: [PATCH 74/74] fix upgrade test failure in CI --- cmd/zetae2e/local/bitcoin.go | 6 ++---- e2e/e2etests/e2etests.go | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/zetae2e/local/bitcoin.go b/cmd/zetae2e/local/bitcoin.go index fd0bc1c3cc..729d5ad999 100644 --- a/cmd/zetae2e/local/bitcoin.go +++ b/cmd/zetae2e/local/bitcoin.go @@ -200,10 +200,8 @@ func createBitcoinTestRoutine(r *runner.E2ERunner, testNames []string, name stri return fmt.Errorf("bitcoin tests failed: %v", err) } - for _, test := range testsToRun { - if err := r.RunE2ETest(test, true); err != nil { - return fmt.Errorf("bitcoin tests failed: %v", err) - } + if err := r.RunE2ETests(testsToRun); err != nil { + return fmt.Errorf("bitcoin tests failed: %v", err) } r.Logger.Print("🍾 bitcoin tests completed in %s", time.Since(startTime).String()) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index a207f1c2ce..5a872c3a48 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -1181,7 +1181,6 @@ var AllE2ETests = []runner.E2ETest{ }, TestBitcoinWithdrawRBF, runner.WithDependencies(DepdencyAllBitcoinDeposits), - runner.WithMinimumVersion("v31.0.0"), ), /* Application tests