From 6790a4ea2768d060bc1af4921deb90aacf4b9336 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Wed, 16 Oct 2024 16:20:33 -0300 Subject: [PATCH 01/90] First version of retry function --- batcher/Cargo.lock | 28 +++++++- batcher/aligned-batcher/Cargo.toml | 1 + batcher/aligned-batcher/src/connection.rs | 82 ++++++++++++++++++++++- 3 files changed, 106 insertions(+), 5 deletions(-) diff --git a/batcher/Cargo.lock b/batcher/Cargo.lock index 210d70bdc4..da0b5a014e 100644 --- a/batcher/Cargo.lock +++ b/batcher/Cargo.lock @@ -78,7 +78,7 @@ dependencies = [ [[package]] name = "aligned" -version = "0.8.0" +version = "0.9.2" dependencies = [ "aligned-sdk", "clap", @@ -104,6 +104,7 @@ dependencies = [ "anyhow", "aws-config", "aws-sdk-s3", + "backon", "bincode", "bytes", "ciborium", @@ -1055,6 +1056,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "backon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4fa97bb310c33c811334143cf64c5bb2b7b3c06e453db6b095d7061eff8f113" +dependencies = [ + "fastrand", + "gloo-timers 0.3.0", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -2854,7 +2866,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" dependencies = [ - "gloo-timers", + "gloo-timers 0.2.6", "send_wrapper 0.4.0", ] @@ -2960,6 +2972,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "group" version = "0.12.1" diff --git a/batcher/aligned-batcher/Cargo.toml b/batcher/aligned-batcher/Cargo.toml index 8928f000b4..2f9e8df4a4 100644 --- a/batcher/aligned-batcher/Cargo.toml +++ b/batcher/aligned-batcher/Cargo.toml @@ -35,3 +35,4 @@ priority-queue = "2.1.0" once_cell = "1.20.2" warp = "0.3.7" prometheus = { version = "0.13.4", features = ["process"] } +backon = "1.2.0" diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index 65bbcf384e..e8a24096e9 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -1,9 +1,12 @@ -use std::sync::Arc; +use std::{future::Future, sync::Arc, time::Duration}; +use crate::types::{batch_queue::BatchQueueEntry, errors::BatcherError}; use aligned_sdk::{ communication::serialization::cbor_serialize, core::types::{BatchInclusionData, ResponseMessage, VerificationCommitmentBatch}, }; +use backon::ExponentialBuilder; +use backon::Retryable; use futures_util::{stream::SplitSink, SinkExt}; use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use log::{error, info}; @@ -14,8 +17,6 @@ use tokio_tungstenite::{ WebSocketStream, }; -use crate::types::{batch_queue::BatchQueueEntry, errors::BatcherError}; - pub(crate) type WsMessageSink = Arc, Message>>>; pub(crate) async fn send_batch_inclusion_data_responses( @@ -66,3 +67,78 @@ pub(crate) async fn send_message(ws_conn_sink: WsMessageSink, mess Err(e) => error!("Error while serializing message: {}", e), } } + +#[derive(Debug)] +enum RetryError { + Transient, + Permanent(E), +} + +impl std::fmt::Display for RetryError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Retry error!") + } +} +impl std::error::Error for RetryError where E: std::fmt::Debug {} + +async fn retry( + function: FutureFn, + min_delay: u64, + factor: f32, + max_tries: usize, +) -> Result> +where + Fut: Future>>, + FutureFn: FnMut() -> Fut, +{ + let backoff = ExponentialBuilder::default() + .with_min_delay(Duration::from_millis(min_delay)) + .with_max_times(max_tries) + .with_factor(factor); + + function + .retry(backoff) + .sleep(tokio::time::sleep) + .when(|e| matches!(e, RetryError::Transient)) + .await +} + +#[cfg(test)] +mod test { + use super::*; + use crate::eth; + use ethers::{providers::Middleware, types::U256}; + use std::time::SystemTime; + + #[tokio::test] + async fn retry_test() { + async fn dummy_action(x: u64) -> Result> { + println!("Doing some operation..."); + println!("Actual time: {:?}", SystemTime::now()); + println!("X: {x}"); + + Err(RetryError::Permanent(())) + } + + assert!(retry(|| dummy_action(10), 2000, 2.0, 3).await.is_err()); + } + + #[tokio::test] + async fn retry_test_eth() { + async fn get_gas_price() -> Result> { + let eth_rpc_provider = + eth::get_provider(String::from("https://ethereum-holesky-rpc.publicnode.com")) + .expect("Failed to get provider"); + + match eth_rpc_provider.get_gas_price().await { + Ok(val) => { + println!("GAS PRICE IS: {:?}", val); + Ok(val) + } + Err(_) => Err(RetryError::Transient), + } + } + + assert!(retry(get_gas_price, 2000, 2.0, 3).await.is_ok()); + } +} From bea9ba21b1e2f832903f65a1587987bd8efd1a20 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 16 Oct 2024 19:35:17 -0300 Subject: [PATCH 02/90] refactor: move retry to its own module --- batcher/aligned-batcher/src/connection.rs | 79 +---------------------- batcher/aligned-batcher/src/lib.rs | 1 + batcher/aligned-batcher/src/retry.rs | 78 ++++++++++++++++++++++ 3 files changed, 80 insertions(+), 78 deletions(-) create mode 100644 batcher/aligned-batcher/src/retry.rs diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index e8a24096e9..501604788e 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -1,12 +1,10 @@ -use std::{future::Future, sync::Arc, time::Duration}; +use std::sync::Arc; use crate::types::{batch_queue::BatchQueueEntry, errors::BatcherError}; use aligned_sdk::{ communication::serialization::cbor_serialize, core::types::{BatchInclusionData, ResponseMessage, VerificationCommitmentBatch}, }; -use backon::ExponentialBuilder; -use backon::Retryable; use futures_util::{stream::SplitSink, SinkExt}; use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use log::{error, info}; @@ -67,78 +65,3 @@ pub(crate) async fn send_message(ws_conn_sink: WsMessageSink, mess Err(e) => error!("Error while serializing message: {}", e), } } - -#[derive(Debug)] -enum RetryError { - Transient, - Permanent(E), -} - -impl std::fmt::Display for RetryError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "Retry error!") - } -} -impl std::error::Error for RetryError where E: std::fmt::Debug {} - -async fn retry( - function: FutureFn, - min_delay: u64, - factor: f32, - max_tries: usize, -) -> Result> -where - Fut: Future>>, - FutureFn: FnMut() -> Fut, -{ - let backoff = ExponentialBuilder::default() - .with_min_delay(Duration::from_millis(min_delay)) - .with_max_times(max_tries) - .with_factor(factor); - - function - .retry(backoff) - .sleep(tokio::time::sleep) - .when(|e| matches!(e, RetryError::Transient)) - .await -} - -#[cfg(test)] -mod test { - use super::*; - use crate::eth; - use ethers::{providers::Middleware, types::U256}; - use std::time::SystemTime; - - #[tokio::test] - async fn retry_test() { - async fn dummy_action(x: u64) -> Result> { - println!("Doing some operation..."); - println!("Actual time: {:?}", SystemTime::now()); - println!("X: {x}"); - - Err(RetryError::Permanent(())) - } - - assert!(retry(|| dummy_action(10), 2000, 2.0, 3).await.is_err()); - } - - #[tokio::test] - async fn retry_test_eth() { - async fn get_gas_price() -> Result> { - let eth_rpc_provider = - eth::get_provider(String::from("https://ethereum-holesky-rpc.publicnode.com")) - .expect("Failed to get provider"); - - match eth_rpc_provider.get_gas_price().await { - Ok(val) => { - println!("GAS PRICE IS: {:?}", val); - Ok(val) - } - Err(_) => Err(RetryError::Transient), - } - } - - assert!(retry(get_gas_price, 2000, 2.0, 3).await.is_ok()); - } -} diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 252c9f1779..e4c5534316 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -36,6 +36,7 @@ use crate::config::{ConfigFromYaml, ContractDeploymentOutput}; mod config; mod connection; mod eth; +mod retry; pub mod gnark; pub mod metrics; pub mod risc_zero; diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs new file mode 100644 index 0000000000..d5251a1ab9 --- /dev/null +++ b/batcher/aligned-batcher/src/retry.rs @@ -0,0 +1,78 @@ +use std::{future::Future, time::Duration}; +use backon::ExponentialBuilder; +use backon::Retryable; + +#[derive(Debug)] +pub enum RetryError { + Transient, + Permanent(E), +} + +impl std::fmt::Display for RetryError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Retry error!") + } +} +impl std::error::Error for RetryError where E: std::fmt::Debug {} + +async fn retry( + function: FutureFn, + min_delay: u64, + factor: f32, + max_tries: usize, +) -> Result> +where + Fut: Future>>, + FutureFn: FnMut() -> Fut, +{ + let backoff = ExponentialBuilder::default() + .with_min_delay(Duration::from_millis(min_delay)) + .with_max_times(max_tries) + .with_factor(factor); + + function + .retry(backoff) + .sleep(tokio::time::sleep) + .when(|e| matches!(e, RetryError::Transient)) + .await +} + +#[cfg(test)] +mod test { + use super::*; + use crate::eth; + use ethers::{providers::Middleware, types::U256}; + use std::time::SystemTime; + + #[tokio::test] + async fn retry_test() { + async fn dummy_action(x: u64) -> Result> { + println!("Doing some operation..."); + println!("Actual time: {:?}", SystemTime::now()); + println!("X: {x}"); + + Err(RetryError::Permanent(())) + } + + assert!(retry(|| dummy_action(10), 2000, 2.0, 3).await.is_err()); + } + + #[tokio::test] + async fn retry_test_eth() { + async fn get_gas_price() -> Result> { + let eth_rpc_provider = + eth::get_provider(String::from("https://ethereum-holesky-rpc.publicnode.com")) + .expect("Failed to get provider"); + + match eth_rpc_provider.get_gas_price().await { + Ok(val) => { + println!("GAS PRICE IS: {:?}", val); + Ok(val) + } + Err(_) => Err(RetryError::Transient), + } + } + + assert!(retry(get_gas_price, 2000, 2.0, 3).await.is_ok()); + } +} From 651a34568eca891595ab17d92458c65e9e76ca66 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 16 Oct 2024 20:03:58 -0300 Subject: [PATCH 03/90] feat: add retry logic to get_gas_price --- batcher/aligned-batcher/src/lib.rs | 28 +++++++++++++++++++--------- batcher/aligned-batcher/src/retry.rs | 10 ++++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index e4c5534316..847aa2a37c 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -4,6 +4,7 @@ use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use ethers::contract::ContractError; use ethers::signers::Signer; +use retry::{retry_function, RetryError}; use types::batch_state::BatchState; use types::user_state::UserState; @@ -36,9 +37,9 @@ use crate::config::{ConfigFromYaml, ContractDeploymentOutput}; mod config; mod connection; mod eth; -mod retry; pub mod gnark; pub mod metrics; +mod retry; pub mod risc_zero; pub mod s3; pub mod sp1; @@ -951,7 +952,7 @@ impl Batcher { /// Receives new block numbers, checks if conditions are met for submission and /// finalizes the batch. async fn handle_new_block(&self, block_number: u64) -> Result<(), BatcherError> { - let gas_price = match self.get_gas_price().await { + let gas_price = match self.get_gas_price_with_retry().await { Some(price) => price, None => { error!("Failed to get gas price"); @@ -1220,23 +1221,32 @@ impl Batcher { false } + async fn get_gas_price_with_retry(&self) -> Option { + if let Ok(gas_price) = retry_function(|| self.get_gas_price(), 2000, 2.0, 3).await { + return Some(gas_price); + } + None + } + /// Gets the current gas price from Ethereum. /// Returns `None` if the gas price couldn't be returned /// FIXME: This should return a `Result` instead. - async fn get_gas_price(&self) -> Option { + async fn get_gas_price(&self) -> Result> { if let Ok(gas_price) = self .eth_ws_provider .get_gas_price() .await .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) { - return Some(gas_price); + return Ok(gas_price); } - self.eth_ws_provider_fallback - .get_gas_price() - .await - .inspect_err(|e| warn!("Failed to get gas price: {e:?}")) - .ok() + match self.eth_ws_provider_fallback.get_gas_price().await { + Ok(gas_price) => Ok(gas_price), + Err(e) => { + warn!("Failed to get gas price: {e:?}"); + Err(RetryError::Transient) + } + } } } diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index d5251a1ab9..9fa5374b19 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -1,6 +1,6 @@ -use std::{future::Future, time::Duration}; use backon::ExponentialBuilder; use backon::Retryable; +use std::{future::Future, time::Duration}; #[derive(Debug)] pub enum RetryError { @@ -15,7 +15,7 @@ impl std::fmt::Display for RetryError { } impl std::error::Error for RetryError where E: std::fmt::Debug {} -async fn retry( +pub async fn retry_function( function: FutureFn, min_delay: u64, factor: f32, @@ -54,7 +54,9 @@ mod test { Err(RetryError::Permanent(())) } - assert!(retry(|| dummy_action(10), 2000, 2.0, 3).await.is_err()); + assert!(retry_function(|| dummy_action(10), 2000, 2.0, 3) + .await + .is_err()); } #[tokio::test] @@ -73,6 +75,6 @@ mod test { } } - assert!(retry(get_gas_price, 2000, 2.0, 3).await.is_ok()); + assert!(retry_function(get_gas_price, 2000, 2.0, 3).await.is_ok()); } } From 32c8ba1727aa7e41dcb01a8430bd060429d8c60c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 16 Oct 2024 20:20:17 -0300 Subject: [PATCH 04/90] refactor: syntactic sugar --- batcher/aligned-batcher/src/lib.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 847aa2a37c..fd88246b4e 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1222,10 +1222,9 @@ impl Batcher { } async fn get_gas_price_with_retry(&self) -> Option { - if let Ok(gas_price) = retry_function(|| self.get_gas_price(), 2000, 2.0, 3).await { - return Some(gas_price); - } - None + retry_function(|| self.get_gas_price(), 2000, 2.0, 3) + .await + .ok() } /// Gets the current gas price from Ethereum. @@ -1241,12 +1240,12 @@ impl Batcher { return Ok(gas_price); } - match self.eth_ws_provider_fallback.get_gas_price().await { - Ok(gas_price) => Ok(gas_price), - Err(e) => { - warn!("Failed to get gas price: {e:?}"); - Err(RetryError::Transient) - } - } + self.eth_ws_provider_fallback + .get_gas_price() + .await + .map_err(|e| { + warn!("Failed to get fallback gas price: {e:?}"); + RetryError::Transient + }) } } From efcfd43726e82416ca1806b29637a63fcc25351c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 11:55:16 -0300 Subject: [PATCH 05/90] refactor: add retry to get_balance_is_unlocked --- batcher/aligned-batcher/src/lib.rs | 39 +++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index fd88246b4e..fa78fdfcd8 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -404,7 +404,7 @@ impl Batcher { // We don't need a batch state lock here, since if the user locks its funds // after the check, some blocks should pass until he can withdraw. // It is safe to do just do this here. - if self.user_balance_is_unlocked(&addr).await { + if self.user_balance_is_unlocked_with_retry(&addr).await { send_message( ws_conn_sink.clone(), ValidityResponseMessage::InsufficientBalance(addr), @@ -952,13 +952,7 @@ impl Batcher { /// Receives new block numbers, checks if conditions are met for submission and /// finalizes the batch. async fn handle_new_block(&self, block_number: u64) -> Result<(), BatcherError> { - let gas_price = match self.get_gas_price_with_retry().await { - Some(price) => price, - None => { - error!("Failed to get gas price"); - return Err(BatcherError::GasPriceError); - } - }; + let gas_price = self.get_gas_price_with_retry().await?; if let Some(finalized_batch) = self.is_batch_ready(block_number, gas_price).await { let batch_finalization_result = self @@ -1205,9 +1199,18 @@ impl Batcher { .ok() } - async fn user_balance_is_unlocked(&self, addr: &Address) -> bool { + /// Checks if the user's balance is unlocked for a given address using exponential backoff. + /// Returns `false` if an error occurs during the retries. + async fn user_balance_is_unlocked_with_retry(&self, addr: &Address) -> bool { + match retry_function(|| self.user_balance_is_unlocked(addr), 2000, 2.0, 3).await { + Ok(result) => result, + Err(_) => false, + } + } + + async fn user_balance_is_unlocked(&self, addr: &Address) -> Result> { if let Ok(unlock_block) = self.payment_service.user_unlock_block(*addr).call().await { - return unlock_block != U256::zero(); + return Ok(unlock_block != U256::zero()); } if let Ok(unlock_block) = self .payment_service_fallback @@ -1215,21 +1218,23 @@ impl Batcher { .call() .await { - return unlock_block != U256::zero(); + return Ok(unlock_block != U256::zero()); } warn!("Could not get user locking state"); - false + Err(RetryError::Transient) } - async fn get_gas_price_with_retry(&self) -> Option { + /// Gets the current gas price from Ethereum using exponential backoff. + /// Returns `None` if the gas price couldn't be returned + async fn get_gas_price_with_retry(&self) -> Result { retry_function(|| self.get_gas_price(), 2000, 2.0, 3) .await - .ok() + .map_err(|_| { + error!("Failed to get gas price"); + BatcherError::GasPriceError + }) } - /// Gets the current gas price from Ethereum. - /// Returns `None` if the gas price couldn't be returned - /// FIXME: This should return a `Result` instead. async fn get_gas_price(&self) -> Result> { if let Ok(gas_price) = self .eth_ws_provider From a5fc3f868f5ab4973e92b03c4e59f852a0b3bc97 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 12:17:34 -0300 Subject: [PATCH 06/90] refactor: add retry to get_user_balance --- batcher/aligned-batcher/src/lib.rs | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index fa78fdfcd8..37ba987c40 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -456,7 +456,7 @@ impl Batcher { // * Perform validations over user state * // * ---------------------------------------------------* - let Some(user_balance) = self.get_user_balance(&addr).await else { + let Ok(user_balance) = self.get_user_balance_with_retry(&addr).await else { error!("Could not get balance for address {addr:?}"); send_message(ws_conn_sink.clone(), ValidityResponseMessage::EthRpcError).await; return Ok(()); @@ -1117,7 +1117,9 @@ impl Batcher { }; let replacement_addr = non_paying_config.replacement.address(); - let Some(replacement_user_balance) = self.get_user_balance(&replacement_addr).await else { + let Ok(replacement_user_balance) = + self.get_user_balance_with_retry(&replacement_addr).await + else { error!("Could not get balance for non-paying address {replacement_addr:?}"); send_message( ws_sink.clone(), @@ -1183,20 +1185,24 @@ impl Batcher { Ok(()) } - /// Gets the balance of user with address `addr` from Ethereum. - /// Returns `None` if the balance couldn't be returned - /// FIXME: This should return a `Result` instead. - async fn get_user_balance(&self, addr: &Address) -> Option { + /// Gets the balance of user with address `addr` from Ethereum using exponential backoff. + async fn get_user_balance_with_retry(&self, addr: &Address) -> Result> { + retry_function(|| self.get_user_balance(addr), 2000, 2.0, 3).await + } + + async fn get_user_balance(&self, addr: &Address) -> Result> { if let Ok(balance) = self.payment_service.user_balances(*addr).call().await { - return Some(balance); + return Ok(balance); }; self.payment_service_fallback .user_balances(*addr) .call() .await - .inspect_err(|_| warn!("Failed to get balance for address {:?}", addr)) - .ok() + .map_err(|_| { + warn!("Failed to get balance for address {:?}", addr); + RetryError::Transient + }) } /// Checks if the user's balance is unlocked for a given address using exponential backoff. @@ -1204,7 +1210,10 @@ impl Batcher { async fn user_balance_is_unlocked_with_retry(&self, addr: &Address) -> bool { match retry_function(|| self.user_balance_is_unlocked(addr), 2000, 2.0, 3).await { Ok(result) => result, - Err(_) => false, + Err(_) => { + warn!("Could not get user locking state"); + false + } } } @@ -1220,7 +1229,6 @@ impl Batcher { { return Ok(unlock_block != U256::zero()); } - warn!("Could not get user locking state"); Err(RetryError::Transient) } From 064cf7b4abc12f28b479cd379ff46551e51130b7 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 12:40:38 -0300 Subject: [PATCH 07/90] refactor: add retry to get_user_nonce --- batcher/aligned-batcher/src/lib.rs | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 37ba987c40..32a148a3f9 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -434,7 +434,8 @@ impl Batcher { } if !is_user_in_state { - let ethereum_user_nonce = match self.get_user_nonce_from_ethereum(addr).await { + let ethereum_user_nonce = match self.get_user_nonce_from_ethereum_with_retry(addr).await + { Ok(ethereum_user_nonce) => ethereum_user_nonce, Err(e) => { error!( @@ -657,14 +658,23 @@ impl Batcher { }; } - async fn get_user_nonce_from_ethereum( + /// Gets the user nonce from Ethereum using exponential backoff. + async fn get_user_nonce_from_ethereum_with_retry( &self, addr: Address, - ) -> Result> { - match self.payment_service.user_nonces(addr).call().await { - Ok(nonce) => Ok(nonce), - Err(_) => self.payment_service_fallback.user_nonces(addr).call().await, + ) -> Result> { + retry_function(|| self.get_user_nonce_from_ethereum(addr), 2000, 2.0, 3).await + } + + async fn get_user_nonce_from_ethereum(&self, addr: Address) -> Result> { + if let Ok(nonce) = self.payment_service.user_nonces(addr).call().await { + return Ok(nonce); } + self.payment_service_fallback + .user_nonces(addr) + .call() + .await + .map_err(|_| RetryError::Transient) } /// Adds verification data to the current batch queue. @@ -934,7 +944,7 @@ impl Batcher { // so that it is already loaded let Ok(nonpaying_replacement_addr_nonce) = self - .get_user_nonce_from_ethereum(nonpaying_replacement_addr) + .get_user_nonce_from_ethereum_with_retry(nonpaying_replacement_addr) .await else { batch_state_lock.batch_queue.clear(); From f620b087d4f731cb9b2c2d31483c00f97e58b109 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 13:02:21 -0300 Subject: [PATCH 08/90] refactor: add retry when uploading batch to s3 --- batcher/aligned-batcher/src/lib.rs | 45 ++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 32a148a3f9..e13994d9a2 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -988,20 +988,13 @@ impl Batcher { finalized_batch: &[BatchQueueEntry], gas_price: U256, ) -> Result<(), BatcherError> { - let s3_client = self.s3_client.clone(); let batch_merkle_root_hex = hex::encode(batch_merkle_root); info!("Batch merkle root: 0x{}", batch_merkle_root_hex); let file_name = batch_merkle_root_hex.clone() + ".json"; info!("Uploading batch to S3..."); - s3::upload_object( - &s3_client, - &self.s3_bucket_name, - batch_bytes.to_vec(), - &file_name, - ) - .await - .map_err(|e| BatcherError::BatchUploadError(e.to_string()))?; + self.upload_batch_to_s3_with_retry(batch_bytes, &file_name) + .await?; info!("Batch sent to S3 with name: {}", file_name); @@ -1271,4 +1264,38 @@ impl Batcher { RetryError::Transient }) } + + /// Gets the current gas price from Ethereum using exponential backoff. + async fn upload_batch_to_s3_with_retry( + &self, + batch_bytes: &[u8], + file_name: &String, + ) -> Result<(), BatcherError> { + retry_function( + || self.upload_batch_to_s3(batch_bytes, file_name), + 2000, + 2.0, + 3, + ) + .await + .map_err(|_| BatcherError::BatchUploadError("Error uploading batch to s3".to_string())) + } + + async fn upload_batch_to_s3( + &self, + batch_bytes: &[u8], + file_name: &String, + ) -> Result<(), RetryError<()>> { + let s3_client = self.s3_client.clone(); + + s3::upload_object( + &s3_client, + &self.s3_bucket_name, + batch_bytes.to_vec(), + &file_name, + ) + .await + .map_err(|_| RetryError::Transient)?; + Ok(()) + } } From 462defe3d0f251f42d153f0ffc3747444a3a751d Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 14:56:52 -0300 Subject: [PATCH 09/90] refactor: add retry to send_responses --- batcher/aligned-batcher/src/connection.rs | 50 +++++++++++++++++------ batcher/aligned-batcher/src/lib.rs | 5 +-- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index 501604788e..e108b3bbc2 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -1,6 +1,9 @@ use std::sync::Arc; -use crate::types::{batch_queue::BatchQueueEntry, errors::BatcherError}; +use crate::{ + retry::{retry_function, RetryError}, + types::{batch_queue::BatchQueueEntry, errors::BatcherError}, +}; use aligned_sdk::{ communication::serialization::cbor_serialize, core::types::{BatchInclusionData, ResponseMessage, VerificationCommitmentBatch}, @@ -32,17 +35,7 @@ pub(crate) async fn send_batch_inclusion_data_responses( return Err(BatcherError::WsSinkEmpty); }; - let sending_result = ws_sink - .write() - .await - .send(Message::binary(serialized_response)) - .await; - - match sending_result { - Err(Error::AlreadyClosed) => (), - Err(e) => error!("Error while sending batch inclusion data response: {}", e), - Ok(_) => (), - } + send_response_with_retry(ws_sink, serialized_response).await; info!("Response sent"); } @@ -65,3 +58,36 @@ pub(crate) async fn send_message(ws_conn_sink: WsMessageSink, mess Err(e) => error!("Error while serializing message: {}", e), } } + +async fn send_response_with_retry( + ws_sink: &Arc, Message>>>, + serialized_response: Vec, +) { + if let Err(e) = retry_function( + || send_response(ws_sink, serialized_response.clone()), + 2000, + 2.0, + 3, + ) + .await + { + error!("Error while sending batch inclusion data response: {}", e); + } +} + +async fn send_response( + ws_sink: &Arc, Message>>>, + serialized_response: Vec, +) -> Result<(), RetryError> { + let sending_result = ws_sink + .write() + .await + .send(Message::binary(serialized_response)) + .await; + + match sending_result { + Err(Error::AlreadyClosed) => Err(RetryError::Permanent(Error::AlreadyClosed)), + Err(_) => Err(RetryError::Transient), + Ok(_) => Ok(()), + } +} diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index e13994d9a2..7c8ceb96c6 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -2,7 +2,6 @@ use aligned_sdk::communication::serialization::{cbor_deserialize, cbor_serialize use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; -use ethers::contract::ContractError; use ethers::signers::Signer; use retry::{retry_function, RetryError}; use types::batch_state::BatchState; @@ -18,7 +17,7 @@ use aligned_sdk::core::types::{ VerificationCommitmentBatch, VerificationData, VerificationDataCommitment, }; use aws_sdk_s3::client::Client as S3Client; -use eth::{try_create_new_task, BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT}; +use eth::{try_create_new_task, BatcherPaymentService, CreateNewTaskFeeParams}; use ethers::prelude::{Middleware, Provider}; use ethers::providers::Ws; use ethers::types::{Address, Signature, TransactionReceipt, U256}; @@ -1265,7 +1264,7 @@ impl Batcher { }) } - /// Gets the current gas price from Ethereum using exponential backoff. + /// Uploads the batch to s3 using exponential backoff. async fn upload_batch_to_s3_with_retry( &self, batch_bytes: &[u8], From 03b598a7085a19766d98513aa51fb310ee7e65dc Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 15:23:35 -0300 Subject: [PATCH 10/90] refactor: use default constants --- batcher/aligned-batcher/src/connection.rs | 8 ++-- batcher/aligned-batcher/src/lib.rs | 50 +++++++++++++++++------ batcher/aligned-batcher/src/retry.rs | 8 +++- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index e108b3bbc2..284aa94072 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use crate::{ - retry::{retry_function, RetryError}, + retry::{retry_function, RetryError, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY}, types::{batch_queue::BatchQueueEntry, errors::BatcherError}, }; use aligned_sdk::{ @@ -65,9 +65,9 @@ async fn send_response_with_retry( ) { if let Err(e) = retry_function( || send_response(ws_sink, serialized_response.clone()), - 2000, - 2.0, - 3, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, ) .await { diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 7c8ceb96c6..d183a58f1e 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -3,7 +3,7 @@ use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use ethers::signers::Signer; -use retry::{retry_function, RetryError}; +use retry::{retry_function, RetryError, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY}; use types::batch_state::BatchState; use types::user_state::UserState; @@ -662,7 +662,13 @@ impl Batcher { &self, addr: Address, ) -> Result> { - retry_function(|| self.get_user_nonce_from_ethereum(addr), 2000, 2.0, 3).await + retry_function( + || self.get_user_nonce_from_ethereum(addr), + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await } async fn get_user_nonce_from_ethereum(&self, addr: Address) -> Result> { @@ -1189,7 +1195,13 @@ impl Batcher { /// Gets the balance of user with address `addr` from Ethereum using exponential backoff. async fn get_user_balance_with_retry(&self, addr: &Address) -> Result> { - retry_function(|| self.get_user_balance(addr), 2000, 2.0, 3).await + retry_function( + || self.get_user_balance(addr), + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await } async fn get_user_balance(&self, addr: &Address) -> Result> { @@ -1210,7 +1222,14 @@ impl Batcher { /// Checks if the user's balance is unlocked for a given address using exponential backoff. /// Returns `false` if an error occurs during the retries. async fn user_balance_is_unlocked_with_retry(&self, addr: &Address) -> bool { - match retry_function(|| self.user_balance_is_unlocked(addr), 2000, 2.0, 3).await { + match retry_function( + || self.user_balance_is_unlocked(addr), + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + { Ok(result) => result, Err(_) => { warn!("Could not get user locking state"); @@ -1237,12 +1256,17 @@ impl Batcher { /// Gets the current gas price from Ethereum using exponential backoff. /// Returns `None` if the gas price couldn't be returned async fn get_gas_price_with_retry(&self) -> Result { - retry_function(|| self.get_gas_price(), 2000, 2.0, 3) - .await - .map_err(|_| { - error!("Failed to get gas price"); - BatcherError::GasPriceError - }) + retry_function( + || self.get_gas_price(), + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .map_err(|_| { + error!("Failed to get gas price"); + BatcherError::GasPriceError + }) } async fn get_gas_price(&self) -> Result> { @@ -1272,9 +1296,9 @@ impl Batcher { ) -> Result<(), BatcherError> { retry_function( || self.upload_batch_to_s3(batch_bytes, file_name), - 2000, - 2.0, - 3, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, ) .await .map_err(|_| BatcherError::BatchUploadError("Error uploading batch to s3".to_string())) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 9fa5374b19..32aa0eb3d0 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -2,6 +2,10 @@ use backon::ExponentialBuilder; use backon::Retryable; use std::{future::Future, time::Duration}; +pub const DEFAULT_MIN_DELAY: u64 = 2000; +pub const DEFAULT_MAX_TIMES: usize = 3; +pub const DEFAULT_FACTOR: f32 = 2.0; + #[derive(Debug)] pub enum RetryError { Transient, @@ -19,7 +23,7 @@ pub async fn retry_function( function: FutureFn, min_delay: u64, factor: f32, - max_tries: usize, + max_times: usize, ) -> Result> where Fut: Future>>, @@ -27,7 +31,7 @@ where { let backoff = ExponentialBuilder::default() .with_min_delay(Duration::from_millis(min_delay)) - .with_max_times(max_tries) + .with_max_times(max_times) .with_factor(factor); function From b91e10dc3c6b634fe63857f741938e28fb9fbf4c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 15:41:46 -0300 Subject: [PATCH 11/90] fix: cargo clippy --- batcher/aligned-batcher/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index d183a58f1e..524d27a48f 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1292,7 +1292,7 @@ impl Batcher { async fn upload_batch_to_s3_with_retry( &self, batch_bytes: &[u8], - file_name: &String, + file_name: &str, ) -> Result<(), BatcherError> { retry_function( || self.upload_batch_to_s3(batch_bytes, file_name), @@ -1307,7 +1307,7 @@ impl Batcher { async fn upload_batch_to_s3( &self, batch_bytes: &[u8], - file_name: &String, + file_name: &str, ) -> Result<(), RetryError<()>> { let s3_client = self.s3_client.clone(); @@ -1315,7 +1315,7 @@ impl Batcher { &s3_client, &self.s3_bucket_name, batch_bytes.to_vec(), - &file_name, + file_name, ) .await .map_err(|_| RetryError::Transient)?; From 5601583edb92d2d9d13af89ce2388ea59d4f506e Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 18:05:50 -0300 Subject: [PATCH 12/90] refactor: remove reconnections --- batcher/aligned-batcher/src/eth/mod.rs | 13 +++---------- batcher/aligned-batcher/src/lib.rs | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/mod.rs b/batcher/aligned-batcher/src/eth/mod.rs index f50e14ecbe..ab77a75272 100644 --- a/batcher/aligned-batcher/src/eth/mod.rs +++ b/batcher/aligned-batcher/src/eth/mod.rs @@ -15,7 +15,7 @@ pub struct BatchVerified { } pub type SignerMiddlewareT = - SignerMiddleware>>, Wallet>; + SignerMiddleware>, Wallet>; pub type BatcherPaymentService = BatcherPaymentServiceContract; @@ -48,18 +48,11 @@ impl CreateNewTaskFeeParams { } } -pub fn get_provider(eth_rpc_url: String) -> Result>, anyhow::Error> { +pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error> { let provider = Http::from_str(eth_rpc_url.as_str()) .map_err(|e| anyhow::Error::msg(format!("Failed to create provider: {}", e)))?; - let client = RetryClient::new( - provider, - Box::::default(), - MAX_RETRIES, - INITIAL_BACKOFF, - ); - - Ok(Provider::>::new(client)) + Ok(Provider::new(provider)) } pub async fn get_batcher_payment_service( diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 524d27a48f..1fc48344a8 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -98,10 +98,9 @@ impl Batcher { let deployment_output = ContractDeploymentOutput::new(config.aligned_layer_deployment_config_file_path); - let eth_ws_provider = - Provider::connect_with_reconnects(&config.eth_ws_url, config.batcher.eth_ws_reconnects) - .await - .expect("Failed to get ethereum websocket provider"); + let eth_ws_provider = Provider::connect(&config.eth_ws_url) + .await + .expect("Failed to get ethereum websocket provider"); log::info!( "Starting metrics server on port {}", @@ -110,12 +109,9 @@ impl Batcher { let metrics = metrics::BatcherMetrics::start(config.batcher.metrics_port) .expect("Failed to start metrics server"); - let eth_ws_provider_fallback = Provider::connect_with_reconnects( - &config.eth_ws_url_fallback, - config.batcher.eth_ws_reconnects, - ) - .await - .expect("Failed to get fallback ethereum websocket provider"); + let eth_ws_provider_fallback = Provider::connect(&config.eth_ws_url_fallback) + .await + .expect("Failed to get fallback ethereum websocket provider"); let eth_rpc_provider = eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get provider"); @@ -1318,7 +1314,10 @@ impl Batcher { file_name, ) .await - .map_err(|_| RetryError::Transient)?; + .map_err(|e| { + warn!("Error uploading batch to s3 {e}"); + RetryError::Transient + })?; Ok(()) } } From 47e3516ac66507c960096c0ffc6d551d048b7a28 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 18:09:41 -0300 Subject: [PATCH 13/90] fix: retry client refactor --- batcher/aligned-batcher/src/eth/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/mod.rs b/batcher/aligned-batcher/src/eth/mod.rs index ab77a75272..19895535af 100644 --- a/batcher/aligned-batcher/src/eth/mod.rs +++ b/batcher/aligned-batcher/src/eth/mod.rs @@ -19,8 +19,6 @@ pub type SignerMiddlewareT = pub type BatcherPaymentService = BatcherPaymentServiceContract; -const MAX_RETRIES: u32 = 15; // Max retries for the retry client. Will only retry on network errors -const INITIAL_BACKOFF: u64 = 1000; // Initial backoff for the retry client in milliseconds, will increase every retry const GAS_MULTIPLIER: f64 = 1.125; // Multiplier for the gas price for gas escalator const GAS_ESCALATOR_INTERVAL: u64 = 12; // Time in seconds between gas escalations @@ -56,7 +54,7 @@ pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error } pub async fn get_batcher_payment_service( - provider: Provider>, + provider: Provider, ecdsa_config: ECDSAConfig, contract_address: String, ) -> Result { From b0853f7c42374a3e5ba5a6d7180e22242bb994e2 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 18:30:49 -0300 Subject: [PATCH 14/90] refactor: update config and docs --- batcher/aligned-batcher/src/config/mod.rs | 1 - config-files/config-batcher.yaml | 1 - config-files/config.yaml | 1 - docs/3_guides/6_setup_aligned.md | 1 - 4 files changed, 4 deletions(-) diff --git a/batcher/aligned-batcher/src/config/mod.rs b/batcher/aligned-batcher/src/config/mod.rs index c39fd22cf8..44ed35c871 100644 --- a/batcher/aligned-batcher/src/config/mod.rs +++ b/batcher/aligned-batcher/src/config/mod.rs @@ -40,7 +40,6 @@ pub struct BatcherConfigFromYaml { pub block_interval: u64, pub max_proof_size: usize, pub max_batch_size: usize, - pub eth_ws_reconnects: usize, pub pre_verification_is_enabled: bool, pub metrics_port: u16, pub non_paying: Option, diff --git a/config-files/config-batcher.yaml b/config-files/config-batcher.yaml index 6a6273e026..7d0d96160b 100644 --- a/config-files/config-batcher.yaml +++ b/config-files/config-batcher.yaml @@ -20,7 +20,6 @@ batcher: batch_size_interval: 10 max_proof_size: 67108864 # 64 MiB max_batch_size: 268435456 # 256 MiB - eth_ws_reconnects: 99999999999999 pre_verification_is_enabled: true metrics_port: 9093 non_paying: diff --git a/config-files/config.yaml b/config-files/config.yaml index 0af66f176a..e4af30d9bd 100644 --- a/config-files/config.yaml +++ b/config-files/config.yaml @@ -25,7 +25,6 @@ batcher: batch_size_interval: 10 max_proof_size: 67108864 # 64 MiB max_batch_size: 268435456 # 256 MiB - eth_ws_reconnects: 99999999999999 pre_verification_is_enabled: true metrics_port: 9093 diff --git a/docs/3_guides/6_setup_aligned.md b/docs/3_guides/6_setup_aligned.md index a7086b0d54..f45dc6e5ed 100644 --- a/docs/3_guides/6_setup_aligned.md +++ b/docs/3_guides/6_setup_aligned.md @@ -356,7 +356,6 @@ batcher: batch_size_interval: max_proof_size: max_batch_size: - eth_ws_reconnects: pre_verification_is_enabled: ## ECDSA Configurations From 2e575f6c3231fc129816e1dbf017e16104152a2e Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 17 Oct 2024 18:42:54 -0300 Subject: [PATCH 15/90] refactor: use ethers instead of our forked version --- batcher/Cargo.lock | 243 +++--------------- batcher/aligned-batcher/Cargo.toml | 5 +- batcher/aligned-sdk/Cargo.toml | 6 +- batcher/aligned/Cargo.toml | 5 +- .../aligned-integration/Cargo.toml | 5 +- examples/zkquiz/quiz/script/Cargo.toml | 6 +- 6 files changed, 36 insertions(+), 234 deletions(-) diff --git a/batcher/Cargo.lock b/batcher/Cargo.lock index da0b5a014e..eea93b8efc 100644 --- a/batcher/Cargo.lock +++ b/batcher/Cargo.lock @@ -83,7 +83,7 @@ dependencies = [ "aligned-sdk", "clap", "env_logger", - "ethers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers", "futures-util", "lambdaworks-crypto", "log", @@ -111,7 +111,7 @@ dependencies = [ "clap", "dotenvy", "env_logger", - "ethers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers", "futures-util", "hex", "lambdaworks-crypto", @@ -136,7 +136,7 @@ version = "0.1.0" dependencies = [ "ciborium", "dialoguer", - "ethers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers", "futures-util", "hex", "lambdaworks-crypto", @@ -2206,26 +2206,13 @@ version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "816841ea989f0c69e459af1cf23a6b0033b19a55424a1ea3a30099becdb8dec0" dependencies = [ - "ethers-addressbook 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-contract 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-middleware 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-providers 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-signers 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ethers" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "ethers-addressbook 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-contract 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-addressbook", + "ethers-contract", + "ethers-core", "ethers-etherscan", - "ethers-middleware 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-providers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-signers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-middleware", + "ethers-providers", + "ethers-signers", "ethers-solc", ] @@ -2235,18 +2222,7 @@ version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5495afd16b4faa556c3bba1f21b98b4983e53c1755022377051a975c3b021759" dependencies = [ - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "once_cell", - "serde", - "serde_json", -] - -[[package]] -name = "ethers-addressbook" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-core", "once_cell", "serde", "serde_json", @@ -2259,28 +2235,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fceafa3578c836eeb874af87abacfb041f92b4da0a78a5edd042564b8ecdaaa" dependencies = [ "const-hex", - "ethers-contract-abigen 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-contract-derive 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-providers 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-util", - "once_cell", - "pin-project", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "ethers-contract" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "const-hex", - "ethers-contract-abigen 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-contract-derive 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-providers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-contract-abigen", + "ethers-contract-derive", + "ethers-core", + "ethers-providers", "futures-util", "once_cell", "pin-project", @@ -2298,28 +2256,7 @@ dependencies = [ "Inflector", "const-hex", "dunce", - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "eyre", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "serde", - "serde_json", - "syn 2.0.77", - "toml", - "walkdir", -] - -[[package]] -name = "ethers-contract-abigen" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "Inflector", - "const-hex", - "dunce", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-core", "ethers-etherscan", "eyre", "prettyplease", @@ -2342,23 +2279,8 @@ checksum = "87689dcabc0051cde10caaade298f9e9093d65f6125c14575db3fd8c669a168f" dependencies = [ "Inflector", "const-hex", - "ethers-contract-abigen 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2", - "quote", - "serde_json", - "syn 2.0.77", -] - -[[package]] -name = "ethers-contract-derive" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "Inflector", - "const-hex", - "ethers-contract-abigen 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-contract-abigen", + "ethers-core", "proc-macro2", "quote", "serde_json", @@ -2395,42 +2317,14 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "ethers-core" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "arrayvec", - "bytes", - "cargo_metadata", - "chrono", - "const-hex", - "elliptic-curve 0.13.8", - "ethabi", - "generic-array 0.14.7", - "k256", - "num_enum 0.7.2", - "once_cell", - "open-fastrlp", - "rand", - "rlp", - "serde", - "serde_json", - "strum", - "syn 2.0.77", - "tempfile", - "thiserror", - "tiny-keccak", - "unicode-xid", -] - [[package]] name = "ethers-etherscan" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79e5973c26d4baf0ce55520bd732314328cabe53193286671b47144145b9649" dependencies = [ "chrono", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-core", "reqwest 0.11.27", "semver 1.0.23", "serde", @@ -2447,36 +2341,11 @@ checksum = "48f9fdf09aec667c099909d91908d5eaf9be1bd0e2500ba4172c1d28bfaa43de" dependencies = [ "async-trait", "auto_impl", - "ethers-contract 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-providers 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "ethers-signers 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-channel", - "futures-locks", - "futures-util", - "instant", - "reqwest 0.11.27", - "serde", - "serde_json", - "thiserror", - "tokio", - "tracing", - "tracing-futures", - "url", -] - -[[package]] -name = "ethers-middleware" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "async-trait", - "auto_impl", - "ethers-contract 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-contract", + "ethers-core", "ethers-etherscan", - "ethers-providers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", - "ethers-signers 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-providers", + "ethers-signers", "futures-channel", "futures-locks", "futures-util", @@ -2503,42 +2372,7 @@ dependencies = [ "bytes", "const-hex", "enr", - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-core", - "futures-timer", - "futures-util", - "hashers", - "http 0.2.12", - "instant", - "jsonwebtoken", - "once_cell", - "pin-project", - "reqwest 0.11.27", - "serde", - "serde_json", - "thiserror", - "tokio", - "tracing", - "tracing-futures", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "ws_stream_wasm", -] - -[[package]] -name = "ethers-providers" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "async-trait", - "auto_impl", - "base64 0.22.1", - "bytes", - "const-hex", - "enr", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-core", "futures-channel", "futures-core", "futures-timer", @@ -2576,25 +2410,7 @@ dependencies = [ "const-hex", "elliptic-curve 0.13.8", "eth-keystore", - "ethers-core 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rand", - "sha2", - "thiserror", - "tracing", -] - -[[package]] -name = "ethers-signers" -version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" -dependencies = [ - "async-trait", - "coins-bip32", - "coins-bip39", - "const-hex", - "elliptic-curve 0.13.8", - "eth-keystore", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-core", "rand", "sha2", "thiserror", @@ -2604,13 +2420,14 @@ dependencies = [ [[package]] name = "ethers-solc" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66244a771d9163282646dbeffe0e6eca4dda4146b6498644e678ac6089b11edd" dependencies = [ "cfg-if", "const-hex", "dirs", "dunce", - "ethers-core 2.0.14 (git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections)", + "ethers-core", "glob", "home", "md-5", @@ -6536,7 +6353,7 @@ dependencies = [ "bincode", "cfg-if", "dirs", - "ethers 2.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "ethers", "futures", "hashbrown 0.14.5", "hex", diff --git a/batcher/aligned-batcher/Cargo.toml b/batcher/aligned-batcher/Cargo.toml index 2f9e8df4a4..b43ccb66b8 100644 --- a/batcher/aligned-batcher/Cargo.toml +++ b/batcher/aligned-batcher/Cargo.toml @@ -19,10 +19,7 @@ bytes = "1.7.1" hex = "0.4.3" dotenvy = "0.15.0" anyhow = "1.0.83" -ethers = { tag = "v2.0.15-fix-reconnections", features = [ - "ws", - "rustls", -], git = "https://github.com/yetanotherco/ethers-rs.git" } +ethers = { version = "2.0", features = ["ws", "rustls"] } lambdaworks-crypto = { git = "https://github.com/lambdaclass/lambdaworks.git", rev = "efd46f0b0aea3aa95d94bba7de86cb96611b40d3", features = ["serde"] } serde_yaml = "0.9.34" sp1-sdk = { git = "https://github.com/succinctlabs/sp1.git", rev = "v1.0.1" } diff --git a/batcher/aligned-sdk/Cargo.toml b/batcher/aligned-sdk/Cargo.toml index 3704c6072a..019a659842 100644 --- a/batcher/aligned-sdk/Cargo.toml +++ b/batcher/aligned-sdk/Cargo.toml @@ -4,11 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -ethers = { tag = "v2.0.15-fix-reconnections", features = [ - "ws", - "rustls", - "eip712", -], git = "https://github.com/yetanotherco/ethers-rs.git" } +ethers = { version = "2.0", features = ["ws", "rustls", "eip712"] } log = { version = "0.4.21" } serde_json = "1.0.117" tokio-tungstenite = { version = "0.23.1", features = ["native-tls"] } diff --git a/batcher/aligned/Cargo.toml b/batcher/aligned/Cargo.toml index e18d05f27a..0d5aaf2087 100644 --- a/batcher/aligned/Cargo.toml +++ b/batcher/aligned/Cargo.toml @@ -24,10 +24,7 @@ log = "0.4.21" env_logger = "0.11.3" clap = { version = "4.5.4", features = ["derive"] } lambdaworks-crypto = { git = "https://github.com/lambdaclass/lambdaworks.git", rev = "efd46f0b0aea3aa95d94bba7de86cb96611b40d3", features = ["serde"] } -ethers = { tag = "v2.0.15-fix-reconnections", features = [ - "ws", - "rustls", -], git = "https://github.com/yetanotherco/ethers-rs.git" } +ethers = { version = "2.0", features = ["ws", "rustls"] } aligned-sdk = { path = "../aligned-sdk" } rpassword = "7.3.1" sha3 = { version = "0.10.8" } diff --git a/examples/validating-public-input/aligned-integration/Cargo.toml b/examples/validating-public-input/aligned-integration/Cargo.toml index 2763a8924f..b95940c4bb 100644 --- a/examples/validating-public-input/aligned-integration/Cargo.toml +++ b/examples/validating-public-input/aligned-integration/Cargo.toml @@ -13,10 +13,7 @@ tokio = { version = "1.37.0", features = [ "rt-multi-thread", "sync", ] } -ethers = { tag = "v2.0.15-fix-reconnections", features = [ - "ws", - "rustls", -], git = "https://github.com/yetanotherco/ethers-rs.git" } +ethers = { version = "2.0", features = ["ws", "rustls"] } serde = { version = "1.0.201", features = ["derive"] } serde_json = "1.0.117" log = "0.4.21" diff --git a/examples/zkquiz/quiz/script/Cargo.toml b/examples/zkquiz/quiz/script/Cargo.toml index b6feda3522..62c54cf035 100644 --- a/examples/zkquiz/quiz/script/Cargo.toml +++ b/examples/zkquiz/quiz/script/Cargo.toml @@ -7,10 +7,8 @@ edition = "2021" [dependencies] sp1-sdk = { git = "https://github.com/succinctlabs/sp1.git", rev = "v1.0.1" } aligned-sdk = { path = "../../../../batcher/aligned-sdk" } -ethers = { tag = "v2.0.15-fix-reconnections", features = [ - "ws", - "rustls", -], git = "https://github.com/yetanotherco/ethers-rs.git" } +ethers = { version = "2.0", features = ["ws", "rustls"] } + rpassword = "7.3.1" tokio = { version = "1.0.0", features = ["rt", "rt-multi-thread", "macros"] } bincode = "1.3.3" From 1c60221682b40d071aa89f477509e3cc6d7be7d5 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Fri, 18 Oct 2024 10:17:08 -0300 Subject: [PATCH 16/90] refactor: add retry to stream.next() --- batcher/aligned-batcher/src/lib.rs | 68 +++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 1fc48344a8..9eb38c2149 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -19,8 +19,8 @@ use aligned_sdk::core::types::{ use aws_sdk_s3::client::Client as S3Client; use eth::{try_create_new_task, BatcherPaymentService, CreateNewTaskFeeParams}; use ethers::prelude::{Middleware, Provider}; -use ethers::providers::Ws; -use ethers::types::{Address, Signature, TransactionReceipt, U256}; +use ethers::providers::{SubscriptionStream, Ws}; +use ethers::types::{Address, Block, Signature, TransactionReceipt, TxHash, H256, U256}; use futures_util::{future, SinkExt, StreamExt, TryStreamExt}; use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use lambdaworks_crypto::merkle_tree::traits::IsMerkleTreeBackend; @@ -227,13 +227,13 @@ impl Batcher { } pub async fn listen_new_blocks(self: Arc) -> Result<(), BatcherError> { - let mut stream = self + let stream = self .eth_ws_provider .subscribe_blocks() .await .map_err(|e| BatcherError::EthereumSubscriptionError(e.to_string()))?; - let mut stream_fallback = self + let stream_fallback = self .eth_ws_provider_fallback .subscribe_blocks() .await @@ -241,10 +241,13 @@ impl Batcher { let last_seen_block = Mutex::::new(0); - while let Some(block) = tokio::select! { - block = stream.next() => block, - block = stream_fallback.next() => block, - } { + let stream = Arc::new(Mutex::new(stream)); + let stream_fallback = Arc::new(Mutex::new(stream_fallback)); + + while let Ok(block) = self + .get_next_block_with_retry(stream.clone(), stream_fallback.clone()) + .await + { let batcher = self.clone(); let block_number = block.number.unwrap_or_default(); let block_number = u64::try_from(block_number).unwrap_or_default(); @@ -265,7 +268,48 @@ impl Batcher { }); } - Ok(()) + Err(BatcherError::EthereumSubscriptionError( + "No more blocks".to_string(), + )) + } + + async fn get_next_block_with_retry( + &self, + stream: Arc>>>, + stream_fallback: Arc>>>, + ) -> Result, BatcherError> { + retry_function( + || { + let stream = Arc::clone(&stream); + let stream_fallback = Arc::clone(&stream_fallback); + async move { + let mut stream_lock = stream.lock().await; + let mut stream_fallback_lock = stream_fallback.lock().await; + self.get_next_block(&mut stream_lock, &mut stream_fallback_lock) + .await + } + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .map_err(|_| BatcherError::EthereumSubscriptionError("No more blocks".to_string())) + } + + async fn get_next_block( + &self, + stream: &mut SubscriptionStream<'_, Ws, Block>, + stream_fallback: &mut SubscriptionStream<'_, Ws, Block>, + ) -> Result, RetryError<()>> { + tokio::select! { + Some(block) = stream.next() => Ok(block), + Some(block) = stream_fallback.next() => Ok(block), + else => { + warn!("Error fetching next block."); + Err(RetryError::Transient) + } + } } async fn handle_connection( @@ -675,7 +719,10 @@ impl Batcher { .user_nonces(addr) .call() .await - .map_err(|_| RetryError::Transient) + .map_err(|_| { + warn!("Error getting user nonce."); + RetryError::Transient + }) } /// Adds verification data to the current batch queue. @@ -1246,6 +1293,7 @@ impl Batcher { { return Ok(unlock_block != U256::zero()); } + warn!("Failed to get user locking state {:?}", addr); Err(RetryError::Transient) } From fe666db829547085abd5748c875a7f78957c95e1 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 18 Oct 2024 12:55:35 -0300 Subject: [PATCH 17/90] Add test with local anvil instance --- batcher/aligned-batcher/src/retry.rs | 42 ++++++++++++++++++++++++++-- batcher/aligned/src/main.rs | 2 -- operator/merkle_tree/lib/Cargo.lock | 41 +++++++++++++++------------ 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 32aa0eb3d0..7b4ef21efb 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -45,8 +45,20 @@ where mod test { use super::*; use crate::eth; - use ethers::{providers::Middleware, types::U256}; - use std::time::SystemTime; + use aligned_sdk::eth::batcher_payment_service::BatcherPaymentService; + use ethers::{ + contract::abigen, + providers::{Http, Middleware, Provider}, + types::{Address, U256}, + utils::Anvil, + }; + use std::str::FromStr; + use std::{sync::Arc, time::SystemTime}; + + abigen!( + BatcherPaymentServiceContract, + "../aligned-sdk/abi/BatcherPaymentService.json" + ); #[tokio::test] async fn retry_test() { @@ -81,4 +93,30 @@ mod test { assert!(retry_function(get_gas_price, 2000, 2.0, 3).await.is_ok()); } + + #[tokio::test] + async fn test_anvil() { + let _anvil = Anvil::new() + .port(8545u16) + .arg("--load-state") + .arg("../../contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json") + .spawn(); + + let eth_rpc_provider: Provider = + eth::get_provider(String::from("http://localhost:8545")) + .expect("Failed to get provider"); + + let payment_service_addr = + Address::from_str("0x7969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + + let payment_service = + BatcherPaymentService::new(payment_service_addr, Arc::new(eth_rpc_provider)); + + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + + if let Ok(balance) = payment_service.user_balances(dummy_user_addr).call().await { + println!("ALIGNED USER BALANCE: {:?}", balance) + }; + } } diff --git a/batcher/aligned/src/main.rs b/batcher/aligned/src/main.rs index d1ffa511d0..09e76146ff 100644 --- a/batcher/aligned/src/main.rs +++ b/batcher/aligned/src/main.rs @@ -300,7 +300,6 @@ async fn main() -> Result<(), AlignedError> { let chain_id = get_chain_id(eth_rpc_url.as_str()).await?; wallet = wallet.with_chain_id(chain_id); - let nonce = match &submit_args.nonce { Some(nonce) => U256::from_dec_str(nonce).map_err(|_| SubmitError::InvalidNonce)?, None => { @@ -342,7 +341,6 @@ async fn main() -> Result<(), AlignedError> { }; let mut unique_batch_merkle_roots = HashSet::new(); - for aligned_verification_data in aligned_verification_data_vec { save_response( batch_inclusion_data_directory_path.clone(), diff --git a/operator/merkle_tree/lib/Cargo.lock b/operator/merkle_tree/lib/Cargo.lock index 6833d25727..c07c0a2630 100644 --- a/operator/merkle_tree/lib/Cargo.lock +++ b/operator/merkle_tree/lib/Cargo.lock @@ -154,12 +154,6 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "base64ct" version = "1.6.0" @@ -847,7 +841,8 @@ dependencies = [ [[package]] name = "ethers" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816841ea989f0c69e459af1cf23a6b0033b19a55424a1ea3a30099becdb8dec0" dependencies = [ "ethers-addressbook", "ethers-contract", @@ -862,7 +857,8 @@ dependencies = [ [[package]] name = "ethers-addressbook" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5495afd16b4faa556c3bba1f21b98b4983e53c1755022377051a975c3b021759" dependencies = [ "ethers-core", "once_cell", @@ -873,7 +869,8 @@ dependencies = [ [[package]] name = "ethers-contract" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fceafa3578c836eeb874af87abacfb041f92b4da0a78a5edd042564b8ecdaaa" dependencies = [ "const-hex", "ethers-contract-abigen", @@ -891,7 +888,8 @@ dependencies = [ [[package]] name = "ethers-contract-abigen" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ba01fbc2331a38c429eb95d4a570166781f14290ef9fdb144278a90b5a739b" dependencies = [ "Inflector", "const-hex", @@ -914,7 +912,8 @@ dependencies = [ [[package]] name = "ethers-contract-derive" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87689dcabc0051cde10caaade298f9e9093d65f6125c14575db3fd8c669a168f" dependencies = [ "Inflector", "const-hex", @@ -929,7 +928,8 @@ dependencies = [ [[package]] name = "ethers-core" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d80cc6ad30b14a48ab786523af33b37f28a8623fc06afd55324816ef18fb1f" dependencies = [ "arrayvec", "bytes", @@ -958,7 +958,8 @@ dependencies = [ [[package]] name = "ethers-etherscan" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79e5973c26d4baf0ce55520bd732314328cabe53193286671b47144145b9649" dependencies = [ "chrono", "ethers-core", @@ -973,7 +974,8 @@ dependencies = [ [[package]] name = "ethers-middleware" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f9fdf09aec667c099909d91908d5eaf9be1bd0e2500ba4172c1d28bfaa43de" dependencies = [ "async-trait", "auto_impl", @@ -999,11 +1001,12 @@ dependencies = [ [[package]] name = "ethers-providers" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6434c9a33891f1effc9c75472e12666db2fa5a0fec4b29af6221680a6fe83ab2" dependencies = [ "async-trait", "auto_impl", - "base64 0.22.1", + "base64 0.21.7", "bytes", "const-hex", "enr", @@ -1036,7 +1039,8 @@ dependencies = [ [[package]] name = "ethers-signers" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228875491c782ad851773b652dd8ecac62cda8571d3bc32a5853644dd26766c2" dependencies = [ "async-trait", "coins-bip32", @@ -1054,7 +1058,8 @@ dependencies = [ [[package]] name = "ethers-solc" version = "2.0.14" -source = "git+https://github.com/yetanotherco/ethers-rs.git?tag=v2.0.15-fix-reconnections#69bba841ff352cf27b014d4fbb7985a180d88e25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66244a771d9163282646dbeffe0e6eca4dda4146b6498644e678ac6089b11edd" dependencies = [ "cfg-if", "const-hex", From 4f01fd71e6167a47978969ddf686964d6c90caaa Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Fri, 18 Oct 2024 13:10:27 -0300 Subject: [PATCH 18/90] refactor: change retryable methods to functions --- batcher/aligned-batcher/src/connection.rs | 8 +- batcher/aligned-batcher/src/lib.rs | 149 ++++++++++++---------- 2 files changed, 89 insertions(+), 68 deletions(-) diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index 284aa94072..b1ba79a589 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -35,7 +35,7 @@ pub(crate) async fn send_batch_inclusion_data_responses( return Err(BatcherError::WsSinkEmpty); }; - send_response_with_retry(ws_sink, serialized_response).await; + send_response(ws_sink, serialized_response).await; info!("Response sent"); } @@ -59,12 +59,12 @@ pub(crate) async fn send_message(ws_conn_sink: WsMessageSink, mess } } -async fn send_response_with_retry( +async fn send_response( ws_sink: &Arc, Message>>>, serialized_response: Vec, ) { if let Err(e) = retry_function( - || send_response(ws_sink, serialized_response.clone()), + || send_response_retryable(ws_sink, serialized_response.clone()), DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, @@ -75,7 +75,7 @@ async fn send_response_with_retry( } } -async fn send_response( +async fn send_response_retryable( ws_sink: &Arc, Message>>>, serialized_response: Vec, ) -> Result<(), RetryError> { diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 9eb38c2149..31588a7123 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -245,7 +245,7 @@ impl Batcher { let stream_fallback = Arc::new(Mutex::new(stream_fallback)); while let Ok(block) = self - .get_next_block_with_retry(stream.clone(), stream_fallback.clone()) + .get_next_block(stream.clone(), stream_fallback.clone()) .await { let batcher = self.clone(); @@ -273,7 +273,7 @@ impl Batcher { )) } - async fn get_next_block_with_retry( + async fn get_next_block( &self, stream: Arc>>>, stream_fallback: Arc>>>, @@ -285,7 +285,7 @@ impl Batcher { async move { let mut stream_lock = stream.lock().await; let mut stream_fallback_lock = stream_fallback.lock().await; - self.get_next_block(&mut stream_lock, &mut stream_fallback_lock) + Self::get_next_block_retryable(&mut stream_lock, &mut stream_fallback_lock) .await } }, @@ -297,8 +297,7 @@ impl Batcher { .map_err(|_| BatcherError::EthereumSubscriptionError("No more blocks".to_string())) } - async fn get_next_block( - &self, + async fn get_next_block_retryable( stream: &mut SubscriptionStream<'_, Ws, Block>, stream_fallback: &mut SubscriptionStream<'_, Ws, Block>, ) -> Result, RetryError<()>> { @@ -443,7 +442,7 @@ impl Batcher { // We don't need a batch state lock here, since if the user locks its funds // after the check, some blocks should pass until he can withdraw. // It is safe to do just do this here. - if self.user_balance_is_unlocked_with_retry(&addr).await { + if self.user_balance_is_unlocked(&addr).await { send_message( ws_conn_sink.clone(), ValidityResponseMessage::InsufficientBalance(addr), @@ -473,8 +472,7 @@ impl Batcher { } if !is_user_in_state { - let ethereum_user_nonce = match self.get_user_nonce_from_ethereum_with_retry(addr).await - { + let ethereum_user_nonce = match self.get_user_nonce_from_ethereum(addr).await { Ok(ethereum_user_nonce) => ethereum_user_nonce, Err(e) => { error!( @@ -496,7 +494,7 @@ impl Batcher { // * Perform validations over user state * // * ---------------------------------------------------* - let Ok(user_balance) = self.get_user_balance_with_retry(&addr).await else { + let Ok(user_balance) = self.get_user_balance(&addr).await else { error!("Could not get balance for address {addr:?}"); send_message(ws_conn_sink.clone(), ValidityResponseMessage::EthRpcError).await; return Ok(()); @@ -698,12 +696,15 @@ impl Batcher { } /// Gets the user nonce from Ethereum using exponential backoff. - async fn get_user_nonce_from_ethereum_with_retry( - &self, - addr: Address, - ) -> Result> { + async fn get_user_nonce_from_ethereum(&self, addr: Address) -> Result> { retry_function( - || self.get_user_nonce_from_ethereum(addr), + || { + Self::get_user_nonce_from_ethereum_retryable( + &self.payment_service, + &self.payment_service_fallback, + addr, + ) + }, DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, @@ -711,11 +712,15 @@ impl Batcher { .await } - async fn get_user_nonce_from_ethereum(&self, addr: Address) -> Result> { - if let Ok(nonce) = self.payment_service.user_nonces(addr).call().await { + async fn get_user_nonce_from_ethereum_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: Address, + ) -> Result> { + if let Ok(nonce) = payment_service.user_nonces(addr).call().await { return Ok(nonce); } - self.payment_service_fallback + payment_service_fallback .user_nonces(addr) .call() .await @@ -992,7 +997,7 @@ impl Batcher { // so that it is already loaded let Ok(nonpaying_replacement_addr_nonce) = self - .get_user_nonce_from_ethereum_with_retry(nonpaying_replacement_addr) + .get_user_nonce_from_ethereum(nonpaying_replacement_addr) .await else { batch_state_lock.batch_queue.clear(); @@ -1010,7 +1015,7 @@ impl Batcher { /// Receives new block numbers, checks if conditions are met for submission and /// finalizes the batch. async fn handle_new_block(&self, block_number: u64) -> Result<(), BatcherError> { - let gas_price = self.get_gas_price_with_retry().await?; + let gas_price = self.get_gas_price().await?; if let Some(finalized_batch) = self.is_batch_ready(block_number, gas_price).await { let batch_finalization_result = self @@ -1041,8 +1046,7 @@ impl Batcher { let file_name = batch_merkle_root_hex.clone() + ".json"; info!("Uploading batch to S3..."); - self.upload_batch_to_s3_with_retry(batch_bytes, &file_name) - .await?; + self.upload_batch_to_s3(batch_bytes, &file_name).await?; info!("Batch sent to S3 with name: {}", file_name); @@ -1168,9 +1172,7 @@ impl Batcher { }; let replacement_addr = non_paying_config.replacement.address(); - let Ok(replacement_user_balance) = - self.get_user_balance_with_retry(&replacement_addr).await - else { + let Ok(replacement_user_balance) = self.get_user_balance(&replacement_addr).await else { error!("Could not get balance for non-paying address {replacement_addr:?}"); send_message( ws_sink.clone(), @@ -1237,9 +1239,15 @@ impl Batcher { } /// Gets the balance of user with address `addr` from Ethereum using exponential backoff. - async fn get_user_balance_with_retry(&self, addr: &Address) -> Result> { + async fn get_user_balance(&self, addr: &Address) -> Result> { retry_function( - || self.get_user_balance(addr), + || { + Self::get_user_balance_retryable( + &self.payment_service, + &self.payment_service_fallback, + addr, + ) + }, DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, @@ -1247,12 +1255,16 @@ impl Batcher { .await } - async fn get_user_balance(&self, addr: &Address) -> Result> { - if let Ok(balance) = self.payment_service.user_balances(*addr).call().await { + async fn get_user_balance_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: &Address, + ) -> Result> { + if let Ok(balance) = payment_service.user_balances(*addr).call().await { return Ok(balance); }; - self.payment_service_fallback + payment_service_fallback .user_balances(*addr) .call() .await @@ -1264,9 +1276,15 @@ impl Batcher { /// Checks if the user's balance is unlocked for a given address using exponential backoff. /// Returns `false` if an error occurs during the retries. - async fn user_balance_is_unlocked_with_retry(&self, addr: &Address) -> bool { + async fn user_balance_is_unlocked(&self, addr: &Address) -> bool { match retry_function( - || self.user_balance_is_unlocked(addr), + || { + Self::user_balance_is_unlocked_retryable( + &self.payment_service, + &self.payment_service_fallback, + addr, + ) + }, DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, @@ -1281,12 +1299,15 @@ impl Batcher { } } - async fn user_balance_is_unlocked(&self, addr: &Address) -> Result> { - if let Ok(unlock_block) = self.payment_service.user_unlock_block(*addr).call().await { + async fn user_balance_is_unlocked_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: &Address, + ) -> Result> { + if let Ok(unlock_block) = payment_service.user_unlock_block(*addr).call().await { return Ok(unlock_block != U256::zero()); } - if let Ok(unlock_block) = self - .payment_service_fallback + if let Ok(unlock_block) = payment_service_fallback .user_unlock_block(*addr) .call() .await @@ -1299,9 +1320,9 @@ impl Batcher { /// Gets the current gas price from Ethereum using exponential backoff. /// Returns `None` if the gas price couldn't be returned - async fn get_gas_price_with_retry(&self) -> Result { + async fn get_gas_price(&self) -> Result { retry_function( - || self.get_gas_price(), + || Self::get_gas_price_retryable(&self.eth_ws_provider, &self.eth_ws_provider_fallback), DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, @@ -1313,9 +1334,11 @@ impl Batcher { }) } - async fn get_gas_price(&self) -> Result> { - if let Ok(gas_price) = self - .eth_ws_provider + async fn get_gas_price_retryable( + eth_ws_provider: &Provider, + eth_ws_provider_fallback: &Provider, + ) -> Result> { + if let Ok(gas_price) = eth_ws_provider .get_gas_price() .await .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) @@ -1323,23 +1346,27 @@ impl Batcher { return Ok(gas_price); } - self.eth_ws_provider_fallback - .get_gas_price() - .await - .map_err(|e| { - warn!("Failed to get fallback gas price: {e:?}"); - RetryError::Transient - }) + eth_ws_provider_fallback.get_gas_price().await.map_err(|e| { + warn!("Failed to get fallback gas price: {e:?}"); + RetryError::Transient + }) } /// Uploads the batch to s3 using exponential backoff. - async fn upload_batch_to_s3_with_retry( + async fn upload_batch_to_s3( &self, batch_bytes: &[u8], file_name: &str, ) -> Result<(), BatcherError> { retry_function( - || self.upload_batch_to_s3(batch_bytes, file_name), + || { + Self::upload_batch_to_s3_retryable( + batch_bytes, + file_name, + self.s3_client.clone(), + &self.s3_bucket_name, + ) + }, DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, @@ -1348,24 +1375,18 @@ impl Batcher { .map_err(|_| BatcherError::BatchUploadError("Error uploading batch to s3".to_string())) } - async fn upload_batch_to_s3( - &self, + async fn upload_batch_to_s3_retryable( batch_bytes: &[u8], file_name: &str, + s3_client: S3Client, + s3_bucket_name: &String, ) -> Result<(), RetryError<()>> { - let s3_client = self.s3_client.clone(); - - s3::upload_object( - &s3_client, - &self.s3_bucket_name, - batch_bytes.to_vec(), - file_name, - ) - .await - .map_err(|e| { - warn!("Error uploading batch to s3 {e}"); - RetryError::Transient - })?; + s3::upload_object(&s3_client, s3_bucket_name, batch_bytes.to_vec(), file_name) + .await + .map_err(|e| { + warn!("Error uploading batch to s3 {e}"); + RetryError::Transient + })?; Ok(()) } } From ef8c721487e94acecb7769e992c4d9ab929731fa Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Fri, 18 Oct 2024 13:26:04 -0300 Subject: [PATCH 19/90] refactor: clean PR --- batcher/aligned-batcher/src/lib.rs | 9 ++++++--- batcher/aligned/src/main.rs | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 31588a7123..4707e17d11 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -494,7 +494,7 @@ impl Batcher { // * Perform validations over user state * // * ---------------------------------------------------* - let Ok(user_balance) = self.get_user_balance(&addr).await else { + let Some(user_balance) = self.get_user_balance(&addr).await else { error!("Could not get balance for address {addr:?}"); send_message(ws_conn_sink.clone(), ValidityResponseMessage::EthRpcError).await; return Ok(()); @@ -1172,7 +1172,7 @@ impl Batcher { }; let replacement_addr = non_paying_config.replacement.address(); - let Ok(replacement_user_balance) = self.get_user_balance(&replacement_addr).await else { + let Some(replacement_user_balance) = self.get_user_balance(&replacement_addr).await else { error!("Could not get balance for non-paying address {replacement_addr:?}"); send_message( ws_sink.clone(), @@ -1239,7 +1239,9 @@ impl Batcher { } /// Gets the balance of user with address `addr` from Ethereum using exponential backoff. - async fn get_user_balance(&self, addr: &Address) -> Result> { + /// Returns `None` if the balance couldn't be returned + /// FIXME: This should return a `Result` instead. + async fn get_user_balance(&self, addr: &Address) -> Option { retry_function( || { Self::get_user_balance_retryable( @@ -1253,6 +1255,7 @@ impl Batcher { DEFAULT_MAX_TIMES, ) .await + .ok() } async fn get_user_balance_retryable( diff --git a/batcher/aligned/src/main.rs b/batcher/aligned/src/main.rs index 09e76146ff..d1ffa511d0 100644 --- a/batcher/aligned/src/main.rs +++ b/batcher/aligned/src/main.rs @@ -300,6 +300,7 @@ async fn main() -> Result<(), AlignedError> { let chain_id = get_chain_id(eth_rpc_url.as_str()).await?; wallet = wallet.with_chain_id(chain_id); + let nonce = match &submit_args.nonce { Some(nonce) => U256::from_dec_str(nonce).map_err(|_| SubmitError::InvalidNonce)?, None => { @@ -341,6 +342,7 @@ async fn main() -> Result<(), AlignedError> { }; let mut unique_batch_merkle_roots = HashSet::new(); + for aligned_verification_data in aligned_verification_data_vec { save_response( batch_inclusion_data_directory_path.clone(), From a83f3044a37ae382eccdedf53577e57c2d83336c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Fri, 18 Oct 2024 18:26:48 -0300 Subject: [PATCH 20/90] test: add unit test to retry functions --- batcher/aligned-batcher/src/connection.rs | 4 +- batcher/aligned-batcher/src/lib.rs | 2 +- batcher/aligned-batcher/src/retry.rs | 296 ++++++++++++++++++---- 3 files changed, 253 insertions(+), 49 deletions(-) diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index b1ba79a589..f6808fbcd2 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -59,7 +59,7 @@ pub(crate) async fn send_message(ws_conn_sink: WsMessageSink, mess } } -async fn send_response( +pub(crate) async fn send_response( ws_sink: &Arc, Message>>>, serialized_response: Vec, ) { @@ -75,7 +75,7 @@ async fn send_response( } } -async fn send_response_retryable( +pub async fn send_response_retryable( ws_sink: &Arc, Message>>>, serialized_response: Vec, ) -> Result<(), RetryError> { diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 4707e17d11..d69953c3b2 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1382,7 +1382,7 @@ impl Batcher { batch_bytes: &[u8], file_name: &str, s3_client: S3Client, - s3_bucket_name: &String, + s3_bucket_name: &str, ) -> Result<(), RetryError<()>> { s3::upload_object(&s3_client, s3_bucket_name, batch_bytes.to_vec(), file_name) .await diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 7b4ef21efb..07fed2e076 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -44,79 +44,283 @@ where #[cfg(test)] mod test { use super::*; - use crate::eth; - use aligned_sdk::eth::batcher_payment_service::BatcherPaymentService; + use crate::{ + config::ECDSAConfig, + connection, + eth::{self, BatcherPaymentService}, + Batcher, + }; use ethers::{ contract::abigen, - providers::{Http, Middleware, Provider}, types::{Address, U256}, - utils::Anvil, + utils::{Anvil, AnvilInstance}, + }; + use futures_util::StreamExt; + use std::{str::FromStr, sync::Arc}; + use tokio::{ + net::{TcpListener, TcpStream}, + sync::RwLock, }; - use std::str::FromStr; - use std::{sync::Arc, time::SystemTime}; abigen!( BatcherPaymentServiceContract, "../aligned-sdk/abi/BatcherPaymentService.json" ); + async fn setup_anvil(port: u16) -> (AnvilInstance, BatcherPaymentService) { + let anvil = Anvil::new() + .port(port) + .arg("--load-state") + .arg("../../contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json") + .spawn(); + + let eth_rpc_provider = eth::get_provider(format!("http://localhost:{}", port)) + .expect("Failed to get provider"); + + let payment_service_addr = String::from("0x7969c5eD335650692Bc04293B07F5BF2e7A673C0"); + + let payment_service = eth::get_batcher_payment_service( + eth_rpc_provider, + ECDSAConfig { + private_key_store_path: "../../config-files/anvil.batcher.ecdsa.key.json" + .to_string(), + private_key_store_password: "".to_string(), + }, + payment_service_addr, + ) + .await + .expect("Failed to get Batcher Payment Service contract"); + (anvil, payment_service) + } + #[tokio::test] - async fn retry_test() { - async fn dummy_action(x: u64) -> Result> { - println!("Doing some operation..."); - println!("Actual time: {:?}", SystemTime::now()); - println!("X: {x}"); + async fn test_get_user_balance_retryable() { + let (_anvil, payment_service) = setup_anvil(8545u16).await; + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - Err(RetryError::Permanent(())) - } + let balance = retry_function( + || { + Batcher::get_user_balance_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, + ) + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .unwrap(); - assert!(retry_function(|| dummy_action(10), 2000, 2.0, 3) - .await - .is_err()); + assert!(balance == U256::zero()); } #[tokio::test] - async fn retry_test_eth() { - async fn get_gas_price() -> Result> { - let eth_rpc_provider = - eth::get_provider(String::from("https://ethereum-holesky-rpc.publicnode.com")) - .expect("Failed to get provider"); - - match eth_rpc_provider.get_gas_price().await { - Ok(val) => { - println!("GAS PRICE IS: {:?}", val); - Ok(val) - } - Err(_) => Err(RetryError::Transient), - } + async fn test_get_user_balance_retryable_kill_anvil() { + let payment_service; + { + // Kill anvil + let _anvil; + (_anvil, payment_service) = setup_anvil(8546u16).await; } + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - assert!(retry_function(get_gas_price, 2000, 2.0, 3).await.is_ok()); + let result = retry_function( + || { + Batcher::get_user_balance_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, + ) + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await; + assert!(matches!(result, Err(RetryError::Transient))); } #[tokio::test] - async fn test_anvil() { - let _anvil = Anvil::new() - .port(8545u16) - .arg("--load-state") - .arg("../../contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json") - .spawn(); + async fn test_user_balance_is_unlocked_retryable() { + let (_anvil, payment_service) = setup_anvil(8547u16).await; + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let eth_rpc_provider: Provider = - eth::get_provider(String::from("http://localhost:8545")) - .expect("Failed to get provider"); + let unlocked = retry_function( + || { + Batcher::user_balance_is_unlocked_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, + ) + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .unwrap(); + + assert!(unlocked == false); + } - let payment_service_addr = - Address::from_str("0x7969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + #[tokio::test] + async fn test_user_balance_is_unlocked_retryable_kill_anvil() { + let payment_service; + { + // Kill anvil + let _anvil; + (_anvil, payment_service) = setup_anvil(8548u16).await; + } + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let payment_service = - BatcherPaymentService::new(payment_service_addr, Arc::new(eth_rpc_provider)); + let result = retry_function( + || { + Batcher::user_balance_is_unlocked_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, + ) + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await; + assert!(matches!(result, Err(RetryError::Transient))); + } + #[tokio::test] + async fn test_get_user_nonce_retryable() { + let (_anvil, payment_service) = setup_anvil(8549u16).await; + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + + let nonce = retry_function( + || { + Batcher::get_user_nonce_from_ethereum_retryable( + &payment_service, + &payment_service, + dummy_user_addr, + ) + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .unwrap(); + + assert!(nonce == U256::zero()); + } + + #[tokio::test] + async fn test_get_user_nonce_retryable_kill_anvil() { + let payment_service; + { + // Kill anvil + let _anvil; + (_anvil, payment_service) = setup_anvil(8550u16).await; + } let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - if let Ok(balance) = payment_service.user_balances(dummy_user_addr).call().await { - println!("ALIGNED USER BALANCE: {:?}", balance) - }; + let result = retry_function( + || { + Batcher::get_user_nonce_from_ethereum_retryable( + &payment_service, + &payment_service, + dummy_user_addr, + ) + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await; + assert!(matches!(result, Err(RetryError::Transient))); + } + + #[tokio::test] + async fn test_get_gas_price_retryable() { + let (_anvil, _payment_service) = setup_anvil(8551u16).await; + let eth_rpc_provider = ethers::prelude::Provider::connect("ws://localhost:8551") + .await + .expect("Failed to get ethereum websocket provider"); + let result = retry_function( + || Batcher::get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider), + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .is_ok(); + + assert_eq!(result, true); + } + + #[tokio::test] + async fn test_get_gas_price_retryable_kill_anvil() { + let eth_rpc_provider; + { + // Kill anvil + let (_anvil, _payment_service) = setup_anvil(8552u16).await; + eth_rpc_provider = ethers::prelude::Provider::connect("ws://localhost:8552") + .await + .expect("Failed to get ethereum websocket provider"); + } + let result = retry_function( + || Batcher::get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider), + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await; + assert!(matches!(result, Err(RetryError::Transient))); + } + + #[tokio::test] + async fn test_send_response_retryable() { + let listener = TcpListener::bind("localhost:8553").await.unwrap(); + + let client_handle = tokio::spawn(async move { + let stream = TcpStream::connect("localhost:8553") + .await + .expect("Failed to connect"); + + let (mut ws_stream, _) = tokio_tungstenite::client_async("ws://localhost:8553", stream) + .await + .expect("WebSocket handshake failed"); + + // Read the response from the server + if let None = ws_stream.next().await { + panic!("Failed to receive valid WebSocket response"); + } + }); + + let (raw_stream, _) = listener + .accept() + .await + .expect("Failed to accept connection"); + let ws_stream = tokio_tungstenite::accept_async(raw_stream).await.unwrap(); + let (outgoing, _incoming) = ws_stream.split(); + let outgoing = Arc::new(RwLock::new(outgoing)); + let message = "Some message".to_string(); + + let result = retry_function( + || connection::send_response_retryable(&outgoing, message.clone().into_bytes()), + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .is_ok(); + assert!(result); + client_handle.await.unwrap() } } From 25becde1e8d622b77742349c6f66e133cc46e8be Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Fri, 18 Oct 2024 19:42:56 -0300 Subject: [PATCH 21/90] refactor: add inner errors to Transient type --- batcher/aligned-batcher/src/connection.rs | 2 +- batcher/aligned-batcher/src/lib.rs | 107 ++++++++-------------- batcher/aligned-batcher/src/main.rs | 27 ++++-- batcher/aligned-batcher/src/retry.rs | 22 +++-- 4 files changed, 69 insertions(+), 89 deletions(-) diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index f6808fbcd2..e4f9f62eaf 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -87,7 +87,7 @@ pub async fn send_response_retryable( match sending_result { Err(Error::AlreadyClosed) => Err(RetryError::Permanent(Error::AlreadyClosed)), - Err(_) => Err(RetryError::Transient), + Err(e) => Err(RetryError::Transient(e)), Ok(_) => Ok(()), } } diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index d69953c3b2..54198680b2 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -19,8 +19,8 @@ use aligned_sdk::core::types::{ use aws_sdk_s3::client::Client as S3Client; use eth::{try_create_new_task, BatcherPaymentService, CreateNewTaskFeeParams}; use ethers::prelude::{Middleware, Provider}; -use ethers::providers::{SubscriptionStream, Ws}; -use ethers::types::{Address, Block, Signature, TransactionReceipt, TxHash, H256, U256}; +use ethers::providers::Ws; +use ethers::types::{Address, Signature, TransactionReceipt, U256}; use futures_util::{future, SinkExt, StreamExt, TryStreamExt}; use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use lambdaworks_crypto::merkle_tree::traits::IsMerkleTreeBackend; @@ -38,7 +38,7 @@ mod connection; mod eth; pub mod gnark; pub mod metrics; -mod retry; +pub mod retry; pub mod risc_zero; pub mod s3; pub mod sp1; @@ -226,28 +226,27 @@ impl Batcher { Ok(()) } - pub async fn listen_new_blocks(self: Arc) -> Result<(), BatcherError> { - let stream = self - .eth_ws_provider - .subscribe_blocks() - .await - .map_err(|e| BatcherError::EthereumSubscriptionError(e.to_string()))?; + pub async fn listen_new_blocks(self: Arc) -> Result<(), RetryError> { + let mut stream = self.eth_ws_provider.subscribe_blocks().await.map_err(|e| { + warn!("Error subscribing to blocks."); + RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) + })?; - let stream_fallback = self + let mut stream_fallback = self .eth_ws_provider_fallback .subscribe_blocks() .await - .map_err(|e| BatcherError::EthereumSubscriptionError(e.to_string()))?; + .map_err(|e| { + warn!("Error subscribing to blocks."); + RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) + })?; let last_seen_block = Mutex::::new(0); - let stream = Arc::new(Mutex::new(stream)); - let stream_fallback = Arc::new(Mutex::new(stream_fallback)); - - while let Ok(block) = self - .get_next_block(stream.clone(), stream_fallback.clone()) - .await - { + while let Some(block) = tokio::select! { + block = stream.next() => block, + block = stream_fallback.next() => block, + } { let batcher = self.clone(); let block_number = block.number.unwrap_or_default(); let block_number = u64::try_from(block_number).unwrap_or_default(); @@ -267,50 +266,13 @@ impl Batcher { }; }); } + warn!("No more blocks"); - Err(BatcherError::EthereumSubscriptionError( - "No more blocks".to_string(), + Err(RetryError::Transient( + BatcherError::EthereumSubscriptionError("No more blocks".to_string()), )) } - async fn get_next_block( - &self, - stream: Arc>>>, - stream_fallback: Arc>>>, - ) -> Result, BatcherError> { - retry_function( - || { - let stream = Arc::clone(&stream); - let stream_fallback = Arc::clone(&stream_fallback); - async move { - let mut stream_lock = stream.lock().await; - let mut stream_fallback_lock = stream_fallback.lock().await; - Self::get_next_block_retryable(&mut stream_lock, &mut stream_fallback_lock) - .await - } - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await - .map_err(|_| BatcherError::EthereumSubscriptionError("No more blocks".to_string())) - } - - async fn get_next_block_retryable( - stream: &mut SubscriptionStream<'_, Ws, Block>, - stream_fallback: &mut SubscriptionStream<'_, Ws, Block>, - ) -> Result, RetryError<()>> { - tokio::select! { - Some(block) = stream.next() => Ok(block), - Some(block) = stream_fallback.next() => Ok(block), - else => { - warn!("Error fetching next block."); - Err(RetryError::Transient) - } - } - } - async fn handle_connection( self: Arc, raw_stream: TcpStream, @@ -696,7 +658,10 @@ impl Batcher { } /// Gets the user nonce from Ethereum using exponential backoff. - async fn get_user_nonce_from_ethereum(&self, addr: Address) -> Result> { + async fn get_user_nonce_from_ethereum( + &self, + addr: Address, + ) -> Result> { retry_function( || { Self::get_user_nonce_from_ethereum_retryable( @@ -716,7 +681,7 @@ impl Batcher { payment_service: &BatcherPaymentService, payment_service_fallback: &BatcherPaymentService, addr: Address, - ) -> Result> { + ) -> Result> { if let Ok(nonce) = payment_service.user_nonces(addr).call().await { return Ok(nonce); } @@ -724,9 +689,9 @@ impl Batcher { .user_nonces(addr) .call() .await - .map_err(|_| { + .map_err(|e| { warn!("Error getting user nonce."); - RetryError::Transient + RetryError::Transient(e.to_string()) }) } @@ -1262,7 +1227,7 @@ impl Batcher { payment_service: &BatcherPaymentService, payment_service_fallback: &BatcherPaymentService, addr: &Address, - ) -> Result> { + ) -> Result> { if let Ok(balance) = payment_service.user_balances(*addr).call().await { return Ok(balance); }; @@ -1271,9 +1236,9 @@ impl Batcher { .user_balances(*addr) .call() .await - .map_err(|_| { + .map_err(|e| { warn!("Failed to get balance for address {:?}", addr); - RetryError::Transient + RetryError::Transient(e.to_string()) }) } @@ -1296,7 +1261,7 @@ impl Batcher { { Ok(result) => result, Err(_) => { - warn!("Could not get user locking state"); + warn!("Could not get user locking state."); false } } @@ -1318,7 +1283,7 @@ impl Batcher { return Ok(unlock_block != U256::zero()); } warn!("Failed to get user locking state {:?}", addr); - Err(RetryError::Transient) + Err(RetryError::Transient(())) } /// Gets the current gas price from Ethereum using exponential backoff. @@ -1340,7 +1305,7 @@ impl Batcher { async fn get_gas_price_retryable( eth_ws_provider: &Provider, eth_ws_provider_fallback: &Provider, - ) -> Result> { + ) -> Result> { if let Ok(gas_price) = eth_ws_provider .get_gas_price() .await @@ -1351,7 +1316,7 @@ impl Batcher { eth_ws_provider_fallback.get_gas_price().await.map_err(|e| { warn!("Failed to get fallback gas price: {e:?}"); - RetryError::Transient + RetryError::Transient(e.to_string()) }) } @@ -1383,12 +1348,12 @@ impl Batcher { file_name: &str, s3_client: S3Client, s3_bucket_name: &str, - ) -> Result<(), RetryError<()>> { + ) -> Result<(), RetryError> { s3::upload_object(&s3_client, s3_bucket_name, batch_bytes.to_vec(), file_name) .await .map_err(|e| { warn!("Error uploading batch to s3 {e}"); - RetryError::Transient + RetryError::Transient(e.to_string()) })?; Ok(()) } diff --git a/batcher/aligned-batcher/src/main.rs b/batcher/aligned-batcher/src/main.rs index bc13885052..e6e5ba8d22 100644 --- a/batcher/aligned-batcher/src/main.rs +++ b/batcher/aligned-batcher/src/main.rs @@ -5,7 +5,11 @@ use std::sync::Arc; use clap::Parser; use env_logger::Env; -use aligned_batcher::{types::errors::BatcherError, Batcher}; +use aligned_batcher::{ + retry::{retry_function, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY}, + types::errors::BatcherError, + Batcher, +}; /// Batcher main flow: /// There are two main tasks spawned: `listen_connections` and `listen_new_blocks` @@ -42,14 +46,21 @@ async fn main() -> Result<(), BatcherError> { let addr = format!("localhost:{}", port); + let batcher_clone = batcher.clone(); + // spawn task to listening for incoming blocks - tokio::spawn({ - let app = batcher.clone(); - async move { - app.listen_new_blocks() - .await - .expect("Error listening for new blocks exiting") - } + tokio::spawn(async move { + retry_function( + || { + let app = batcher_clone.clone(); + async move { app.listen_new_blocks().await } + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .unwrap(); }); batcher.metrics.inc_batcher_restart(); diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 07fed2e076..e3110293dd 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -8,16 +8,20 @@ pub const DEFAULT_FACTOR: f32 = 2.0; #[derive(Debug)] pub enum RetryError { - Transient, + Transient(E), Permanent(E), } -impl std::fmt::Display for RetryError { +impl std::fmt::Display for RetryError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "Retry error!") + match self { + RetryError::Transient(e) => write!(f, "{}", e), + RetryError::Permanent(e) => write!(f, "{}", e), + } } } -impl std::error::Error for RetryError where E: std::fmt::Debug {} + +impl std::error::Error for RetryError where E: std::fmt::Debug {} pub async fn retry_function( function: FutureFn, @@ -37,7 +41,7 @@ where function .retry(backoff) .sleep(tokio::time::sleep) - .when(|e| matches!(e, RetryError::Transient)) + .when(|e| matches!(e, RetryError::Transient(_))) .await } @@ -141,7 +145,7 @@ mod test { DEFAULT_MAX_TIMES, ) .await; - assert!(matches!(result, Err(RetryError::Transient))); + assert!(matches!(result, Err(RetryError::Transient(_)))); } #[tokio::test] @@ -192,7 +196,7 @@ mod test { DEFAULT_MAX_TIMES, ) .await; - assert!(matches!(result, Err(RetryError::Transient))); + assert!(matches!(result, Err(RetryError::Transient(_)))); } #[tokio::test] @@ -243,7 +247,7 @@ mod test { DEFAULT_MAX_TIMES, ) .await; - assert!(matches!(result, Err(RetryError::Transient))); + assert!(matches!(result, Err(RetryError::Transient(_)))); } #[tokio::test] @@ -281,7 +285,7 @@ mod test { DEFAULT_MAX_TIMES, ) .await; - assert!(matches!(result, Err(RetryError::Transient))); + assert!(matches!(result, Err(RetryError::Transient(_)))); } #[tokio::test] From bb68b83f13f3653ebb70991b2740fa2ff181d99b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Sun, 20 Oct 2024 19:34:35 -0300 Subject: [PATCH 22/90] refactor: improve logging --- batcher/aligned-batcher/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 54198680b2..202afa8e04 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -690,7 +690,7 @@ impl Batcher { .call() .await .map_err(|e| { - warn!("Error getting user nonce."); + warn!("Error getting user nonce: {e}"); RetryError::Transient(e.to_string()) }) } @@ -1237,7 +1237,7 @@ impl Batcher { .call() .await .map_err(|e| { - warn!("Failed to get balance for address {:?}", addr); + warn!("Failed to get balance for address {:?}. Error: {e}", addr); RetryError::Transient(e.to_string()) }) } @@ -1296,8 +1296,8 @@ impl Batcher { DEFAULT_MAX_TIMES, ) .await - .map_err(|_| { - error!("Failed to get gas price"); + .map_err(|e| { + error!("Could't get gas price: {e}"); BatcherError::GasPriceError }) } From 6c3b76c000b9d4d47c7d4e8886ea0db84fbdbfab Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 21 Oct 2024 11:36:54 -0300 Subject: [PATCH 23/90] test: debugging ci --- batcher/aligned-batcher/src/retry.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 2702edd589..bbd9f52e52 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -60,7 +60,7 @@ mod test { utils::{Anvil, AnvilInstance}, }; use futures_util::StreamExt; - use std::{str::FromStr, sync::Arc}; + use std::{path::PathBuf, str::FromStr, sync::Arc}; use tokio::{ net::{TcpListener, TcpStream}, sync::RwLock, @@ -97,6 +97,10 @@ mod test { (anvil, payment_service) } + #[tokio::test] + async fn test() { + assert_eq!(std::env::current_dir().unwrap(), PathBuf::new()); + } #[tokio::test] async fn test_get_user_balance_retryable() { let (_anvil, payment_service) = setup_anvil(8545u16).await; From bb89b6902149e2358b8fa479d81c6eb1c931319e Mon Sep 17 00:00:00 2001 From: samoht9277 Date: Mon, 21 Oct 2024 11:57:18 -0300 Subject: [PATCH 24/90] Added foundry toolchain for CI. --- .github/workflows/build-and-test-rust.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-and-test-rust.yml b/.github/workflows/build-and-test-rust.yml index 2ba2a67392..3a1633891a 100644 --- a/.github/workflows/build-and-test-rust.yml +++ b/.github/workflows/build-and-test-rust.yml @@ -52,6 +52,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + + - name: foundry-toolchain + uses: foundry-rs/foundry-toolchain@v1.2.0 + - name: Cache Rust dependencies uses: actions/cache@v3 with: @@ -62,6 +66,7 @@ jobs: key: ${{ runner.os }}-rust-${{ hashFiles('**/Cargo.lock') }} restore-keys: | ${{ runner.os }}-rust- + - name: Run tests run: | cd batcher From 712a5550e1fe7dab3aaa5b87f6bc3abcbc95c435 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 21 Oct 2024 12:11:46 -0300 Subject: [PATCH 25/90] Revert "test: debugging ci" This reverts commit 6c3b76c000b9d4d47c7d4e8886ea0db84fbdbfab. --- batcher/aligned-batcher/src/retry.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index bbd9f52e52..2702edd589 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -60,7 +60,7 @@ mod test { utils::{Anvil, AnvilInstance}, }; use futures_util::StreamExt; - use std::{path::PathBuf, str::FromStr, sync::Arc}; + use std::{str::FromStr, sync::Arc}; use tokio::{ net::{TcpListener, TcpStream}, sync::RwLock, @@ -97,10 +97,6 @@ mod test { (anvil, payment_service) } - #[tokio::test] - async fn test() { - assert_eq!(std::env::current_dir().unwrap(), PathBuf::new()); - } #[tokio::test] async fn test_get_user_balance_retryable() { let (_anvil, payment_service) = setup_anvil(8545u16).await; From eb8c39bdf5ba177cc755d2b528b40e20f80c64c8 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 21 Oct 2024 13:59:38 -0300 Subject: [PATCH 26/90] refactor: address comments --- .../src/eth/payment_service.rs | 60 +++++++- batcher/aligned-batcher/src/eth/utils.rs | 27 +++- batcher/aligned-batcher/src/lib.rs | 105 +++---------- batcher/aligned-batcher/src/main.rs | 27 +--- batcher/aligned-batcher/src/retry.rs | 142 ++++++------------ 5 files changed, 163 insertions(+), 198 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/payment_service.rs b/batcher/aligned-batcher/src/eth/payment_service.rs index b73326450b..a3f5ef2962 100644 --- a/batcher/aligned-batcher/src/eth/payment_service.rs +++ b/batcher/aligned-batcher/src/eth/payment_service.rs @@ -5,9 +5,9 @@ use aligned_sdk::eth::batcher_payment_service::BatcherPaymentServiceContract; use ethers::prelude::k256::ecdsa::SigningKey; use ethers::prelude::*; use gas_escalator::{Frequency, GeometricGasPrice}; -use log::info; +use log::{info, warn}; -use crate::{config::ECDSAConfig, types::errors::BatcherSendError}; +use crate::{config::ECDSAConfig, retry::RetryError, types::errors::BatcherSendError}; use super::utils::{GAS_ESCALATOR_INTERVAL, GAS_MULTIPLIER}; @@ -71,6 +71,25 @@ pub async fn get_batcher_payment_service( Ok(payment_service) } +pub async fn user_balance_is_unlocked_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: &Address, +) -> Result> { + if let Ok(unlock_block) = payment_service.user_unlock_block(*addr).call().await { + return Ok(unlock_block != U256::zero()); + } + if let Ok(unlock_block) = payment_service_fallback + .user_unlock_block(*addr) + .call() + .await + { + return Ok(unlock_block != U256::zero()); + } + warn!("Failed to get user locking state {:?}", addr); + Err(RetryError::Transient(())) +} + pub async fn try_create_new_task( batch_merkle_root: [u8; 32], batch_data_pointer: String, @@ -101,3 +120,40 @@ pub async fn try_create_new_task( .map_err(|err| BatcherSendError::UnknownError(err.to_string()))? .ok_or(BatcherSendError::ReceiptNotFound) } + +pub async fn get_user_balance_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: &Address, +) -> Result> { + if let Ok(balance) = payment_service.user_balances(*addr).call().await { + return Ok(balance); + }; + + payment_service_fallback + .user_balances(*addr) + .call() + .await + .map_err(|e| { + warn!("Failed to get balance for address {:?}. Error: {e}", addr); + RetryError::Transient(e.to_string()) + }) +} + +pub async fn get_user_nonce_from_ethereum_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: Address, +) -> Result> { + if let Ok(nonce) = payment_service.user_nonces(addr).call().await { + return Ok(nonce); + } + payment_service_fallback + .user_nonces(addr) + .call() + .await + .map_err(|e| { + warn!("Error getting user nonce: {e}"); + RetryError::Transient(e.to_string()) + }) +} diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 8b9e6f3f64..7c670944b6 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -1,6 +1,13 @@ use std::str::FromStr; -use ethers::providers::{Http, Provider}; +use ethers::{ + providers::{Http, Provider, Ws}, + types::U256, +}; +use log::warn; + +use crate::retry::RetryError; +use ethers::prelude::Middleware; pub(crate) const GAS_MULTIPLIER: f64 = 1.125; // Multiplier for the gas price for gas escalator pub(crate) const GAS_ESCALATOR_INTERVAL: u64 = 12; // Time in seconds between gas escalations @@ -10,3 +17,21 @@ pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error .map_err(|e| anyhow::Error::msg(format!("Failed to create provider: {}", e)))?; Ok(Provider::new(provider)) } + +pub async fn get_gas_price_retryable( + eth_ws_provider: &Provider, + eth_ws_provider_fallback: &Provider, +) -> Result> { + if let Ok(gas_price) = eth_ws_provider + .get_gas_price() + .await + .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) + { + return Ok(gas_price); + } + + eth_ws_provider_fallback.get_gas_price().await.map_err(|e| { + warn!("Failed to get fallback gas price: {e:?}"); + RetryError::Transient(e.to_string()) + }) +} diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index ab09b4f7ef..e8c5206186 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -3,6 +3,7 @@ use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use eth::service_manager::ServiceManager; +use eth::utils::get_gas_price_retryable; use ethers::contract::ContractError; use ethers::signers::Signer; use retry::{retry_function, RetryError, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY}; @@ -28,7 +29,9 @@ use aligned_sdk::core::types::{ use aws_sdk_s3::client::Client as S3Client; use eth::payment_service::{ - try_create_new_task, BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT, + get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, try_create_new_task, + user_balance_is_unlocked_retryable, BatcherPaymentService, CreateNewTaskFeeParams, + SignerMiddlewareT, }; use ethers::prelude::{Middleware, Provider}; use ethers::providers::Ws; @@ -258,7 +261,23 @@ impl Batcher { Ok(()) } - pub async fn listen_new_blocks(self: Arc) -> Result<(), RetryError> { + pub async fn listen_new_blocks(self: Arc) -> Result<(), BatcherError> { + retry_function( + || { + let app = self.clone(); + async move { app.listen_new_blocks_retryable().await } + }, + DEFAULT_MIN_DELAY, + DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, + ) + .await + .map_err(|e| e.inner()) + } + + pub async fn listen_new_blocks_retryable( + self: Arc, + ) -> Result<(), RetryError> { let mut stream = self.eth_ws_provider.subscribe_blocks().await.map_err(|e| { warn!("Error subscribing to blocks."); RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) @@ -738,7 +757,7 @@ impl Batcher { ) -> Result> { retry_function( || { - Self::get_user_nonce_from_ethereum_retryable( + get_user_nonce_from_ethereum_retryable( &self.payment_service, &self.payment_service_fallback, addr, @@ -751,24 +770,6 @@ impl Batcher { .await } - async fn get_user_nonce_from_ethereum_retryable( - payment_service: &BatcherPaymentService, - payment_service_fallback: &BatcherPaymentService, - addr: Address, - ) -> Result> { - if let Ok(nonce) = payment_service.user_nonces(addr).call().await { - return Ok(nonce); - } - payment_service_fallback - .user_nonces(addr) - .call() - .await - .map_err(|e| { - warn!("Error getting user nonce: {e}"); - RetryError::Transient(e.to_string()) - }) - } - /// Adds verification data to the current batch queue. async fn add_to_batch( &self, @@ -1301,7 +1302,7 @@ impl Batcher { async fn get_user_balance(&self, addr: &Address) -> Option { retry_function( || { - Self::get_user_balance_retryable( + get_user_balance_retryable( &self.payment_service, &self.payment_service_fallback, addr, @@ -1315,31 +1316,12 @@ impl Batcher { .ok() } - async fn get_user_balance_retryable( - payment_service: &BatcherPaymentService, - payment_service_fallback: &BatcherPaymentService, - addr: &Address, - ) -> Result> { - if let Ok(balance) = payment_service.user_balances(*addr).call().await { - return Ok(balance); - }; - - payment_service_fallback - .user_balances(*addr) - .call() - .await - .map_err(|e| { - warn!("Failed to get balance for address {:?}. Error: {e}", addr); - RetryError::Transient(e.to_string()) - }) - } - /// Checks if the user's balance is unlocked for a given address using exponential backoff. /// Returns `false` if an error occurs during the retries. async fn user_balance_is_unlocked(&self, addr: &Address) -> bool { match retry_function( || { - Self::user_balance_is_unlocked_retryable( + user_balance_is_unlocked_retryable( &self.payment_service, &self.payment_service_fallback, addr, @@ -1359,30 +1341,11 @@ impl Batcher { } } - async fn user_balance_is_unlocked_retryable( - payment_service: &BatcherPaymentService, - payment_service_fallback: &BatcherPaymentService, - addr: &Address, - ) -> Result> { - if let Ok(unlock_block) = payment_service.user_unlock_block(*addr).call().await { - return Ok(unlock_block != U256::zero()); - } - if let Ok(unlock_block) = payment_service_fallback - .user_unlock_block(*addr) - .call() - .await - { - return Ok(unlock_block != U256::zero()); - } - warn!("Failed to get user locking state {:?}", addr); - Err(RetryError::Transient(())) - } - /// Gets the current gas price from Ethereum using exponential backoff. /// Returns `None` if the gas price couldn't be returned async fn get_gas_price(&self) -> Result { retry_function( - || Self::get_gas_price_retryable(&self.eth_ws_provider, &self.eth_ws_provider_fallback), + || get_gas_price_retryable(&self.eth_ws_provider, &self.eth_ws_provider_fallback), DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, @@ -1394,24 +1357,6 @@ impl Batcher { }) } - async fn get_gas_price_retryable( - eth_ws_provider: &Provider, - eth_ws_provider_fallback: &Provider, - ) -> Result> { - if let Ok(gas_price) = eth_ws_provider - .get_gas_price() - .await - .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) - { - return Ok(gas_price); - } - - eth_ws_provider_fallback.get_gas_price().await.map_err(|e| { - warn!("Failed to get fallback gas price: {e:?}"); - RetryError::Transient(e.to_string()) - }) - } - /// Uploads the batch to s3 using exponential backoff. async fn upload_batch_to_s3( &self, diff --git a/batcher/aligned-batcher/src/main.rs b/batcher/aligned-batcher/src/main.rs index e6e5ba8d22..bc13885052 100644 --- a/batcher/aligned-batcher/src/main.rs +++ b/batcher/aligned-batcher/src/main.rs @@ -5,11 +5,7 @@ use std::sync::Arc; use clap::Parser; use env_logger::Env; -use aligned_batcher::{ - retry::{retry_function, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY}, - types::errors::BatcherError, - Batcher, -}; +use aligned_batcher::{types::errors::BatcherError, Batcher}; /// Batcher main flow: /// There are two main tasks spawned: `listen_connections` and `listen_new_blocks` @@ -46,21 +42,14 @@ async fn main() -> Result<(), BatcherError> { let addr = format!("localhost:{}", port); - let batcher_clone = batcher.clone(); - // spawn task to listening for incoming blocks - tokio::spawn(async move { - retry_function( - || { - let app = batcher_clone.clone(); - async move { app.listen_new_blocks().await } - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await - .unwrap(); + tokio::spawn({ + let app = batcher.clone(); + async move { + app.listen_new_blocks() + .await + .expect("Error listening for new blocks exiting") + } }); batcher.metrics.inc_batcher_restart(); diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 2702edd589..336e3ece5e 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -21,6 +21,15 @@ impl std::fmt::Display for RetryError { } } +impl RetryError { + pub fn inner(self) -> E { + match self { + RetryError::Transient(e) => e, + RetryError::Permanent(e) => e, + } + } +} + impl std::error::Error for RetryError where E: std::fmt::Debug {} pub async fn retry_function( @@ -51,8 +60,14 @@ mod test { use crate::{ config::ECDSAConfig, connection, - eth::{self, payment_service::BatcherPaymentService}, - Batcher, + eth::{ + self, + payment_service::{ + get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, + user_balance_is_unlocked_retryable, BatcherPaymentService, + }, + utils::get_gas_price_retryable, + }, }; use ethers::{ contract::abigen, @@ -103,20 +118,10 @@ mod test { let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let balance = retry_function( - || { - Batcher::get_user_balance_retryable( - &payment_service, - &payment_service, - &dummy_user_addr, - ) - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await - .unwrap(); + let balance = + get_user_balance_retryable(&payment_service, &payment_service, &dummy_user_addr) + .await + .unwrap(); assert!(balance == U256::zero()); } @@ -132,19 +137,8 @@ mod test { let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let result = retry_function( - || { - Batcher::get_user_balance_retryable( - &payment_service, - &payment_service, - &dummy_user_addr, - ) - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await; + let result = + get_user_balance_retryable(&payment_service, &payment_service, &dummy_user_addr).await; assert!(matches!(result, Err(RetryError::Transient(_)))); } @@ -154,17 +148,10 @@ mod test { let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let unlocked = retry_function( - || { - Batcher::user_balance_is_unlocked_retryable( - &payment_service, - &payment_service, - &dummy_user_addr, - ) - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + let unlocked = user_balance_is_unlocked_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, ) .await .unwrap(); @@ -183,17 +170,10 @@ mod test { let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let result = retry_function( - || { - Batcher::user_balance_is_unlocked_retryable( - &payment_service, - &payment_service, - &dummy_user_addr, - ) - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + let result = user_balance_is_unlocked_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, ) .await; assert!(matches!(result, Err(RetryError::Transient(_)))); @@ -205,17 +185,10 @@ mod test { let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let nonce = retry_function( - || { - Batcher::get_user_nonce_from_ethereum_retryable( - &payment_service, - &payment_service, - dummy_user_addr, - ) - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + let nonce = get_user_nonce_from_ethereum_retryable( + &payment_service, + &payment_service, + dummy_user_addr, ) .await .unwrap(); @@ -234,17 +207,10 @@ mod test { let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let result = retry_function( - || { - Batcher::get_user_nonce_from_ethereum_retryable( - &payment_service, - &payment_service, - dummy_user_addr, - ) - }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + let result = get_user_nonce_from_ethereum_retryable( + &payment_service, + &payment_service, + dummy_user_addr, ) .await; assert!(matches!(result, Err(RetryError::Transient(_)))); @@ -256,14 +222,9 @@ mod test { let eth_rpc_provider = ethers::prelude::Provider::connect("ws://localhost:8551") .await .expect("Failed to get ethereum websocket provider"); - let result = retry_function( - || Batcher::get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider), - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await - .is_ok(); + let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider) + .await + .is_ok(); assert_eq!(result, true); } @@ -278,13 +239,7 @@ mod test { .await .expect("Failed to get ethereum websocket provider"); } - let result = retry_function( - || Batcher::get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider), - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await; + let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; assert!(matches!(result, Err(RetryError::Transient(_)))); } @@ -316,14 +271,9 @@ mod test { let outgoing = Arc::new(RwLock::new(outgoing)); let message = "Some message".to_string(); - let result = retry_function( - || connection::send_response_retryable(&outgoing, message.clone().into_bytes()), - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await - .is_ok(); + let result = connection::send_response_retryable(&outgoing, message.clone().into_bytes()) + .await + .is_ok(); assert!(result); client_handle.await.unwrap() } From 9d3c513932368ff902472c082d0e03baeeb371fd Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 21 Oct 2024 14:05:31 -0300 Subject: [PATCH 27/90] refactor: improve error logging --- batcher/aligned-batcher/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index e8c5206186..a14018cad5 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -317,10 +317,10 @@ impl Batcher { }; }); } - warn!("No more blocks"); + error!("Failed to fetch blocks"); Err(RetryError::Transient( - BatcherError::EthereumSubscriptionError("No more blocks".to_string()), + BatcherError::EthereumSubscriptionError("Could not get new blocks".to_string()), )) } From ba32e2eb5b883b145c428debe05596fe33306b55 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 21 Oct 2024 17:58:16 -0300 Subject: [PATCH 28/90] refactor: remove web sockets from batcher struct --- batcher/aligned-batcher/src/eth/utils.rs | 6 +- batcher/aligned-batcher/src/lib.rs | 62 ++++++---- batcher/aligned-batcher/src/retry.rs | 137 ++++++++++++----------- 3 files changed, 115 insertions(+), 90 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 7c670944b6..86ca660822 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -1,7 +1,7 @@ use std::str::FromStr; use ethers::{ - providers::{Http, Provider, Ws}, + providers::{Http, Provider}, types::U256, }; use log::warn; @@ -19,8 +19,8 @@ pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error } pub async fn get_gas_price_retryable( - eth_ws_provider: &Provider, - eth_ws_provider_fallback: &Provider, + eth_ws_provider: &Provider, + eth_ws_provider_fallback: &Provider, ) -> Result> { if let Ok(gas_price) = eth_ws_provider .get_gas_price() diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index a14018cad5..4275bb176c 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -33,8 +33,7 @@ use eth::payment_service::{ user_balance_is_unlocked_retryable, BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT, }; -use ethers::prelude::{Middleware, Provider}; -use ethers::providers::Ws; +use ethers::prelude::{Http, Middleware, Provider}; use ethers::types::{Address, Signature, TransactionReceipt, U256}; use futures_util::{future, SinkExt, StreamExt, TryStreamExt}; use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; @@ -64,8 +63,10 @@ pub struct Batcher { s3_client: S3Client, s3_bucket_name: String, download_endpoint: String, - eth_ws_provider: Provider, - eth_ws_provider_fallback: Provider, + eth_ws_url: String, + eth_ws_url_fallback: String, + eth_http_provider: Provider, + eth_http_provider_fallback: Provider, chain_id: U256, payment_service: BatcherPaymentService, payment_service_fallback: BatcherPaymentService, @@ -102,10 +103,6 @@ impl Batcher { let deployment_output = ContractDeploymentOutput::new(config.aligned_layer_deployment_config_file_path); - let eth_ws_provider = Provider::connect(&config.eth_ws_url) - .await - .expect("Failed to get ethereum websocket provider"); - log::info!( "Starting metrics server on port {}", config.batcher.metrics_port @@ -113,10 +110,6 @@ impl Batcher { let metrics = metrics::BatcherMetrics::start(config.batcher.metrics_port) .expect("Failed to start metrics server"); - let eth_ws_provider_fallback = Provider::connect(&config.eth_ws_url_fallback) - .await - .expect("Failed to get fallback ethereum websocket provider"); - let eth_rpc_provider = eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get provider"); @@ -221,12 +214,20 @@ impl Batcher { } .expect("Failed to get disabled verifiers"); + let eth_http_provider = + eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get http provider"); + + let eth_http_provider_fallback = eth::get_provider(config.eth_rpc_url_fallback.clone()) + .expect("Failed to get fallback http provider"); + Self { s3_client, s3_bucket_name, download_endpoint, - eth_ws_provider, - eth_ws_provider_fallback, + eth_ws_url: config.eth_ws_url, + eth_ws_url_fallback: config.eth_ws_url_fallback, + eth_http_provider, + eth_http_provider_fallback, chain_id, payment_service, payment_service_fallback, @@ -278,19 +279,32 @@ impl Batcher { pub async fn listen_new_blocks_retryable( self: Arc, ) -> Result<(), RetryError> { - let mut stream = self.eth_ws_provider.subscribe_blocks().await.map_err(|e| { + let eth_ws_provider = Provider::connect(&self.eth_ws_url).await.map_err(|e| { + warn!("Failed to get ethereum websocket provider"); + RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) + })?; + + let eth_ws_provider_fallback = + Provider::connect(&self.eth_ws_url_fallback) + .await + .map_err(|e| { + warn!("Failed to get fallback ethereum websocket provider"); + RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) + })?; + + let mut stream = eth_ws_provider.subscribe_blocks().await.map_err(|e| { warn!("Error subscribing to blocks."); RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) })?; - let mut stream_fallback = self - .eth_ws_provider_fallback - .subscribe_blocks() - .await - .map_err(|e| { - warn!("Error subscribing to blocks."); - RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) - })?; + let mut stream_fallback = + eth_ws_provider_fallback + .subscribe_blocks() + .await + .map_err(|e| { + warn!("Error subscribing to blocks."); + RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) + })?; let last_seen_block = Mutex::::new(0); @@ -1345,7 +1359,7 @@ impl Batcher { /// Returns `None` if the gas price couldn't be returned async fn get_gas_price(&self) -> Result { retry_function( - || get_gas_price_retryable(&self.eth_ws_provider, &self.eth_ws_provider_fallback), + || get_gas_price_retryable(&self.eth_http_provider, &self.eth_http_provider_fallback), DEFAULT_MIN_DELAY, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 336e3ece5e..eb60e84c7f 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -61,7 +61,7 @@ mod test { config::ECDSAConfig, connection, eth::{ - self, + self, get_provider, payment_service::{ get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, user_balance_is_unlocked_retryable, BatcherPaymentService, @@ -114,10 +114,28 @@ mod test { #[tokio::test] async fn test_get_user_balance_retryable() { - let (_anvil, payment_service) = setup_anvil(8545u16).await; + let payment_service; let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + { + let _anvil; + (_anvil, payment_service) = setup_anvil(8545u16).await; + + let balance = + get_user_balance_retryable(&payment_service, &payment_service, &dummy_user_addr) + .await + .unwrap(); + + assert!(balance == U256::zero()); + // Kill anvil + } + + let result = + get_user_balance_retryable(&payment_service, &payment_service, &dummy_user_addr).await; + assert!(matches!(result, Err(RetryError::Transient(_)))); + // restart anvil + let (_anvil, _) = setup_anvil(8545u16).await; let balance = get_user_balance_retryable(&payment_service, &payment_service, &dummy_user_addr) .await @@ -127,27 +145,36 @@ mod test { } #[tokio::test] - async fn test_get_user_balance_retryable_kill_anvil() { + async fn test_user_balance_is_unlocked_retryable_kill_anvil() { let payment_service; + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + { - // Kill anvil let _anvil; (_anvil, payment_service) = setup_anvil(8546u16).await; + let unlocked = user_balance_is_unlocked_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, + ) + .await + .unwrap(); + + assert!(unlocked == false); + // Kill Anvil } - let dummy_user_addr = - Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let result = - get_user_balance_retryable(&payment_service, &payment_service, &dummy_user_addr).await; + let result = user_balance_is_unlocked_retryable( + &payment_service, + &payment_service, + &dummy_user_addr, + ) + .await; assert!(matches!(result, Err(RetryError::Transient(_)))); - } - - #[tokio::test] - async fn test_user_balance_is_unlocked_retryable() { - let (_anvil, payment_service) = setup_anvil(8547u16).await; - let dummy_user_addr = - Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + // restart Anvil + let (_anvil, payment_service) = setup_anvil(8546u16).await; let unlocked = user_balance_is_unlocked_retryable( &payment_service, &payment_service, @@ -160,30 +187,35 @@ mod test { } #[tokio::test] - async fn test_user_balance_is_unlocked_retryable_kill_anvil() { + async fn test_get_user_nonce_retryable_kill_anvil() { let payment_service; + let dummy_user_addr = + Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); { - // Kill anvil let _anvil; - (_anvil, payment_service) = setup_anvil(8548u16).await; + (_anvil, payment_service) = setup_anvil(8547u16).await; + let nonce = get_user_nonce_from_ethereum_retryable( + &payment_service, + &payment_service, + dummy_user_addr, + ) + .await + .unwrap(); + + assert!(nonce == U256::zero()); + // Kill Anvil } - let dummy_user_addr = - Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); - let result = user_balance_is_unlocked_retryable( + let result = get_user_nonce_from_ethereum_retryable( &payment_service, &payment_service, - &dummy_user_addr, + dummy_user_addr, ) .await; assert!(matches!(result, Err(RetryError::Transient(_)))); - } - #[tokio::test] - async fn test_get_user_nonce_retryable() { - let (_anvil, payment_service) = setup_anvil(8549u16).await; - let dummy_user_addr = - Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + // restart Anvil + let (_anvil, payment_service) = setup_anvil(8547u16).await; let nonce = get_user_nonce_from_ethereum_retryable( &payment_service, @@ -197,31 +229,24 @@ mod test { } #[tokio::test] - async fn test_get_user_nonce_retryable_kill_anvil() { - let payment_service; + async fn test_get_gas_price_retryable_kill_anvil() { + let eth_rpc_provider; { - // Kill anvil - let _anvil; - (_anvil, payment_service) = setup_anvil(8550u16).await; - } - let dummy_user_addr = - Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); + let (_anvil, _payment_service) = setup_anvil(8548u16).await; + eth_rpc_provider = get_provider("http://localhost:8548".to_string()) + .expect("Failed to get ethereum websocket provider"); + let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider) + .await + .is_ok(); - let result = get_user_nonce_from_ethereum_retryable( - &payment_service, - &payment_service, - dummy_user_addr, - ) - .await; + assert_eq!(result, true); + // kill Anvil + } + let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; assert!(matches!(result, Err(RetryError::Transient(_)))); - } - #[tokio::test] - async fn test_get_gas_price_retryable() { - let (_anvil, _payment_service) = setup_anvil(8551u16).await; - let eth_rpc_provider = ethers::prelude::Provider::connect("ws://localhost:8551") - .await - .expect("Failed to get ethereum websocket provider"); + // restart Anvil + let (_anvil, _payment_service) = setup_anvil(8548u16).await; let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider) .await .is_ok(); @@ -229,20 +254,6 @@ mod test { assert_eq!(result, true); } - #[tokio::test] - async fn test_get_gas_price_retryable_kill_anvil() { - let eth_rpc_provider; - { - // Kill anvil - let (_anvil, _payment_service) = setup_anvil(8552u16).await; - eth_rpc_provider = ethers::prelude::Provider::connect("ws://localhost:8552") - .await - .expect("Failed to get ethereum websocket provider"); - } - let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; - assert!(matches!(result, Err(RetryError::Transient(_)))); - } - #[tokio::test] async fn test_send_response_retryable() { let listener = TcpListener::bind("localhost:8553").await.unwrap(); From b9f186bf9dbf093ee92bd375750d4a1965b622e0 Mon Sep 17 00:00:00 2001 From: "Mariano A. Nicolini" Date: Tue, 22 Oct 2024 12:25:03 -0300 Subject: [PATCH 29/90] Update batcher/aligned-batcher/src/lib.rs --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 4275bb176c..9edc50906c 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -280,7 +280,7 @@ impl Batcher { self: Arc, ) -> Result<(), RetryError> { let eth_ws_provider = Provider::connect(&self.eth_ws_url).await.map_err(|e| { - warn!("Failed to get ethereum websocket provider"); + warn!("Failed to instantiate Ethereum websocket provider"); RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) })?; From a7aad12ce1b3d4eb47872dc6cdf199e8ed08107b Mon Sep 17 00:00:00 2001 From: "Mariano A. Nicolini" Date: Tue, 22 Oct 2024 12:26:09 -0300 Subject: [PATCH 30/90] Update batcher/aligned-batcher/src/lib.rs --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 9edc50906c..53154def0f 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -288,7 +288,7 @@ impl Batcher { Provider::connect(&self.eth_ws_url_fallback) .await .map_err(|e| { - warn!("Failed to get fallback ethereum websocket provider"); + warn!("Failed to instantiate fallback Ethereum websocket provider"); RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) })?; From 32fd9a61af3f4fcf0797e50cc43b87b4c0e4be88 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 22 Oct 2024 12:26:04 -0300 Subject: [PATCH 31/90] refactor: use assert_eq in unit test --- batcher/aligned-batcher/src/retry.rs | 31 ++++++++++++---------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index eb60e84c7f..b1df4ac9fc 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -126,7 +126,7 @@ mod test { .await .unwrap(); - assert!(balance == U256::zero()); + assert_eq!(balance, U256::zero()); // Kill anvil } @@ -141,7 +141,7 @@ mod test { .await .unwrap(); - assert!(balance == U256::zero()); + assert_eq!(balance, U256::zero()); } #[tokio::test] @@ -161,7 +161,7 @@ mod test { .await .unwrap(); - assert!(unlocked == false); + assert_eq!(unlocked, false); // Kill Anvil } @@ -183,7 +183,7 @@ mod test { .await .unwrap(); - assert!(unlocked == false); + assert_eq!(unlocked, false); } #[tokio::test] @@ -202,7 +202,7 @@ mod test { .await .unwrap(); - assert!(nonce == U256::zero()); + assert_eq!(nonce, U256::zero()); // Kill Anvil } @@ -225,7 +225,7 @@ mod test { .await .unwrap(); - assert!(nonce == U256::zero()); + assert_eq!(nonce, U256::zero()); } #[tokio::test] @@ -235,11 +235,9 @@ mod test { let (_anvil, _payment_service) = setup_anvil(8548u16).await; eth_rpc_provider = get_provider("http://localhost:8548".to_string()) .expect("Failed to get ethereum websocket provider"); - let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider) - .await - .is_ok(); + let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; - assert_eq!(result, true); + assert!(result.is_ok()); // kill Anvil } let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; @@ -247,11 +245,9 @@ mod test { // restart Anvil let (_anvil, _payment_service) = setup_anvil(8548u16).await; - let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider) - .await - .is_ok(); + let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; - assert_eq!(result, true); + assert!(result.is_ok()); } #[tokio::test] @@ -282,10 +278,9 @@ mod test { let outgoing = Arc::new(RwLock::new(outgoing)); let message = "Some message".to_string(); - let result = connection::send_response_retryable(&outgoing, message.clone().into_bytes()) - .await - .is_ok(); - assert!(result); + let result = + connection::send_response_retryable(&outgoing, message.clone().into_bytes()).await; + assert!(result.is_ok()); client_handle.await.unwrap() } } From 9a57cbdaae7ed12322438d19da8259c389a697bf Mon Sep 17 00:00:00 2001 From: "Mariano A. Nicolini" Date: Tue, 22 Oct 2024 12:34:47 -0300 Subject: [PATCH 32/90] Update batcher/aligned-batcher/src/retry.rs --- batcher/aligned-batcher/src/retry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index b1df4ac9fc..d8bfce5147 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -145,7 +145,7 @@ mod test { } #[tokio::test] - async fn test_user_balance_is_unlocked_retryable_kill_anvil() { + async fn test_user_balance_is_unlocked_retryable() { let payment_service; let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); From 9894bbd248ed94319b26d813e87b99a076d42796 Mon Sep 17 00:00:00 2001 From: "Mariano A. Nicolini" Date: Tue, 22 Oct 2024 12:35:27 -0300 Subject: [PATCH 33/90] Update batcher/aligned-batcher/src/retry.rs --- batcher/aligned-batcher/src/retry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index d8bfce5147..077dab5fa5 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -187,7 +187,7 @@ mod test { } #[tokio::test] - async fn test_get_user_nonce_retryable_kill_anvil() { + async fn test_get_user_nonce_retryable() { let payment_service; let dummy_user_addr = Address::from_str("0x8969c5eD335650692Bc04293B07F5BF2e7A673C0").unwrap(); From 4827352d28803310e8b0f1b096ef630c72f971ed Mon Sep 17 00:00:00 2001 From: "Mariano A. Nicolini" Date: Tue, 22 Oct 2024 12:57:37 -0300 Subject: [PATCH 34/90] Update batcher/aligned-batcher/src/retry.rs --- batcher/aligned-batcher/src/retry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 077dab5fa5..282fd37826 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -229,7 +229,7 @@ mod test { } #[tokio::test] - async fn test_get_gas_price_retryable_kill_anvil() { + async fn test_get_gas_price_retryable() { let eth_rpc_provider; { let (_anvil, _payment_service) = setup_anvil(8548u16).await; From 5f040424067597a70c5ace94f5a98dd568633e33 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 22 Oct 2024 14:20:19 -0300 Subject: [PATCH 35/90] refactor: move retryable functions to retry --- batcher/aligned-batcher/src/connection.rs | 27 +--- .../src/eth/payment_service.rs | 60 +-------- batcher/aligned-batcher/src/eth/utils.rs | 27 +--- batcher/aligned-batcher/src/lib.rs | 25 ++-- batcher/aligned-batcher/src/retry.rs | 116 ++++++++++++++++-- 5 files changed, 125 insertions(+), 130 deletions(-) diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index e4f9f62eaf..755d7c7259 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -1,7 +1,10 @@ use std::sync::Arc; use crate::{ - retry::{retry_function, RetryError, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY}, + retry::{ + retry_function, send_response_retryable, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, + DEFAULT_MIN_DELAY, + }, types::{batch_queue::BatchQueueEntry, errors::BatcherError}, }; use aligned_sdk::{ @@ -13,10 +16,7 @@ use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use log::{error, info}; use serde::Serialize; use tokio::{net::TcpStream, sync::RwLock}; -use tokio_tungstenite::{ - tungstenite::{Error, Message}, - WebSocketStream, -}; +use tokio_tungstenite::{tungstenite::Message, WebSocketStream}; pub(crate) type WsMessageSink = Arc, Message>>>; @@ -74,20 +74,3 @@ pub(crate) async fn send_response( error!("Error while sending batch inclusion data response: {}", e); } } - -pub async fn send_response_retryable( - ws_sink: &Arc, Message>>>, - serialized_response: Vec, -) -> Result<(), RetryError> { - let sending_result = ws_sink - .write() - .await - .send(Message::binary(serialized_response)) - .await; - - match sending_result { - Err(Error::AlreadyClosed) => Err(RetryError::Permanent(Error::AlreadyClosed)), - Err(e) => Err(RetryError::Transient(e)), - Ok(_) => Ok(()), - } -} diff --git a/batcher/aligned-batcher/src/eth/payment_service.rs b/batcher/aligned-batcher/src/eth/payment_service.rs index a3f5ef2962..b73326450b 100644 --- a/batcher/aligned-batcher/src/eth/payment_service.rs +++ b/batcher/aligned-batcher/src/eth/payment_service.rs @@ -5,9 +5,9 @@ use aligned_sdk::eth::batcher_payment_service::BatcherPaymentServiceContract; use ethers::prelude::k256::ecdsa::SigningKey; use ethers::prelude::*; use gas_escalator::{Frequency, GeometricGasPrice}; -use log::{info, warn}; +use log::info; -use crate::{config::ECDSAConfig, retry::RetryError, types::errors::BatcherSendError}; +use crate::{config::ECDSAConfig, types::errors::BatcherSendError}; use super::utils::{GAS_ESCALATOR_INTERVAL, GAS_MULTIPLIER}; @@ -71,25 +71,6 @@ pub async fn get_batcher_payment_service( Ok(payment_service) } -pub async fn user_balance_is_unlocked_retryable( - payment_service: &BatcherPaymentService, - payment_service_fallback: &BatcherPaymentService, - addr: &Address, -) -> Result> { - if let Ok(unlock_block) = payment_service.user_unlock_block(*addr).call().await { - return Ok(unlock_block != U256::zero()); - } - if let Ok(unlock_block) = payment_service_fallback - .user_unlock_block(*addr) - .call() - .await - { - return Ok(unlock_block != U256::zero()); - } - warn!("Failed to get user locking state {:?}", addr); - Err(RetryError::Transient(())) -} - pub async fn try_create_new_task( batch_merkle_root: [u8; 32], batch_data_pointer: String, @@ -120,40 +101,3 @@ pub async fn try_create_new_task( .map_err(|err| BatcherSendError::UnknownError(err.to_string()))? .ok_or(BatcherSendError::ReceiptNotFound) } - -pub async fn get_user_balance_retryable( - payment_service: &BatcherPaymentService, - payment_service_fallback: &BatcherPaymentService, - addr: &Address, -) -> Result> { - if let Ok(balance) = payment_service.user_balances(*addr).call().await { - return Ok(balance); - }; - - payment_service_fallback - .user_balances(*addr) - .call() - .await - .map_err(|e| { - warn!("Failed to get balance for address {:?}. Error: {e}", addr); - RetryError::Transient(e.to_string()) - }) -} - -pub async fn get_user_nonce_from_ethereum_retryable( - payment_service: &BatcherPaymentService, - payment_service_fallback: &BatcherPaymentService, - addr: Address, -) -> Result> { - if let Ok(nonce) = payment_service.user_nonces(addr).call().await { - return Ok(nonce); - } - payment_service_fallback - .user_nonces(addr) - .call() - .await - .map_err(|e| { - warn!("Error getting user nonce: {e}"); - RetryError::Transient(e.to_string()) - }) -} diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 86ca660822..8b9e6f3f64 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -1,13 +1,6 @@ use std::str::FromStr; -use ethers::{ - providers::{Http, Provider}, - types::U256, -}; -use log::warn; - -use crate::retry::RetryError; -use ethers::prelude::Middleware; +use ethers::providers::{Http, Provider}; pub(crate) const GAS_MULTIPLIER: f64 = 1.125; // Multiplier for the gas price for gas escalator pub(crate) const GAS_ESCALATOR_INTERVAL: u64 = 12; // Time in seconds between gas escalations @@ -17,21 +10,3 @@ pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error .map_err(|e| anyhow::Error::msg(format!("Failed to create provider: {}", e)))?; Ok(Provider::new(provider)) } - -pub async fn get_gas_price_retryable( - eth_ws_provider: &Provider, - eth_ws_provider_fallback: &Provider, -) -> Result> { - if let Ok(gas_price) = eth_ws_provider - .get_gas_price() - .await - .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) - { - return Ok(gas_price); - } - - eth_ws_provider_fallback.get_gas_price().await.map_err(|e| { - warn!("Failed to get fallback gas price: {e:?}"); - RetryError::Transient(e.to_string()) - }) -} diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 53154def0f..20795a6344 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -3,10 +3,13 @@ use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use eth::service_manager::ServiceManager; -use eth::utils::get_gas_price_retryable; use ethers::contract::ContractError; use ethers::signers::Signer; -use retry::{retry_function, RetryError, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY}; +use retry::{ + get_gas_price_retryable, get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, + retry_function, user_balance_is_unlocked_retryable, RetryError, DEFAULT_FACTOR, + DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY, +}; use types::batch_state::BatchState; use types::user_state::UserState; @@ -29,9 +32,7 @@ use aligned_sdk::core::types::{ use aws_sdk_s3::client::Client as S3Client; use eth::payment_service::{ - get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, try_create_new_task, - user_balance_is_unlocked_retryable, BatcherPaymentService, CreateNewTaskFeeParams, - SignerMiddlewareT, + try_create_new_task, BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT, }; use ethers::prelude::{Http, Middleware, Provider}; use ethers::types::{Address, Signature, TransactionReceipt, U256}; @@ -1333,7 +1334,7 @@ impl Batcher { /// Checks if the user's balance is unlocked for a given address using exponential backoff. /// Returns `false` if an error occurs during the retries. async fn user_balance_is_unlocked(&self, addr: &Address) -> bool { - match retry_function( + let Ok(unlocked) = retry_function( || { user_balance_is_unlocked_retryable( &self.payment_service, @@ -1346,13 +1347,11 @@ impl Batcher { DEFAULT_MAX_TIMES, ) .await - { - Ok(result) => result, - Err(_) => { - warn!("Could not get user locking state."); - false - } - } + else { + warn!("Could not get user locking state."); + return false; + }; + unlocked } /// Gets the current gas price from Ethereum using exponential backoff. diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 282fd37826..53076ffb53 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -1,6 +1,16 @@ use backon::ExponentialBuilder; use backon::Retryable; +use ethers::prelude::*; +use futures_util::{stream::SplitSink, SinkExt}; +use log::warn; +use std::sync::Arc; use std::{future::Future, time::Duration}; +use tokio::net::TcpStream; +use tokio::sync::RwLock; +use tokio_tungstenite::tungstenite; +use tokio_tungstenite::WebSocketStream; + +use crate::eth::payment_service::BatcherPaymentService; pub const DEFAULT_MIN_DELAY: u64 = 2000; pub const DEFAULT_MAX_TIMES: usize = 3; @@ -54,20 +64,105 @@ where .await } +pub async fn get_user_balance_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: &Address, +) -> Result> { + if let Ok(balance) = payment_service.user_balances(*addr).call().await { + return Ok(balance); + }; + + payment_service_fallback + .user_balances(*addr) + .call() + .await + .map_err(|e| { + warn!("Failed to get balance for address {:?}. Error: {e}", addr); + RetryError::Transient(e.to_string()) + }) +} + +pub async fn get_user_nonce_from_ethereum_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: Address, +) -> Result> { + if let Ok(nonce) = payment_service.user_nonces(addr).call().await { + return Ok(nonce); + } + payment_service_fallback + .user_nonces(addr) + .call() + .await + .map_err(|e| { + warn!("Error getting user nonce: {e}"); + RetryError::Transient(e.to_string()) + }) +} + +pub async fn user_balance_is_unlocked_retryable( + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, + addr: &Address, +) -> Result> { + if let Ok(unlock_block) = payment_service.user_unlock_block(*addr).call().await { + return Ok(unlock_block != U256::zero()); + } + if let Ok(unlock_block) = payment_service_fallback + .user_unlock_block(*addr) + .call() + .await + { + return Ok(unlock_block != U256::zero()); + } + warn!("Failed to get user locking state {:?}", addr); + Err(RetryError::Transient(())) +} + +pub async fn get_gas_price_retryable( + eth_ws_provider: &Provider, + eth_ws_provider_fallback: &Provider, +) -> Result> { + if let Ok(gas_price) = eth_ws_provider + .get_gas_price() + .await + .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) + { + return Ok(gas_price); + } + + eth_ws_provider_fallback.get_gas_price().await.map_err(|e| { + warn!("Failed to get fallback gas price: {e:?}"); + RetryError::Transient(e.to_string()) + }) +} + +pub async fn send_response_retryable( + ws_sink: &Arc, tungstenite::Message>>>, + serialized_response: Vec, +) -> Result<(), RetryError> { + let sending_result = ws_sink + .write() + .await + .send(tungstenite::Message::binary(serialized_response)) + .await; + + match sending_result { + Err(tungstenite::Error::AlreadyClosed) => { + Err(RetryError::Permanent(tungstenite::Error::AlreadyClosed)) + } + Err(e) => Err(RetryError::Transient(e)), + Ok(_) => Ok(()), + } +} + #[cfg(test)] mod test { use super::*; use crate::{ config::ECDSAConfig, - connection, - eth::{ - self, get_provider, - payment_service::{ - get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, - user_balance_is_unlocked_retryable, BatcherPaymentService, - }, - utils::get_gas_price_retryable, - }, + eth::{self, get_provider, payment_service::BatcherPaymentService}, }; use ethers::{ contract::abigen, @@ -278,8 +373,7 @@ mod test { let outgoing = Arc::new(RwLock::new(outgoing)); let message = "Some message".to_string(); - let result = - connection::send_response_retryable(&outgoing, message.clone().into_bytes()).await; + let result = send_response_retryable(&outgoing, message.clone().into_bytes()).await; assert!(result.is_ok()); client_handle.await.unwrap() } From cbe15a7eca114ac78e4eb36d6ea5c016cbd18eff Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 22 Oct 2024 14:53:49 -0300 Subject: [PATCH 36/90] refactor: remove retry to send_response --- batcher/aligned-batcher/src/connection.rs | 41 ++++++-------- batcher/aligned-batcher/src/retry.rs | 65 +---------------------- 2 files changed, 17 insertions(+), 89 deletions(-) diff --git a/batcher/aligned-batcher/src/connection.rs b/batcher/aligned-batcher/src/connection.rs index 755d7c7259..501604788e 100644 --- a/batcher/aligned-batcher/src/connection.rs +++ b/batcher/aligned-batcher/src/connection.rs @@ -1,12 +1,6 @@ use std::sync::Arc; -use crate::{ - retry::{ - retry_function, send_response_retryable, DEFAULT_FACTOR, DEFAULT_MAX_TIMES, - DEFAULT_MIN_DELAY, - }, - types::{batch_queue::BatchQueueEntry, errors::BatcherError}, -}; +use crate::types::{batch_queue::BatchQueueEntry, errors::BatcherError}; use aligned_sdk::{ communication::serialization::cbor_serialize, core::types::{BatchInclusionData, ResponseMessage, VerificationCommitmentBatch}, @@ -16,7 +10,10 @@ use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use log::{error, info}; use serde::Serialize; use tokio::{net::TcpStream, sync::RwLock}; -use tokio_tungstenite::{tungstenite::Message, WebSocketStream}; +use tokio_tungstenite::{ + tungstenite::{Error, Message}, + WebSocketStream, +}; pub(crate) type WsMessageSink = Arc, Message>>>; @@ -35,7 +32,17 @@ pub(crate) async fn send_batch_inclusion_data_responses( return Err(BatcherError::WsSinkEmpty); }; - send_response(ws_sink, serialized_response).await; + let sending_result = ws_sink + .write() + .await + .send(Message::binary(serialized_response)) + .await; + + match sending_result { + Err(Error::AlreadyClosed) => (), + Err(e) => error!("Error while sending batch inclusion data response: {}", e), + Ok(_) => (), + } info!("Response sent"); } @@ -58,19 +65,3 @@ pub(crate) async fn send_message(ws_conn_sink: WsMessageSink, mess Err(e) => error!("Error while serializing message: {}", e), } } - -pub(crate) async fn send_response( - ws_sink: &Arc, Message>>>, - serialized_response: Vec, -) { - if let Err(e) = retry_function( - || send_response_retryable(ws_sink, serialized_response.clone()), - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, - ) - .await - { - error!("Error while sending batch inclusion data response: {}", e); - } -} diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 53076ffb53..361cec4ad7 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -1,14 +1,8 @@ use backon::ExponentialBuilder; use backon::Retryable; use ethers::prelude::*; -use futures_util::{stream::SplitSink, SinkExt}; use log::warn; -use std::sync::Arc; use std::{future::Future, time::Duration}; -use tokio::net::TcpStream; -use tokio::sync::RwLock; -use tokio_tungstenite::tungstenite; -use tokio_tungstenite::WebSocketStream; use crate::eth::payment_service::BatcherPaymentService; @@ -138,25 +132,6 @@ pub async fn get_gas_price_retryable( }) } -pub async fn send_response_retryable( - ws_sink: &Arc, tungstenite::Message>>>, - serialized_response: Vec, -) -> Result<(), RetryError> { - let sending_result = ws_sink - .write() - .await - .send(tungstenite::Message::binary(serialized_response)) - .await; - - match sending_result { - Err(tungstenite::Error::AlreadyClosed) => { - Err(RetryError::Permanent(tungstenite::Error::AlreadyClosed)) - } - Err(e) => Err(RetryError::Transient(e)), - Ok(_) => Ok(()), - } -} - #[cfg(test)] mod test { use super::*; @@ -169,12 +144,7 @@ mod test { types::{Address, U256}, utils::{Anvil, AnvilInstance}, }; - use futures_util::StreamExt; - use std::{str::FromStr, sync::Arc}; - use tokio::{ - net::{TcpListener, TcpStream}, - sync::RwLock, - }; + use std::str::FromStr; abigen!( BatcherPaymentServiceContract, @@ -344,37 +314,4 @@ mod test { assert!(result.is_ok()); } - - #[tokio::test] - async fn test_send_response_retryable() { - let listener = TcpListener::bind("localhost:8553").await.unwrap(); - - let client_handle = tokio::spawn(async move { - let stream = TcpStream::connect("localhost:8553") - .await - .expect("Failed to connect"); - - let (mut ws_stream, _) = tokio_tungstenite::client_async("ws://localhost:8553", stream) - .await - .expect("WebSocket handshake failed"); - - // Read the response from the server - if let None = ws_stream.next().await { - panic!("Failed to receive valid WebSocket response"); - } - }); - - let (raw_stream, _) = listener - .accept() - .await - .expect("Failed to accept connection"); - let ws_stream = tokio_tungstenite::accept_async(raw_stream).await.unwrap(); - let (outgoing, _incoming) = ws_stream.split(); - let outgoing = Arc::new(RwLock::new(outgoing)); - let message = "Some message".to_string(); - - let result = send_response_retryable(&outgoing, message.clone().into_bytes()).await; - assert!(result.is_ok()); - client_handle.await.unwrap() - } } From 85176a97f5bd46a0625eab083306ba05e80a36b1 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 22 Oct 2024 15:42:05 -0300 Subject: [PATCH 37/90] refactor: use a large number to listen_blocks retry --- batcher/aligned-batcher/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 20795a6344..bb7fb2b986 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -60,6 +60,8 @@ pub mod sp1; pub mod types; mod zk_utils; +pub const LISTEN_NEW_BLOCKS_MAX_TIMES: usize = usize::MAX; + pub struct Batcher { s3_client: S3Client, s3_bucket_name: String, @@ -271,7 +273,7 @@ impl Batcher { }, DEFAULT_MIN_DELAY, DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + LISTEN_NEW_BLOCKS_MAX_TIMES, ) .await .map_err(|e| e.inner()) From 153b87d583be26e9bd9a467f2f5fb58cb8c54fc6 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 22 Oct 2024 16:03:31 -0300 Subject: [PATCH 38/90] fix: test --- batcher/aligned-batcher/src/retry.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index 361cec4ad7..b493efe08f 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -239,7 +239,7 @@ mod test { assert!(matches!(result, Err(RetryError::Transient(_)))); // restart Anvil - let (_anvil, payment_service) = setup_anvil(8546u16).await; + let (_anvil, _) = setup_anvil(8546u16).await; let unlocked = user_balance_is_unlocked_retryable( &payment_service, &payment_service, @@ -280,7 +280,7 @@ mod test { assert!(matches!(result, Err(RetryError::Transient(_)))); // restart Anvil - let (_anvil, payment_service) = setup_anvil(8547u16).await; + let (_anvil, _) = setup_anvil(8547u16).await; let nonce = get_user_nonce_from_ethereum_retryable( &payment_service, @@ -309,7 +309,7 @@ mod test { assert!(matches!(result, Err(RetryError::Transient(_)))); // restart Anvil - let (_anvil, _payment_service) = setup_anvil(8548u16).await; + let (_anvil, _) = setup_anvil(8548u16).await; let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; assert!(result.is_ok()); From 0c7b2b3aa584ae5630a5db98c8d8062dfe922822 Mon Sep 17 00:00:00 2001 From: "Mariano A. Nicolini" Date: Tue, 22 Oct 2024 16:48:57 -0300 Subject: [PATCH 39/90] Update batcher/aligned-batcher/src/lib.rs --- batcher/aligned-batcher/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index bb7fb2b986..49c6504e9f 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1357,7 +1357,6 @@ impl Batcher { } /// Gets the current gas price from Ethereum using exponential backoff. - /// Returns `None` if the gas price couldn't be returned async fn get_gas_price(&self) -> Result { retry_function( || get_gas_price_retryable(&self.eth_http_provider, &self.eth_http_provider_fallback), From 13ade7c834d9bc5d56cacb701b9403b303add86a Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:22:41 -0300 Subject: [PATCH 40/90] chore: remove commented line --- config-files/config-aggregator.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 83ff4e49d3..dc258e2724 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -24,7 +24,6 @@ bls: # batch_size_interval: 10 # max_proof_size: 67108864 # 64 MiB # max_batch_size: 268435456 # 256 MiB -# eth_ws_reconnects: 99999999999999 # pre_verification_is_enabled: true ## Aggregator Configurations From c9c6e70aea6435d0c3202985f555590fa9a79f18 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 23 Oct 2024 16:19:14 -0300 Subject: [PATCH 41/90] refactor: move retry constants to constants.rs --- batcher/aligned-batcher/src/lib.rs | 11 +++++------ batcher/aligned-batcher/src/retry.rs | 4 ---- batcher/aligned-sdk/src/core/constants.rs | 4 ++++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 49c6504e9f..0968eb14e2 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -7,8 +7,7 @@ use ethers::contract::ContractError; use ethers::signers::Signer; use retry::{ get_gas_price_retryable, get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, - retry_function, user_balance_is_unlocked_retryable, RetryError, DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY, + retry_function, user_balance_is_unlocked_retryable, RetryError, }; use types::batch_state::BatchState; use types::user_state::UserState; @@ -20,9 +19,9 @@ use std::sync::Arc; use aligned_sdk::core::constants::{ ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CONSTANT_GAS_COST, - DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_MAX_FEE_PER_PROOF, - GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, PERCENTAGE_DIVIDER, - RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, + DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_FACTOR, DEFAULT_MAX_FEE_PER_PROOF, + DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY, GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, + PERCENTAGE_DIVIDER, RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, }; use aligned_sdk::core::types::{ ClientMessage, NoncedVerificationData, ProofInvalidReason, ProvingSystemId, ResponseMessage, @@ -1391,7 +1390,7 @@ impl Batcher { DEFAULT_MAX_TIMES, ) .await - .map_err(|_| BatcherError::BatchUploadError("Error uploading batch to s3".to_string())) + .map_err(|e| BatcherError::BatchUploadError(e.to_string())) } async fn upload_batch_to_s3_retryable( diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry.rs index b493efe08f..5c8fea9d20 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry.rs @@ -6,10 +6,6 @@ use std::{future::Future, time::Duration}; use crate::eth::payment_service::BatcherPaymentService; -pub const DEFAULT_MIN_DELAY: u64 = 2000; -pub const DEFAULT_MAX_TIMES: usize = 3; -pub const DEFAULT_FACTOR: f32 = 2.0; - #[derive(Debug)] pub enum RetryError { Transient(E), diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index 6b63dc4918..991023a767 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -8,6 +8,10 @@ pub const CONSTANT_GAS_COST: u128 = pub const DEFAULT_MAX_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 100_000_000_000; // gas_price = 100 Gwei = 0.0000001 ether (high gas price) pub const MIN_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 100_000_000; // gas_price = 0.1 Gwei = 0.0000000001 ether (low gas price) +/// retry +pub const DEFAULT_MIN_DELAY: u64 = 2000; +pub const DEFAULT_MAX_TIMES: usize = 3; +pub const DEFAULT_FACTOR: f32 = 2.0; // % modifiers: (100% is x1, 10% is x0.1, 1000% is x10) pub const RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER: u128 = 250; // fee_for_aggregator -> respondToTaskFeeLimit modifier From bb2d464db8ec28d64aa4fd3ba23e07ed5e9d6eab Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 23 Oct 2024 16:35:31 -0300 Subject: [PATCH 42/90] refactor: move retryable functions to batcher_retry --- batcher/aligned-batcher/src/lib.rs | 5 +- .../src/{retry.rs => retry/batcher_retry.rs} | 53 +------------------ batcher/aligned-batcher/src/retry/mod.rs | 53 +++++++++++++++++++ 3 files changed, 57 insertions(+), 54 deletions(-) rename batcher/aligned-batcher/src/{retry.rs => retry/batcher_retry.rs} (85%) create mode 100644 batcher/aligned-batcher/src/retry/mod.rs diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 0968eb14e2..04886f63d1 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -5,10 +5,11 @@ use dotenvy::dotenv; use eth::service_manager::ServiceManager; use ethers::contract::ContractError; use ethers::signers::Signer; -use retry::{ +use retry::batcher_retry::{ get_gas_price_retryable, get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, - retry_function, user_balance_is_unlocked_retryable, RetryError, + user_balance_is_unlocked_retryable, }; +use retry::{retry_function, RetryError}; use types::batch_state::BatchState; use types::user_state::UserState; diff --git a/batcher/aligned-batcher/src/retry.rs b/batcher/aligned-batcher/src/retry/batcher_retry.rs similarity index 85% rename from batcher/aligned-batcher/src/retry.rs rename to batcher/aligned-batcher/src/retry/batcher_retry.rs index 5c8fea9d20..6ca6fd39cf 100644 --- a/batcher/aligned-batcher/src/retry.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retry.rs @@ -1,58 +1,7 @@ -use backon::ExponentialBuilder; -use backon::Retryable; use ethers::prelude::*; use log::warn; -use std::{future::Future, time::Duration}; -use crate::eth::payment_service::BatcherPaymentService; - -#[derive(Debug)] -pub enum RetryError { - Transient(E), - Permanent(E), -} - -impl std::fmt::Display for RetryError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - RetryError::Transient(e) => write!(f, "{}", e), - RetryError::Permanent(e) => write!(f, "{}", e), - } - } -} - -impl RetryError { - pub fn inner(self) -> E { - match self { - RetryError::Transient(e) => e, - RetryError::Permanent(e) => e, - } - } -} - -impl std::error::Error for RetryError where E: std::fmt::Debug {} - -pub async fn retry_function( - function: FutureFn, - min_delay: u64, - factor: f32, - max_times: usize, -) -> Result> -where - Fut: Future>>, - FutureFn: FnMut() -> Fut, -{ - let backoff = ExponentialBuilder::default() - .with_min_delay(Duration::from_millis(min_delay)) - .with_max_times(max_times) - .with_factor(factor); - - function - .retry(backoff) - .sleep(tokio::time::sleep) - .when(|e| matches!(e, RetryError::Transient(_))) - .await -} +use crate::{eth::payment_service::BatcherPaymentService, retry::RetryError}; pub async fn get_user_balance_retryable( payment_service: &BatcherPaymentService, diff --git a/batcher/aligned-batcher/src/retry/mod.rs b/batcher/aligned-batcher/src/retry/mod.rs new file mode 100644 index 0000000000..9df8115d06 --- /dev/null +++ b/batcher/aligned-batcher/src/retry/mod.rs @@ -0,0 +1,53 @@ +pub mod batcher_retry; + +use backon::ExponentialBuilder; +use backon::Retryable; +use std::{future::Future, time::Duration}; + +#[derive(Debug)] +pub enum RetryError { + Transient(E), + Permanent(E), +} + +impl std::fmt::Display for RetryError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + RetryError::Transient(e) => write!(f, "{}", e), + RetryError::Permanent(e) => write!(f, "{}", e), + } + } +} + +impl RetryError { + pub fn inner(self) -> E { + match self { + RetryError::Transient(e) => e, + RetryError::Permanent(e) => e, + } + } +} + +impl std::error::Error for RetryError where E: std::fmt::Debug {} + +pub async fn retry_function( + function: FutureFn, + min_delay: u64, + factor: f32, + max_times: usize, +) -> Result> +where + Fut: Future>>, + FutureFn: FnMut() -> Fut, +{ + let backoff = ExponentialBuilder::default() + .with_min_delay(Duration::from_millis(min_delay)) + .with_max_times(max_times) + .with_factor(factor); + + function + .retry(backoff) + .sleep(tokio::time::sleep) + .when(|e| matches!(e, RetryError::Transient(_))) + .await +} From 53e92045543750c97f02e289882a3fcd7d806665 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 23 Oct 2024 17:24:55 -0300 Subject: [PATCH 43/90] refactor: remove harcoded address from test --- batcher/aligned-batcher/src/lib.rs | 41 ++++++++++--------- .../src/retry/batcher_retry.rs | 8 +++- batcher/aligned-sdk/src/core/constants.rs | 9 ++-- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 04886f63d1..376d55acda 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -20,9 +20,10 @@ use std::sync::Arc; use aligned_sdk::core::constants::{ ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CONSTANT_GAS_COST, - DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_FACTOR, DEFAULT_MAX_FEE_PER_PROOF, - DEFAULT_MAX_TIMES, DEFAULT_MIN_DELAY, GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, - PERCENTAGE_DIVIDER, RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, + DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_FEE_PER_PROOF, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, + GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, PERCENTAGE_DIVIDER, + RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, }; use aligned_sdk::core::types::{ ClientMessage, NoncedVerificationData, ProofInvalidReason, ProvingSystemId, ResponseMessage, @@ -271,8 +272,8 @@ impl Batcher { let app = self.clone(); async move { app.listen_new_blocks_retryable().await } }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, LISTEN_NEW_BLOCKS_MAX_TIMES, ) .await @@ -780,9 +781,9 @@ impl Batcher { addr, ) }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, ) .await } @@ -1325,9 +1326,9 @@ impl Batcher { addr, ) }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, ) .await .ok() @@ -1344,9 +1345,9 @@ impl Batcher { addr, ) }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, ) .await else { @@ -1360,9 +1361,9 @@ impl Batcher { async fn get_gas_price(&self) -> Result { retry_function( || get_gas_price_retryable(&self.eth_http_provider, &self.eth_http_provider_fallback), - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, ) .await .map_err(|e| { @@ -1386,9 +1387,9 @@ impl Batcher { &self.s3_bucket_name, ) }, - DEFAULT_MIN_DELAY, - DEFAULT_FACTOR, - DEFAULT_MAX_TIMES, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, ) .await .map_err(|e| BatcherError::BatchUploadError(e.to_string())) diff --git a/batcher/aligned-batcher/src/retry/batcher_retry.rs b/batcher/aligned-batcher/src/retry/batcher_retry.rs index 6ca6fd39cf..9dcd73dad6 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retry.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retry.rs @@ -81,7 +81,7 @@ pub async fn get_gas_price_retryable( mod test { use super::*; use crate::{ - config::ECDSAConfig, + config::{ContractDeploymentOutput, ECDSAConfig}, eth::{self, get_provider, payment_service::BatcherPaymentService}, }; use ethers::{ @@ -106,7 +106,11 @@ mod test { let eth_rpc_provider = eth::get_provider(format!("http://localhost:{}", port)) .expect("Failed to get provider"); - let payment_service_addr = String::from("0x7969c5eD335650692Bc04293B07F5BF2e7A673C0"); + let deployment_output = ContractDeploymentOutput::new( + "../../contracts/script/output/devnet/alignedlayer_deployment_output.json".to_string(), + ); + + let payment_service_addr = deployment_output.addresses.batcher_payment_service.clone(); let payment_service = eth::payment_service::get_batcher_payment_service( eth_rpc_provider, diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index 991023a767..8ffc5fcc5c 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -8,10 +8,6 @@ pub const CONSTANT_GAS_COST: u128 = pub const DEFAULT_MAX_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 100_000_000_000; // gas_price = 100 Gwei = 0.0000001 ether (high gas price) pub const MIN_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 100_000_000; // gas_price = 0.1 Gwei = 0.0000000001 ether (low gas price) -/// retry -pub const DEFAULT_MIN_DELAY: u64 = 2000; -pub const DEFAULT_MAX_TIMES: usize = 3; -pub const DEFAULT_FACTOR: f32 = 2.0; // % modifiers: (100% is x1, 10% is x0.1, 1000% is x10) pub const RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER: u128 = 250; // fee_for_aggregator -> respondToTaskFeeLimit modifier @@ -27,3 +23,8 @@ pub const MAX_FEE_BATCH_PROOF_NUMBER: usize = 32; /// Estimated number of proofs for batch submission. /// This corresponds to the number of proofs to compute for a default max_fee. pub const MAX_FEE_DEFAULT_PROOF_NUMBER: usize = 10; + +/// retry +pub const DEFAULT_MIN_RETRY_DELAY: u64 = 2000; +pub const DEFAULT_MAX_RETRIES: usize = 3; +pub const DEFAULT_BACKOFF_FACTOR: f32 = 2.0; From 1f0465b6269a193e775c465bb7edda180a13b3b3 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 09:27:38 -0300 Subject: [PATCH 44/90] fix: add retry and timeout to create_new_task --- .../src/eth/payment_service.rs | 56 +-------- batcher/aligned-batcher/src/eth/utils.rs | 27 +++++ batcher/aligned-batcher/src/lib.rs | 108 ++++++++++++------ .../src/retry/batcher_retry.rs | 79 ++++++++++++- batcher/aligned-batcher/src/types/errors.rs | 17 --- 5 files changed, 172 insertions(+), 115 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/payment_service.rs b/batcher/aligned-batcher/src/eth/payment_service.rs index b73326450b..aaddbff12a 100644 --- a/batcher/aligned-batcher/src/eth/payment_service.rs +++ b/batcher/aligned-batcher/src/eth/payment_service.rs @@ -4,12 +4,6 @@ use std::sync::Arc; use aligned_sdk::eth::batcher_payment_service::BatcherPaymentServiceContract; use ethers::prelude::k256::ecdsa::SigningKey; use ethers::prelude::*; -use gas_escalator::{Frequency, GeometricGasPrice}; -use log::info; - -use crate::{config::ECDSAConfig, types::errors::BatcherSendError}; - -use super::utils::{GAS_ESCALATOR_INTERVAL, GAS_MULTIPLIER}; #[derive(Debug, Clone, EthEvent)] pub struct BatchVerified { @@ -46,58 +40,10 @@ impl CreateNewTaskFeeParams { } pub async fn get_batcher_payment_service( - provider: Provider, - ecdsa_config: ECDSAConfig, + signer: Arc, contract_address: String, ) -> Result { - let chain_id = provider.get_chainid().await?; - - let escalator = GeometricGasPrice::new(GAS_MULTIPLIER, GAS_ESCALATOR_INTERVAL, None::); - - let provider = GasEscalatorMiddleware::new(provider, escalator, Frequency::PerBlock); - - // get private key from keystore - let wallet = Wallet::decrypt_keystore( - &ecdsa_config.private_key_store_path, - &ecdsa_config.private_key_store_password, - )? - .with_chain_id(chain_id.as_u64()); - - let signer = Arc::new(SignerMiddleware::new(provider, wallet)); - let payment_service = BatcherPaymentService::new(H160::from_str(contract_address.as_str())?, signer); - Ok(payment_service) } - -pub async fn try_create_new_task( - batch_merkle_root: [u8; 32], - batch_data_pointer: String, - proofs_submitters: Vec
, - fee_params: CreateNewTaskFeeParams, - payment_service: &BatcherPaymentService, -) -> Result { - let call = payment_service - .create_new_task( - batch_merkle_root, - batch_data_pointer, - proofs_submitters, - fee_params.fee_for_aggregator, - fee_params.fee_per_proof, - fee_params.respond_to_task_fee_limit, - ) - .gas_price(fee_params.gas_price); - - info!("Creating task for: {}", hex::encode(batch_merkle_root)); - - let pending_tx = call.send().await.map_err(|err| match err { - ContractError::Revert(err) => BatcherSendError::TransactionReverted(err.to_string()), - _ => BatcherSendError::UnknownError(err.to_string()), - })?; - - pending_tx - .await - .map_err(|err| BatcherSendError::UnknownError(err.to_string()))? - .ok_or(BatcherSendError::ReceiptNotFound) -} diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 8b9e6f3f64..a057deb096 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -1,7 +1,13 @@ use std::str::FromStr; +use std::sync::Arc; +use crate::config::ECDSAConfig; +use ethers::prelude::*; use ethers::providers::{Http, Provider}; +use super::payment_service::SignerMiddlewareT; +use gas_escalator::{Frequency, GeometricGasPrice}; + pub(crate) const GAS_MULTIPLIER: f64 = 1.125; // Multiplier for the gas price for gas escalator pub(crate) const GAS_ESCALATOR_INTERVAL: u64 = 12; // Time in seconds between gas escalations @@ -10,3 +16,24 @@ pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error .map_err(|e| anyhow::Error::msg(format!("Failed to create provider: {}", e)))?; Ok(Provider::new(provider)) } + +pub async fn get_batcher_signer( + provider: Provider, + ecdsa_config: ECDSAConfig, +) -> Result, anyhow::Error> { + let chain_id = provider.get_chainid().await?; + + let escalator = GeometricGasPrice::new(GAS_MULTIPLIER, GAS_ESCALATOR_INTERVAL, None::); + + let provider = GasEscalatorMiddleware::new(provider, escalator, Frequency::PerBlock); + + // get private key from keystore + let wallet = Wallet::decrypt_keystore( + &ecdsa_config.private_key_store_path, + &ecdsa_config.private_key_store_password, + )? + .with_chain_id(chain_id.as_u64()); + + let signer = Arc::new(SignerMiddleware::new(provider, wallet)); + Ok(signer) +} diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 376d55acda..1a2d3a7ca0 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -3,11 +3,12 @@ use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use eth::service_manager::ServiceManager; +use eth::utils::get_batcher_signer; use ethers::contract::ContractError; use ethers::signers::Signer; use retry::batcher_retry::{ - get_gas_price_retryable, get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, - user_balance_is_unlocked_retryable, + create_new_task_retryable, get_gas_price_retryable, get_user_balance_retryable, + get_user_nonce_from_ethereum_retryable, user_balance_is_unlocked_retryable, }; use retry::{retry_function, RetryError}; use types::batch_state::BatchState; @@ -32,11 +33,9 @@ use aligned_sdk::core::types::{ }; use aws_sdk_s3::client::Client as S3Client; -use eth::payment_service::{ - try_create_new_task, BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT, -}; +use eth::payment_service::{BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT}; use ethers::prelude::{Http, Middleware, Provider}; -use ethers::types::{Address, Signature, TransactionReceipt, U256}; +use ethers::types::{Address, Signature, TransactionReceipt, TransactionRequest, U256}; use futures_util::{future, SinkExt, StreamExt, TryStreamExt}; use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use lambdaworks_crypto::merkle_tree::traits::IsMerkleTreeBackend; @@ -45,7 +44,7 @@ use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{Mutex, MutexGuard, RwLock}; use tokio_tungstenite::tungstenite::{Error, Message}; use types::batch_queue::{self, BatchQueueEntry, BatchQueueEntryPriority}; -use types::errors::{BatcherError, BatcherSendError}; +use types::errors::BatcherError; use crate::config::{ConfigFromYaml, ContractDeploymentOutput}; @@ -74,6 +73,8 @@ pub struct Batcher { chain_id: U256, payment_service: BatcherPaymentService, payment_service_fallback: BatcherPaymentService, + batcher_signer: Arc, + batcher_signer_fallback: Arc, service_manager: ServiceManager, service_manager_fallback: ServiceManager, batch_state: Mutex, @@ -155,17 +156,24 @@ impl Batcher { } }; + let batcher_signer = get_batcher_signer(eth_rpc_provider, config.ecdsa.clone()) + .await + .expect("Failed to get Batcher signer"); + + let batcher_signer_fallback = + get_batcher_signer(eth_rpc_provider_fallback, config.ecdsa.clone()) + .await + .expect("Failed to get Batcher signer fallback"); + let payment_service = eth::payment_service::get_batcher_payment_service( - eth_rpc_provider, - config.ecdsa.clone(), + batcher_signer.clone(), deployment_output.addresses.batcher_payment_service.clone(), ) .await .expect("Failed to get Batcher Payment Service contract"); let payment_service_fallback = eth::payment_service::get_batcher_payment_service( - eth_rpc_provider_fallback, - config.ecdsa.clone(), + batcher_signer_fallback.clone(), deployment_output.addresses.batcher_payment_service, ) .await @@ -235,6 +243,8 @@ impl Batcher { chain_id, payment_service, payment_service_fallback, + batcher_signer, + batcher_signer_fallback, service_manager, service_manager_fallback, max_block_interval: config.batcher.block_interval, @@ -1189,39 +1199,63 @@ impl Batcher { proof_submitters: Vec
, fee_params: CreateNewTaskFeeParams, ) -> Result { - info!("Creating task for: 0x{}", hex::encode(batch_merkle_root)); - - match try_create_new_task( - batch_merkle_root, - batch_data_pointer.clone(), - proof_submitters.clone(), - fee_params.clone(), - &self.payment_service, - ) - .await - { - Ok(receipt) => Ok(receipt), - Err(BatcherSendError::TransactionReverted(err)) => { - // Since transaction was reverted, we don't want to retry with fallback. - warn!("Transaction reverted {:?}", err); - - Err(BatcherError::TransactionSendError) - } - Err(_) => { - let receipt = try_create_new_task( + let result = retry_function( + || { + create_new_task_retryable( batch_merkle_root, - batch_data_pointer, - proof_submitters, - fee_params, + batch_data_pointer.clone(), + proof_submitters.clone(), + fee_params.clone(), + &self.payment_service, &self.payment_service_fallback, ) - .await?; - - Ok(receipt) + }, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, + ) + .await; + match result { + Ok(receipt) => Ok(receipt), + Err(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) => { + self.cancel_created_task(fee_params.gas_price).await; + Err(BatcherError::ReceiptNotFoundError) } + Err(_) => Err(BatcherError::TransactionSendError), } } + pub async fn cancel_created_task(&self, gas_price: U256) { + let from_address = self.batcher_signer.address(); + let current_nonce = self + .batcher_signer + .get_transaction_count(from_address, None) + .await + .unwrap(); + + let tx = TransactionRequest::new() + .to("0xRECIPIENT_ADDRESS") + .value(U256::zero()) + .nonce(current_nonce) + .gas_price(gas_price * 125 / 100); + + // Override transaction + if self + .batcher_signer + .send_transaction(tx.clone(), None) + .await + .is_err() + { + if let Err(e) = self + .batcher_signer_fallback + .send_transaction(tx, None) + .await + { + warn!("Couldn cancel created task: {e}"); + } + }; + } + /// Only relevant for testing and for users to easily use Aligned fn is_nonpaying(&self, addr: &Address) -> bool { self.non_paying_config diff --git a/batcher/aligned-batcher/src/retry/batcher_retry.rs b/batcher/aligned-batcher/src/retry/batcher_retry.rs index 9dcd73dad6..dceb25d6f3 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retry.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retry.rs @@ -1,7 +1,11 @@ use ethers::prelude::*; -use log::warn; +use log::{info, warn}; -use crate::{eth::payment_service::BatcherPaymentService, retry::RetryError}; +use crate::{ + eth::payment_service::{BatcherPaymentService, CreateNewTaskFeeParams}, + retry::RetryError, + types::errors::BatcherError, +}; pub async fn get_user_balance_retryable( payment_service: &BatcherPaymentService, @@ -77,12 +81,71 @@ pub async fn get_gas_price_retryable( }) } +pub async fn create_new_task_retryable( + batch_merkle_root: [u8; 32], + batch_data_pointer: String, + proofs_submitters: Vec
, + fee_params: CreateNewTaskFeeParams, + payment_service: &BatcherPaymentService, + payment_service_fallback: &BatcherPaymentService, +) -> Result> { + info!("Creating task for: 0x{}", hex::encode(batch_merkle_root)); + let call_fallback; + let call = payment_service + .create_new_task( + batch_merkle_root, + batch_data_pointer.clone(), + proofs_submitters.clone(), + fee_params.fee_for_aggregator, + fee_params.fee_per_proof, + fee_params.respond_to_task_fee_limit, + ) + .gas_price(fee_params.gas_price); + + let pending_tx = match call.send().await { + Ok(pending_tx) => pending_tx, + Err(ContractError::Revert(err)) => { + // Since transaction was reverted, we don't want to retry with fallback. + warn!("Transaction reverted {:?}", err); + return Err(RetryError::Permanent(BatcherError::TransactionSendError)); + } + _ => { + call_fallback = payment_service_fallback + .create_new_task( + batch_merkle_root, + batch_data_pointer, + proofs_submitters, + fee_params.fee_for_aggregator, + fee_params.fee_per_proof, + fee_params.respond_to_task_fee_limit, + ) + .gas_price(fee_params.gas_price); + match call_fallback.send().await { + Ok(pending_tx) => pending_tx, + Err(ContractError::Revert(err)) => { + warn!("Transaction reverted {:?}", err); + return Err(RetryError::Permanent(BatcherError::TransactionSendError)); + } + _ => return Err(RetryError::Transient(BatcherError::TransactionSendError)), + } + } + }; + + pending_tx + .retries(5) + .await + .map_err(|_| RetryError::Transient(BatcherError::TransactionSendError))? + .ok_or(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) +} + #[cfg(test)] mod test { use super::*; use crate::{ config::{ContractDeploymentOutput, ECDSAConfig}, - eth::{self, get_provider, payment_service::BatcherPaymentService}, + eth::{ + self, get_provider, payment_service::BatcherPaymentService, utils::get_batcher_signer, + }, }; use ethers::{ contract::abigen, @@ -112,17 +175,21 @@ mod test { let payment_service_addr = deployment_output.addresses.batcher_payment_service.clone(); - let payment_service = eth::payment_service::get_batcher_payment_service( + let batcher_signer = get_batcher_signer( eth_rpc_provider, ECDSAConfig { private_key_store_path: "../../config-files/anvil.batcher.ecdsa.key.json" .to_string(), private_key_store_password: "".to_string(), }, - payment_service_addr, ) .await - .expect("Failed to get Batcher Payment Service contract"); + .unwrap(); + + let payment_service = + eth::payment_service::get_batcher_payment_service(batcher_signer, payment_service_addr) + .await + .expect("Failed to get Batcher Payment Service contract"); (anvil, payment_service) } diff --git a/batcher/aligned-batcher/src/types/errors.rs b/batcher/aligned-batcher/src/types/errors.rs index 9305e97582..7e0785c386 100644 --- a/batcher/aligned-batcher/src/types/errors.rs +++ b/batcher/aligned-batcher/src/types/errors.rs @@ -101,20 +101,3 @@ impl fmt::Debug for BatcherError { } } } - -#[derive(Debug)] -pub enum BatcherSendError { - TransactionReverted(String), - ReceiptNotFound, - UnknownError(String), -} - -impl From for BatcherError { - fn from(value: BatcherSendError) -> Self { - match value { - BatcherSendError::TransactionReverted(_) => BatcherError::TransactionSendError, - BatcherSendError::ReceiptNotFound => BatcherError::ReceiptNotFoundError, - BatcherSendError::UnknownError(err) => BatcherError::TaskCreationError(err), - } - } -} From 84fe36d4293e91543ce7dab6d1527a641701077b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 14:46:14 -0300 Subject: [PATCH 45/90] fix: use tokio timeout instead of retries --- batcher/aligned-batcher/src/lib.rs | 6 +++--- batcher/aligned-batcher/src/retry/batcher_retry.rs | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 1a2d3a7ca0..231230c75c 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1234,12 +1234,12 @@ impl Batcher { .unwrap(); let tx = TransactionRequest::new() - .to("0xRECIPIENT_ADDRESS") + .to(from_address) .value(U256::zero()) .nonce(current_nonce) .gas_price(gas_price * 125 / 100); - // Override transaction + info!("Canceling task"); if self .batcher_signer .send_transaction(tx.clone(), None) @@ -1251,7 +1251,7 @@ impl Batcher { .send_transaction(tx, None) .await { - warn!("Couldn cancel created task: {e}"); + warn!("Could not cancel created task: {e}"); } }; } diff --git a/batcher/aligned-batcher/src/retry/batcher_retry.rs b/batcher/aligned-batcher/src/retry/batcher_retry.rs index dceb25d6f3..c3f7890adc 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retry.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retry.rs @@ -1,5 +1,8 @@ +use std::time::Duration; + use ethers::prelude::*; use log::{info, warn}; +use tokio::time::timeout; use crate::{ eth::payment_service::{BatcherPaymentService, CreateNewTaskFeeParams}, @@ -131,9 +134,9 @@ pub async fn create_new_task_retryable( } }; - pending_tx - .retries(5) + timeout(Duration::from_millis(1), pending_tx) .await + .map_err(|_| RetryError::Permanent(BatcherError::ReceiptNotFoundError))? .map_err(|_| RetryError::Transient(BatcherError::TransactionSendError))? .ok_or(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) } From 630a0f3789f0dfd6da053d26900db88d58b028b0 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 15:22:01 -0300 Subject: [PATCH 46/90] refactor: use delay constant --- batcher/aligned-batcher/src/retry/batcher_retry.rs | 3 ++- batcher/aligned-sdk/src/core/constants.rs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/retry/batcher_retry.rs b/batcher/aligned-batcher/src/retry/batcher_retry.rs index c3f7890adc..5e527b4f5d 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retry.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retry.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use aligned_sdk::core::constants::BATCH_INCLUSION_DELAY; use ethers::prelude::*; use log::{info, warn}; use tokio::time::timeout; @@ -134,7 +135,7 @@ pub async fn create_new_task_retryable( } }; - timeout(Duration::from_millis(1), pending_tx) + timeout(Duration::from_millis(BATCH_INCLUSION_DELAY), pending_tx) .await .map_err(|_| RetryError::Permanent(BatcherError::ReceiptNotFoundError))? .map_err(|_| RetryError::Transient(BatcherError::TransactionSendError))? diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index 8ffc5fcc5c..8d4572a115 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -1,4 +1,5 @@ /// Batcher /// +pub const BATCH_INCLUSION_DELAY: u64 = 36000; // 3 blocks pub const AGGREGATOR_GAS_COST: u128 = 400_000; pub const BATCHER_SUBMISSION_BASE_GAS_COST: u128 = 125_000; pub const ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF: u128 = 13_000; From 229f202d5b5b54944997a35ff4c176686c5be14a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:24:04 -0300 Subject: [PATCH 47/90] refactor: rename batcher_retry to batcher_retryables --- batcher/aligned-batcher/src/lib.rs | 4 ++-- .../src/retry/{batcher_retry.rs => batcher_retryables.rs} | 4 ++++ batcher/aligned-batcher/src/retry/mod.rs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) rename batcher/aligned-batcher/src/retry/{batcher_retry.rs => batcher_retryables.rs} (99%) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 231230c75c..1b2820f94a 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -6,7 +6,7 @@ use eth::service_manager::ServiceManager; use eth::utils::get_batcher_signer; use ethers::contract::ContractError; use ethers::signers::Signer; -use retry::batcher_retry::{ +use retry::batcher_retryables::{ create_new_task_retryable, get_gas_price_retryable, get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, user_balance_is_unlocked_retryable, }; @@ -1237,7 +1237,7 @@ impl Batcher { .to(from_address) .value(U256::zero()) .nonce(current_nonce) - .gas_price(gas_price * 125 / 100); + .gas_price(gas_price * 150 / 100); info!("Canceling task"); if self diff --git a/batcher/aligned-batcher/src/retry/batcher_retry.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs similarity index 99% rename from batcher/aligned-batcher/src/retry/batcher_retry.rs rename to batcher/aligned-batcher/src/retry/batcher_retryables.rs index 5e527b4f5d..1c92b577c5 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retry.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -197,6 +197,7 @@ mod test { (anvil, payment_service) } + #[ignore] #[tokio::test] async fn test_get_user_balance_retryable() { let payment_service; @@ -229,6 +230,7 @@ mod test { assert_eq!(balance, U256::zero()); } + #[ignore] #[tokio::test] async fn test_user_balance_is_unlocked_retryable() { let payment_service; @@ -271,6 +273,7 @@ mod test { assert_eq!(unlocked, false); } + #[ignore] #[tokio::test] async fn test_get_user_nonce_retryable() { let payment_service; @@ -313,6 +316,7 @@ mod test { assert_eq!(nonce, U256::zero()); } + #[ignore] #[tokio::test] async fn test_get_gas_price_retryable() { let eth_rpc_provider; diff --git a/batcher/aligned-batcher/src/retry/mod.rs b/batcher/aligned-batcher/src/retry/mod.rs index 9df8115d06..0af5c3bab3 100644 --- a/batcher/aligned-batcher/src/retry/mod.rs +++ b/batcher/aligned-batcher/src/retry/mod.rs @@ -1,4 +1,4 @@ -pub mod batcher_retry; +pub mod batcher_retryables; use backon::ExponentialBuilder; use backon::Retryable; From 9e488c9cc762e7a5051ff61881fc9331c3f357fc Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:25:26 -0300 Subject: [PATCH 48/90] refactor: adjust default constants --- batcher/aligned-sdk/src/core/constants.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index 8d4572a115..beac2791b4 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -26,6 +26,6 @@ pub const MAX_FEE_BATCH_PROOF_NUMBER: usize = 32; pub const MAX_FEE_DEFAULT_PROOF_NUMBER: usize = 10; /// retry -pub const DEFAULT_MIN_RETRY_DELAY: u64 = 2000; -pub const DEFAULT_MAX_RETRIES: usize = 3; +pub const DEFAULT_MIN_RETRY_DELAY: u64 = 500; // milliseconds +pub const DEFAULT_MAX_RETRIES: usize = 5; pub const DEFAULT_BACKOFF_FACTOR: f32 = 2.0; From a07871ac1b74c9c9219c2d83f4443e1b6421d064 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:29:51 -0300 Subject: [PATCH 49/90] refactor: rename batcher_retry to batcher_retryables --- batcher/aligned-batcher/src/lib.rs | 2 +- .../src/retry/{batcher_retry.rs => batcher_retryables.rs} | 4 ++++ batcher/aligned-batcher/src/retry/mod.rs | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) rename batcher/aligned-batcher/src/retry/{batcher_retry.rs => batcher_retryables.rs} (99%) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 376d55acda..a153d7e48b 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -5,7 +5,7 @@ use dotenvy::dotenv; use eth::service_manager::ServiceManager; use ethers::contract::ContractError; use ethers::signers::Signer; -use retry::batcher_retry::{ +use retry::batcher_retryables::{ get_gas_price_retryable, get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, user_balance_is_unlocked_retryable, }; diff --git a/batcher/aligned-batcher/src/retry/batcher_retry.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs similarity index 99% rename from batcher/aligned-batcher/src/retry/batcher_retry.rs rename to batcher/aligned-batcher/src/retry/batcher_retryables.rs index 9dcd73dad6..ff527f1730 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retry.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -126,6 +126,7 @@ mod test { (anvil, payment_service) } + #[ignore] #[tokio::test] async fn test_get_user_balance_retryable() { let payment_service; @@ -158,6 +159,7 @@ mod test { assert_eq!(balance, U256::zero()); } + #[ignore] #[tokio::test] async fn test_user_balance_is_unlocked_retryable() { let payment_service; @@ -200,6 +202,7 @@ mod test { assert_eq!(unlocked, false); } + #[ignore] #[tokio::test] async fn test_get_user_nonce_retryable() { let payment_service; @@ -242,6 +245,7 @@ mod test { assert_eq!(nonce, U256::zero()); } + #[ignore] #[tokio::test] async fn test_get_gas_price_retryable() { let eth_rpc_provider; diff --git a/batcher/aligned-batcher/src/retry/mod.rs b/batcher/aligned-batcher/src/retry/mod.rs index 9df8115d06..0af5c3bab3 100644 --- a/batcher/aligned-batcher/src/retry/mod.rs +++ b/batcher/aligned-batcher/src/retry/mod.rs @@ -1,4 +1,4 @@ -pub mod batcher_retry; +pub mod batcher_retryables; use backon::ExponentialBuilder; use backon::Retryable; From 3dec0bdf55973fec6ed182bc79d7e66b040ab67d Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:30:38 -0300 Subject: [PATCH 50/90] refactor: adjust retry constants --- batcher/aligned-sdk/src/core/constants.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index 8ffc5fcc5c..f546c70e72 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -25,6 +25,6 @@ pub const MAX_FEE_BATCH_PROOF_NUMBER: usize = 32; pub const MAX_FEE_DEFAULT_PROOF_NUMBER: usize = 10; /// retry -pub const DEFAULT_MIN_RETRY_DELAY: u64 = 2000; -pub const DEFAULT_MAX_RETRIES: usize = 3; +pub const DEFAULT_MIN_RETRY_DELAY: u64 = 500; // milliseconds +pub const DEFAULT_MAX_RETRIES: usize = 5; pub const DEFAULT_BACKOFF_FACTOR: f32 = 2.0; From 09cdabf39b893fda2890c1ab02b0cb9865de4aa6 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:36:59 -0300 Subject: [PATCH 51/90] fix: test CI --- batcher/aligned-batcher/src/retry/batcher_retryables.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 1c92b577c5..5e527b4f5d 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -197,7 +197,6 @@ mod test { (anvil, payment_service) } - #[ignore] #[tokio::test] async fn test_get_user_balance_retryable() { let payment_service; @@ -230,7 +229,6 @@ mod test { assert_eq!(balance, U256::zero()); } - #[ignore] #[tokio::test] async fn test_user_balance_is_unlocked_retryable() { let payment_service; @@ -273,7 +271,6 @@ mod test { assert_eq!(unlocked, false); } - #[ignore] #[tokio::test] async fn test_get_user_nonce_retryable() { let payment_service; @@ -316,7 +313,6 @@ mod test { assert_eq!(nonce, U256::zero()); } - #[ignore] #[tokio::test] async fn test_get_gas_price_retryable() { let eth_rpc_provider; From f83105027262584cb86338ab27dbd15a9432b30a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:38:18 -0300 Subject: [PATCH 52/90] fix: test CI --- .github/workflows/build-and-test-rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-rust.yml b/.github/workflows/build-and-test-rust.yml index 3a1633891a..05d9a4f94b 100644 --- a/.github/workflows/build-and-test-rust.yml +++ b/.github/workflows/build-and-test-rust.yml @@ -54,7 +54,7 @@ jobs: uses: actions/checkout@v4 - name: foundry-toolchain - uses: foundry-rs/foundry-toolchain@v1.2.0 + uses: foundry-rs/foundry-toolchain@v1 - name: Cache Rust dependencies uses: actions/cache@v3 From 7d6e9d3f8dbb9c907e018d29b09e19fa088b7b13 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:47:20 -0300 Subject: [PATCH 53/90] refactor: removed ignored --- batcher/aligned-batcher/src/retry/batcher_retryables.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 1c92b577c5..5e527b4f5d 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -197,7 +197,6 @@ mod test { (anvil, payment_service) } - #[ignore] #[tokio::test] async fn test_get_user_balance_retryable() { let payment_service; @@ -230,7 +229,6 @@ mod test { assert_eq!(balance, U256::zero()); } - #[ignore] #[tokio::test] async fn test_user_balance_is_unlocked_retryable() { let payment_service; @@ -273,7 +271,6 @@ mod test { assert_eq!(unlocked, false); } - #[ignore] #[tokio::test] async fn test_get_user_nonce_retryable() { let payment_service; @@ -316,7 +313,6 @@ mod test { assert_eq!(nonce, U256::zero()); } - #[ignore] #[tokio::test] async fn test_get_gas_price_retryable() { let eth_rpc_provider; From 8349e2f5326a6f896364680fe64ba8c4cfbd40a9 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:54:31 -0300 Subject: [PATCH 54/90] refactor: remove foundry from the ci --- .github/workflows/build-and-test-rust.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-and-test-rust.yml b/.github/workflows/build-and-test-rust.yml index 05d9a4f94b..61d004da63 100644 --- a/.github/workflows/build-and-test-rust.yml +++ b/.github/workflows/build-and-test-rust.yml @@ -53,9 +53,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: foundry-toolchain - uses: foundry-rs/foundry-toolchain@v1 - - name: Cache Rust dependencies uses: actions/cache@v3 with: From 7b8a7a255e326674fcd2cd9a17dd53878dc4daa6 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:55:27 -0300 Subject: [PATCH 55/90] refactor: remove foundry from ci --- .github/workflows/build-and-test-rust.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-and-test-rust.yml b/.github/workflows/build-and-test-rust.yml index 3a1633891a..61d004da63 100644 --- a/.github/workflows/build-and-test-rust.yml +++ b/.github/workflows/build-and-test-rust.yml @@ -53,9 +53,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: foundry-toolchain - uses: foundry-rs/foundry-toolchain@v1.2.0 - - name: Cache Rust dependencies uses: actions/cache@v3 with: From 9c728a4d3f90970072d02ca79bf53a0867e6e1b2 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 16:57:11 -0300 Subject: [PATCH 56/90] refactor: formatting issues --- .github/workflows/build-and-test-rust.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-test-rust.yml b/.github/workflows/build-and-test-rust.yml index 61d004da63..2ba2a67392 100644 --- a/.github/workflows/build-and-test-rust.yml +++ b/.github/workflows/build-and-test-rust.yml @@ -52,7 +52,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - name: Cache Rust dependencies uses: actions/cache@v3 with: @@ -63,7 +62,6 @@ jobs: key: ${{ runner.os }}-rust-${{ hashFiles('**/Cargo.lock') }} restore-keys: | ${{ runner.os }}-rust- - - name: Run tests run: | cd batcher From c50635bec3daca5539fcbe0b06dfda2f9b9e5eef Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 17:08:42 -0300 Subject: [PATCH 57/90] refactor: ignore test --- batcher/aligned-batcher/src/retry/batcher_retryables.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 5e527b4f5d..1c92b577c5 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -197,6 +197,7 @@ mod test { (anvil, payment_service) } + #[ignore] #[tokio::test] async fn test_get_user_balance_retryable() { let payment_service; @@ -229,6 +230,7 @@ mod test { assert_eq!(balance, U256::zero()); } + #[ignore] #[tokio::test] async fn test_user_balance_is_unlocked_retryable() { let payment_service; @@ -271,6 +273,7 @@ mod test { assert_eq!(unlocked, false); } + #[ignore] #[tokio::test] async fn test_get_user_nonce_retryable() { let payment_service; @@ -313,6 +316,7 @@ mod test { assert_eq!(nonce, U256::zero()); } + #[ignore] #[tokio::test] async fn test_get_gas_price_retryable() { let eth_rpc_provider; From c36aca88fca7437c591521fff362bd4e09143a4c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 21:22:45 -0300 Subject: [PATCH 58/90] refactor: improve readability --- batcher/aligned-batcher/src/lib.rs | 15 +++++++++------ .../src/retry/batcher_retryables.rs | 11 +++++++++-- batcher/aligned-sdk/src/core/constants.rs | 1 + 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 1b2820f94a..52ab5e5231 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -23,8 +23,8 @@ use aligned_sdk::core::constants::{ ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CONSTANT_GAS_COST, DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_FEE_PER_PROOF, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, - GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, PERCENTAGE_DIVIDER, - RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, + GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, OVERRIDE_GAS_PRICE_MULTIPLIER, + PERCENTAGE_DIVIDER, RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, }; use aligned_sdk::core::types::{ ClientMessage, NoncedVerificationData, ProofInvalidReason, ProvingSystemId, ResponseMessage, @@ -1218,14 +1218,14 @@ impl Batcher { match result { Ok(receipt) => Ok(receipt), Err(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) => { - self.cancel_created_task(fee_params.gas_price).await; + self.override_created_task(fee_params.gas_price).await; Err(BatcherError::ReceiptNotFoundError) } Err(_) => Err(BatcherError::TransactionSendError), } } - pub async fn cancel_created_task(&self, gas_price: U256) { + pub async fn override_created_task(&self, task_gas_price: U256) { let from_address = self.batcher_signer.address(); let current_nonce = self .batcher_signer @@ -1233,13 +1233,16 @@ impl Batcher { .await .unwrap(); + let modified_gas_price = task_gas_price * U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) + / U256::from(PERCENTAGE_DIVIDER); + let tx = TransactionRequest::new() .to(from_address) .value(U256::zero()) .nonce(current_nonce) - .gas_price(gas_price * 150 / 100); + .gas_price(modified_gas_price); - info!("Canceling task"); + info!("Canceling created task"); if self .batcher_signer .send_transaction(tx.clone(), None) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 1c92b577c5..4fe44d77ed 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -135,10 +135,17 @@ pub async fn create_new_task_retryable( } }; + // timeout to prevent a deadlock while waiting for the transaction to be included in a block. timeout(Duration::from_millis(BATCH_INCLUSION_DELAY), pending_tx) .await - .map_err(|_| RetryError::Permanent(BatcherError::ReceiptNotFoundError))? - .map_err(|_| RetryError::Transient(BatcherError::TransactionSendError))? + .map_err(|e| { + warn!("Error while waiting for batch inclusion: {e}"); + RetryError::Permanent(BatcherError::ReceiptNotFoundError) + })? + .map_err(|e| { + warn!("Error while waiting for batch inclusion: {e}"); + RetryError::Transient(BatcherError::TransactionSendError) + })? .ok_or(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) } diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index beac2791b4..c7c9bcf3da 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -14,6 +14,7 @@ pub const MIN_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 1 pub const RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER: u128 = 250; // fee_for_aggregator -> respondToTaskFeeLimit modifier pub const DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER: u128 = 150; // feeForAggregator modifier pub const GAS_PRICE_PERCENTAGE_MULTIPLIER: u128 = 110; // gasPrice modifier +pub const OVERRIDE_GAS_PRICE_MULTIPLIER: u128 = 150; // gasPrice modifier to override previous transactions pub const PERCENTAGE_DIVIDER: u128 = 100; /// SDK /// From ed459764ef0669eb6e0ef31710486e52ae029a3b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 24 Oct 2024 21:57:56 -0300 Subject: [PATCH 59/90] refactor: improve logging --- batcher/aligned-batcher/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 52ab5e5231..e08e5bc476 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1257,6 +1257,7 @@ impl Batcher { warn!("Could not cancel created task: {e}"); } }; + info!("Task successfully canceled"); } /// Only relevant for testing and for users to easily use Aligned From eec74e15eb6bbf55e8f52a9ac234888341a8ab12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Avila=20Gast=C3=B3n?= <72628438+avilagaston9@users.noreply.github.com> Date: Fri, 25 Oct 2024 12:17:01 -0300 Subject: [PATCH 60/90] Update batcher/aligned-batcher/src/lib.rs Co-authored-by: Mariano A. Nicolini --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index e08e5bc476..555fbdc17e 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1226,7 +1226,7 @@ impl Batcher { } pub async fn override_created_task(&self, task_gas_price: U256) { - let from_address = self.batcher_signer.address(); + let batcher_addr = self.batcher_signer.address(); let current_nonce = self .batcher_signer .get_transaction_count(from_address, None) From 23cda020edf779e52df3aeeab32c928f8fc1c5da Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Sun, 27 Oct 2024 22:40:04 -0300 Subject: [PATCH 61/90] refactor: cance_create_task should retry infinitely --- .../src/eth/payment_service.rs | 3 +- .../src/eth/service_manager.rs | 14 +-- batcher/aligned-batcher/src/eth/utils.rs | 65 +++++++++-- batcher/aligned-batcher/src/lib.rs | 96 ++++++--------- .../src/retry/batcher_retryables.rs | 109 ++++++++++++++++-- batcher/aligned-sdk/src/core/constants.rs | 5 +- 6 files changed, 194 insertions(+), 98 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/payment_service.rs b/batcher/aligned-batcher/src/eth/payment_service.rs index aaddbff12a..2f14e2a096 100644 --- a/batcher/aligned-batcher/src/eth/payment_service.rs +++ b/batcher/aligned-batcher/src/eth/payment_service.rs @@ -10,8 +10,7 @@ pub struct BatchVerified { pub batch_merkle_root: [u8; 32], } -pub type SignerMiddlewareT = - SignerMiddleware>, Wallet>; +pub type SignerMiddlewareT = SignerMiddleware, Wallet>; pub type BatcherPaymentService = BatcherPaymentServiceContract; diff --git a/batcher/aligned-batcher/src/eth/service_manager.rs b/batcher/aligned-batcher/src/eth/service_manager.rs index dd61d0aeda..db610ca9d1 100644 --- a/batcher/aligned-batcher/src/eth/service_manager.rs +++ b/batcher/aligned-batcher/src/eth/service_manager.rs @@ -3,10 +3,7 @@ use std::{str::FromStr, sync::Arc}; use aligned_sdk::eth::aligned_service_manager::AlignedLayerServiceManagerContract; use ethers::{ core::k256::ecdsa::SigningKey, - middleware::{ - gas_escalator::{Frequency, GeometricGasPrice}, - GasEscalatorMiddleware, SignerMiddleware, - }, + middleware::SignerMiddleware, providers::{Http, Middleware, Provider}, signers::{Signer, Wallet}, types::H160, @@ -14,10 +11,7 @@ use ethers::{ use crate::config::ECDSAConfig; -use super::utils::{GAS_ESCALATOR_INTERVAL, GAS_MULTIPLIER}; - -pub type SignerMiddlewareT = - SignerMiddleware>, Wallet>; +pub type SignerMiddlewareT = SignerMiddleware, Wallet>; pub type ServiceManager = AlignedLayerServiceManagerContract; @@ -28,10 +22,6 @@ pub async fn get_service_manager( ) -> Result { let chain_id = provider.get_chainid().await?; - let escalator = GeometricGasPrice::new(GAS_MULTIPLIER, GAS_ESCALATOR_INTERVAL, None::); - - let provider = GasEscalatorMiddleware::new(provider, escalator, Frequency::PerBlock); - // get private key from keystore let wallet = Wallet::decrypt_keystore( &ecdsa_config.private_key_store_path, diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index a057deb096..66d1765eb5 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -2,14 +2,18 @@ use std::str::FromStr; use std::sync::Arc; use crate::config::ECDSAConfig; +use crate::retry::batcher_retryables::{get_current_nonce_retryable, get_gas_price_retryable}; +use crate::retry::retry_function; +use crate::types::errors::BatcherError; +use aligned_sdk::core::constants::{ + DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, + OVERRIDE_GAS_PRICE_MULTIPLIER, PERCENTAGE_DIVIDER, +}; use ethers::prelude::*; use ethers::providers::{Http, Provider}; +use log::error; use super::payment_service::SignerMiddlewareT; -use gas_escalator::{Frequency, GeometricGasPrice}; - -pub(crate) const GAS_MULTIPLIER: f64 = 1.125; // Multiplier for the gas price for gas escalator -pub(crate) const GAS_ESCALATOR_INTERVAL: u64 = 12; // Time in seconds between gas escalations pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error> { let provider = Http::from_str(eth_rpc_url.as_str()) @@ -23,10 +27,6 @@ pub async fn get_batcher_signer( ) -> Result, anyhow::Error> { let chain_id = provider.get_chainid().await?; - let escalator = GeometricGasPrice::new(GAS_MULTIPLIER, GAS_ESCALATOR_INTERVAL, None::); - - let provider = GasEscalatorMiddleware::new(provider, escalator, Frequency::PerBlock); - // get private key from keystore let wallet = Wallet::decrypt_keystore( &ecdsa_config.private_key_store_path, @@ -37,3 +37,52 @@ pub async fn get_batcher_signer( let signer = Arc::new(SignerMiddleware::new(provider, wallet)); Ok(signer) } + +pub fn get_bumped_gas_price( + previous_gas_price: U256, + current_gas_price: U256, + iteration: usize, +) -> U256 { + let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) + (20 * iteration); + let bumped_previous_gas_price = + previous_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); + + let bumped_current_gas_price = + current_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); + bumped_current_gas_price.max(bumped_previous_gas_price) +} +pub async fn get_current_nonce( + eth_http_provider: &Provider, + eth_http_provider_fallback: &Provider, + addr: H160, +) -> Result { + retry_function( + || get_current_nonce_retryable(eth_http_provider, eth_http_provider_fallback, addr), + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, + ) + .await + .map_err(|e| { + error!("Could't get nonce: {:?}", e); + e.to_string() + }) +} + +/// Gets the current gas price from Ethereum using exponential backoff. +pub async fn get_gas_price( + eth_http_provider: &Provider, + eth_http_provider_fallback: &Provider, +) -> Result { + retry_function( + || get_gas_price_retryable(eth_http_provider, eth_http_provider_fallback), + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + DEFAULT_MAX_RETRIES, + ) + .await + .map_err(|e| { + error!("Could't get gas price: {e}"); + BatcherError::GasPriceError + }) +} diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 555fbdc17e..2ae149bd21 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -3,11 +3,11 @@ use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use eth::service_manager::ServiceManager; -use eth::utils::get_batcher_signer; +use eth::utils::{get_batcher_signer, get_gas_price}; use ethers::contract::ContractError; use ethers::signers::Signer; use retry::batcher_retryables::{ - create_new_task_retryable, get_gas_price_retryable, get_user_balance_retryable, + cancel_create_new_task_retryable, create_new_task_retryable, get_user_balance_retryable, get_user_nonce_from_ethereum_retryable, user_balance_is_unlocked_retryable, }; use retry::{retry_function, RetryError}; @@ -15,16 +15,16 @@ use types::batch_state::BatchState; use types::user_state::UserState; use std::collections::HashMap; -use std::env; use std::net::SocketAddr; use std::sync::Arc; +use std::{env, usize}; use aligned_sdk::core::constants::{ ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CONSTANT_GAS_COST, DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_FEE_PER_PROOF, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, - GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, OVERRIDE_GAS_PRICE_MULTIPLIER, - PERCENTAGE_DIVIDER, RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, + GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, PERCENTAGE_DIVIDER, + RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, }; use aligned_sdk::core::types::{ ClientMessage, NoncedVerificationData, ProofInvalidReason, ProvingSystemId, ResponseMessage, @@ -34,8 +34,8 @@ use aligned_sdk::core::types::{ use aws_sdk_s3::client::Client as S3Client; use eth::payment_service::{BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT}; -use ethers::prelude::{Http, Middleware, Provider}; -use ethers::types::{Address, Signature, TransactionReceipt, TransactionRequest, U256}; +use ethers::prelude::{Middleware, Provider}; +use ethers::types::{Address, Signature, TransactionReceipt, U256}; use futures_util::{future, SinkExt, StreamExt, TryStreamExt}; use lambdaworks_crypto::merkle_tree::merkle::MerkleTree; use lambdaworks_crypto::merkle_tree::traits::IsMerkleTreeBackend; @@ -68,8 +68,6 @@ pub struct Batcher { download_endpoint: String, eth_ws_url: String, eth_ws_url_fallback: String, - eth_http_provider: Provider, - eth_http_provider_fallback: Provider, chain_id: U256, payment_service: BatcherPaymentService, payment_service_fallback: BatcherPaymentService, @@ -226,20 +224,12 @@ impl Batcher { } .expect("Failed to get disabled verifiers"); - let eth_http_provider = - eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get http provider"); - - let eth_http_provider_fallback = eth::get_provider(config.eth_rpc_url_fallback.clone()) - .expect("Failed to get fallback http provider"); - Self { s3_client, s3_bucket_name, download_endpoint, eth_ws_url: config.eth_ws_url, eth_ws_url_fallback: config.eth_ws_url_fallback, - eth_http_provider, - eth_http_provider_fallback, chain_id, payment_service, payment_service_fallback, @@ -1083,7 +1073,10 @@ impl Batcher { /// Receives new block numbers, checks if conditions are met for submission and /// finalizes the batch. async fn handle_new_block(&self, block_number: u64) -> Result<(), BatcherError> { - let gas_price_future = self.get_gas_price(); + let gas_price_future = get_gas_price( + self.batcher_signer.provider(), + self.batcher_signer_fallback.provider(), + ); let disabled_verifiers_future = self.disabled_verifiers(); let (gas_price, disable_verifiers) = @@ -1218,45 +1211,37 @@ impl Batcher { match result { Ok(receipt) => Ok(receipt), Err(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) => { - self.override_created_task(fee_params.gas_price).await; + self.cancel_create_new_task_tx(fee_params.gas_price).await; Err(BatcherError::ReceiptNotFoundError) } Err(_) => Err(BatcherError::TransactionSendError), } } - pub async fn override_created_task(&self, task_gas_price: U256) { - let batcher_addr = self.batcher_signer.address(); - let current_nonce = self - .batcher_signer - .get_transaction_count(from_address, None) - .await - .unwrap(); - - let modified_gas_price = task_gas_price * U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) - / U256::from(PERCENTAGE_DIVIDER); - - let tx = TransactionRequest::new() - .to(from_address) - .value(U256::zero()) - .nonce(current_nonce) - .gas_price(modified_gas_price); - + pub async fn cancel_create_new_task_tx(&self, old_task_gas_price: U256) { info!("Canceling created task"); - if self - .batcher_signer - .send_transaction(tx.clone(), None) - .await - .is_err() + let iteration = Arc::new(Mutex::new(0)); + let previous_gas_price = Arc::new(Mutex::new(old_task_gas_price)); + + if let Err(e) = retry_function( + || { + cancel_create_new_task_retryable( + &self.batcher_signer, + &self.batcher_signer_fallback, + iteration.clone(), + previous_gas_price.clone(), + ) + }, + DEFAULT_MIN_RETRY_DELAY, + DEFAULT_BACKOFF_FACTOR, + usize::MAX, + ) + .await { - if let Err(e) = self - .batcher_signer_fallback - .send_transaction(tx, None) - .await - { - warn!("Could not cancel created task: {e}"); - } + error!("Could not cancel created task: {e}"); + return; }; + info!("Task successfully canceled"); } @@ -1395,21 +1380,6 @@ impl Batcher { unlocked } - /// Gets the current gas price from Ethereum using exponential backoff. - async fn get_gas_price(&self) -> Result { - retry_function( - || get_gas_price_retryable(&self.eth_http_provider, &self.eth_http_provider_fallback), - DEFAULT_MIN_RETRY_DELAY, - DEFAULT_BACKOFF_FACTOR, - DEFAULT_MAX_RETRIES, - ) - .await - .map_err(|e| { - error!("Could't get gas price: {e}"); - BatcherError::GasPriceError - }) - } - /// Uploads the batch to s3 using exponential backoff. async fn upload_batch_to_s3( &self, diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 4fe44d77ed..0503f4977c 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -1,12 +1,15 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; -use aligned_sdk::core::constants::BATCH_INCLUSION_DELAY; +use aligned_sdk::core::constants::{BATCH_INCLUSION_DELAY, TRANSACTIONS_INCLUSION_DELAY}; use ethers::prelude::*; use log::{info, warn}; -use tokio::time::timeout; +use tokio::{sync::Mutex, time::timeout}; use crate::{ - eth::payment_service::{BatcherPaymentService, CreateNewTaskFeeParams}, + eth::{ + payment_service::{BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT}, + utils::{get_bumped_gas_price, get_current_nonce, get_gas_price}, + }, retry::RetryError, types::errors::BatcherError, }; @@ -48,6 +51,23 @@ pub async fn get_user_nonce_from_ethereum_retryable( }) } +pub async fn get_current_nonce_retryable( + eth_http_provider: &Provider, + eth_http_provider_fallback: &Provider, + addr: Address, +) -> Result> { + if let Ok(current_nonce) = eth_http_provider.get_transaction_count(addr, None).await { + return Ok(current_nonce); + } + eth_http_provider_fallback + .get_transaction_count(addr, None) + .await + .map_err(|e| { + warn!("Error getting user nonce: {e}"); + RetryError::Transient(e.to_string()) + }) +} + pub async fn user_balance_is_unlocked_retryable( payment_service: &BatcherPaymentService, payment_service_fallback: &BatcherPaymentService, @@ -68,10 +88,10 @@ pub async fn user_balance_is_unlocked_retryable( } pub async fn get_gas_price_retryable( - eth_ws_provider: &Provider, - eth_ws_provider_fallback: &Provider, + eth_http_provider: &Provider, + eth_http_provider_fallback: &Provider, ) -> Result> { - if let Ok(gas_price) = eth_ws_provider + if let Ok(gas_price) = eth_http_provider .get_gas_price() .await .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) @@ -79,10 +99,13 @@ pub async fn get_gas_price_retryable( return Ok(gas_price); } - eth_ws_provider_fallback.get_gas_price().await.map_err(|e| { - warn!("Failed to get fallback gas price: {e:?}"); - RetryError::Transient(e.to_string()) - }) + eth_http_provider_fallback + .get_gas_price() + .await + .map_err(|e| { + warn!("Failed to get fallback gas price: {e:?}"); + RetryError::Transient(e.to_string()) + }) } pub async fn create_new_task_retryable( @@ -149,6 +172,70 @@ pub async fn create_new_task_retryable( .ok_or(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) } +pub async fn cancel_create_new_task_retryable( + batcher_signer: &SignerMiddlewareT, + batcher_signer_fallback: &SignerMiddlewareT, + iteration: Arc>, + previous_gas_price: Arc>, +) -> Result> { + let mut iteration = iteration.lock().await; + let mut previous_gas_price = previous_gas_price.lock().await; + let batcher_addr = batcher_signer.address(); + + let current_nonce = get_current_nonce( + batcher_signer.provider(), + batcher_signer_fallback.provider(), + batcher_addr, + ) + .await + .map_err(|e| RetryError::Transient(e.to_string()))?; + + let current_gas_price = get_gas_price( + batcher_signer.provider(), + batcher_signer_fallback.provider(), + ) + .await + .map_err(|e| RetryError::Transient(format!("{:?}", e)))?; + + let bumped_gas_price = get_bumped_gas_price(*previous_gas_price, current_gas_price, *iteration); + + let tx = TransactionRequest::new() + .to(batcher_addr) + .value(U256::zero()) + .nonce(current_nonce) + .gas_price(bumped_gas_price); + + let pending_tx = match batcher_signer.send_transaction(tx.clone(), None).await { + Ok(pending_tx) => pending_tx, + Err(_) => batcher_signer_fallback + .send_transaction(tx.clone(), None) + .await + .map_err(|e| RetryError::Transient(e.to_string()))?, + }; + + // timeout to prevent a deadlock while waiting for the transaction to be included in a block. + timeout( + Duration::from_millis(TRANSACTIONS_INCLUSION_DELAY), + pending_tx, + ) + .await + .map_err(|e| { + *iteration += 1; + *previous_gas_price = bumped_gas_price; + warn!("Timeout while waiting for tx inclusion: {e}"); + RetryError::Transient(e.to_string()) + })? + .map_err(|e| { + warn!("Error while waiting for tx inclusion: {e}"); + RetryError::Transient(e.to_string()) + })? + .ok_or({ + *iteration += 1; + *previous_gas_price = bumped_gas_price; + RetryError::Transient("Receipt not found".to_string()) + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index c7c9bcf3da..d9ce840bce 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -1,5 +1,6 @@ /// Batcher /// -pub const BATCH_INCLUSION_DELAY: u64 = 36000; // 3 blocks +pub const BATCH_INCLUSION_DELAY: u64 = 36_000; // 3 blocks +pub const TRANSACTIONS_INCLUSION_DELAY: u64 = 14_000; // 1 block + 2 secs pub const AGGREGATOR_GAS_COST: u128 = 400_000; pub const BATCHER_SUBMISSION_BASE_GAS_COST: u128 = 125_000; pub const ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF: u128 = 13_000; @@ -14,7 +15,7 @@ pub const MIN_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 1 pub const RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER: u128 = 250; // fee_for_aggregator -> respondToTaskFeeLimit modifier pub const DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER: u128 = 150; // feeForAggregator modifier pub const GAS_PRICE_PERCENTAGE_MULTIPLIER: u128 = 110; // gasPrice modifier -pub const OVERRIDE_GAS_PRICE_MULTIPLIER: u128 = 150; // gasPrice modifier to override previous transactions +pub const OVERRIDE_GAS_PRICE_MULTIPLIER: u128 = 120; // gasPrice modifier to override previous transactions pub const PERCENTAGE_DIVIDER: u128 = 100; /// SDK /// From 6ec3022b9d0243f3f8876c68478ffe28c5e0344c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 10:12:11 -0300 Subject: [PATCH 62/90] test: add unit test to get_bumped_gas --- batcher/aligned-batcher/src/eth/utils.rs | 49 ++++++++++++++++++++++++ batcher/aligned-batcher/src/lib.rs | 4 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 66d1765eb5..1c7f94ccef 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -38,6 +38,8 @@ pub async fn get_batcher_signer( Ok(signer) } +/// Calculates an increased gas price for retrying a transaction override. +/// The gas price rises with each retry by applying a multiplier based on the iteration count. pub fn get_bumped_gas_price( previous_gas_price: U256, current_gas_price: U256, @@ -51,6 +53,7 @@ pub fn get_bumped_gas_price( current_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); bumped_current_gas_price.max(bumped_previous_gas_price) } + pub async fn get_current_nonce( eth_http_provider: &Provider, eth_http_provider_fallback: &Provider, @@ -86,3 +89,49 @@ pub async fn get_gas_price( BatcherError::GasPriceError }) } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_bumped_gas_price_initial_iteration() { + let previous_gas_price = U256::from(1000); + let current_gas_price = U256::from(1200); + let iteration = 0; + + let expected = U256::from(1440); // (1200 * (120 + 0)) / 100 + + assert_eq!( + get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), + expected + ); + } + + #[test] + fn test_get_bumped_gas_price_with_iteration() { + let previous_gas_price = U256::from(1000); + let current_gas_price = U256::from(1200); + let iteration = 2; + + let expected = U256::from(1920); // (1200 * (120 + 40) / 100 + + assert_eq!( + get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), + expected + ); + } + + #[test] + fn test_get_bumped_gas_price_previous_higher() { + let previous_gas_price = U256::from(1500); + let current_gas_price = U256::from(1200); + let iteration = 1; + + let expected = U256::from(2100); // (1500 * 140) / 100 + + assert_eq!( + get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), + expected + ); + } +} diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 2ae149bd21..41c3684423 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -15,9 +15,9 @@ use types::batch_state::BatchState; use types::user_state::UserState; use std::collections::HashMap; +use std::env; use std::net::SocketAddr; use std::sync::Arc; -use std::{env, usize}; use aligned_sdk::core::constants::{ ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CONSTANT_GAS_COST, @@ -1218,6 +1218,8 @@ impl Batcher { } } + // Sends a transaction to Ethereum with the same nonce as the previous one to override it. + // We use the maximum number of retries here because this transaction must be included. pub async fn cancel_create_new_task_tx(&self, old_task_gas_price: U256) { info!("Canceling created task"); let iteration = Arc::new(Mutex::new(0)); From 0456399f1c315e4e1b8efa41c2d7b18667af974a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 10:41:47 -0300 Subject: [PATCH 63/90] telemetry: add counter for canceled batches --- batcher/aligned-batcher/src/lib.rs | 1 + batcher/aligned-batcher/src/metrics.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 41c3684423..ab0e929fec 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1211,6 +1211,7 @@ impl Batcher { match result { Ok(receipt) => Ok(receipt), Err(RetryError::Permanent(BatcherError::ReceiptNotFoundError)) => { + self.metrics.canceled_batches.inc(); self.cancel_create_new_task_tx(fee_params.gas_price).await; Err(BatcherError::ReceiptNotFoundError) } diff --git a/batcher/aligned-batcher/src/metrics.rs b/batcher/aligned-batcher/src/metrics.rs index c9a08cd2a7..e3cf07d6fd 100644 --- a/batcher/aligned-batcher/src/metrics.rs +++ b/batcher/aligned-batcher/src/metrics.rs @@ -11,6 +11,7 @@ pub struct BatcherMetrics { pub received_proofs: IntCounter, pub sent_batches: IntCounter, pub reverted_batches: IntCounter, + pub canceled_batches: IntCounter, pub batcher_started: IntCounter, pub gas_price_used_on_latest_batch: IntGauge, pub broken_ws_connections: IntCounter, @@ -25,6 +26,8 @@ impl BatcherMetrics { let sent_batches = register_int_counter!(opts!("sent_batches", "Sent Batches"))?; let reverted_batches = register_int_counter!(opts!("reverted_batches", "Reverted Batches"))?; + let canceled_batches = + register_int_counter!(opts!("canceled_batches", "Canceled Batches"))?; let batcher_started = register_int_counter!(opts!("batcher_started", "Batcher Started"))?; let gas_price_used_on_latest_batch = register_int_gauge!(opts!("gas_price_used_on_latest_batch", "Gas Price"))?; @@ -37,6 +40,7 @@ impl BatcherMetrics { registry.register(Box::new(received_proofs.clone()))?; registry.register(Box::new(sent_batches.clone()))?; registry.register(Box::new(reverted_batches.clone()))?; + registry.register(Box::new(canceled_batches.clone()))?; registry.register(Box::new(gas_price_used_on_latest_batch.clone()))?; registry.register(Box::new(batcher_started.clone()))?; registry.register(Box::new(broken_ws_connections.clone()))?; @@ -56,6 +60,7 @@ impl BatcherMetrics { received_proofs, sent_batches, reverted_batches, + canceled_batches, batcher_started, gas_price_used_on_latest_batch, broken_ws_connections, From 77931ca5c347c55ceb83aedf1e8c8dcc46351679 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 10:48:31 -0300 Subject: [PATCH 64/90] refactor: use the same constant than the aggregator bump fee --- batcher/aligned-batcher/src/eth/utils.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 1c7f94ccef..dba67c3d7e 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -45,7 +45,7 @@ pub fn get_bumped_gas_price( current_gas_price: U256, iteration: usize, ) -> U256 { - let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) + (20 * iteration); + let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) + (5 * iteration); let bumped_previous_gas_price = previous_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); @@ -113,7 +113,7 @@ mod tests { let current_gas_price = U256::from(1200); let iteration = 2; - let expected = U256::from(1920); // (1200 * (120 + 40) / 100 + let expected = U256::from(1560); // (1200 * (120 + 10) / 100 assert_eq!( get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), @@ -127,7 +127,7 @@ mod tests { let current_gas_price = U256::from(1200); let iteration = 1; - let expected = U256::from(2100); // (1500 * 140) / 100 + let expected = U256::from(1875); // (1500 * (120 + 5) / 100 assert_eq!( get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), From c20427a5b5715b45ec78eede124321872dd1e85a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 12:06:14 -0300 Subject: [PATCH 65/90] feat: add panels to dashboard --- .../src/retry/batcher_retryables.rs | 2 +- .../aligned/aggregator_batcher.json | 228 ++++++++++++++++-- 2 files changed, 210 insertions(+), 20 deletions(-) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 0503f4977c..c18a9bc993 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -222,7 +222,7 @@ pub async fn cancel_create_new_task_retryable( .map_err(|e| { *iteration += 1; *previous_gas_price = bumped_gas_price; - warn!("Timeout while waiting for tx inclusion: {e}"); + warn!("Timeout while waiting for transaction inclusion: {e}"); RetryError::Transient(e.to_string()) })? .map_err(|e| { diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index cdcaa39cbe..edd8df1dca 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -71,7 +71,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -136,7 +138,9 @@ "options": { "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -308,7 +312,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -576,7 +582,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -702,6 +710,178 @@ "title": "Batches Reverted", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 3 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 19 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "canceled_batches", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Canceled Batches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 4, + "y": 19 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "canceled_batches", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Batches Canceled", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -765,7 +945,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 25 }, "id": 20, "interval": "1m", @@ -808,7 +988,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 33 }, "id": 10, "panels": [ @@ -850,7 +1030,7 @@ "h": 7, "w": 10, "x": 0, - "y": 42 + "y": 58 }, "id": 9, "options": { @@ -859,7 +1039,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1000,7 +1182,7 @@ "h": 7, "w": 10, "x": 10, - "y": 42 + "y": 58 }, "id": 1, "options": { @@ -1064,7 +1246,7 @@ "h": 7, "w": 5, "x": 0, - "y": 49 + "y": 65 }, "id": 8, "options": { @@ -1073,7 +1255,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1132,7 +1316,7 @@ "h": 7, "w": 5, "x": 5, - "y": 49 + "y": 65 }, "id": 7, "options": { @@ -1141,7 +1325,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1195,7 +1381,7 @@ "h": 7, "w": 5, "x": 0, - "y": 56 + "y": 72 }, "id": 2, "options": { @@ -1204,7 +1390,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1263,7 +1451,7 @@ "h": 7, "w": 5, "x": 5, - "y": 56 + "y": 72 }, "id": 5, "options": { @@ -1272,7 +1460,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1323,6 +1513,6 @@ "timezone": "browser", "title": "Aggregator Data", "uid": "aggregator", - "version": 6, + "version": 2, "weekStart": "" -} +} \ No newline at end of file From ebf1b661a20dcfec5eb19abe807c678df5cd593d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Avila=20Gast=C3=B3n?= <72628438+avilagaston9@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:12:47 -0300 Subject: [PATCH 66/90] Update batcher/aligned-batcher/src/lib.rs Co-authored-by: Mariano A. Nicolini --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index ab0e929fec..7ed4a97f82 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1222,7 +1222,7 @@ impl Batcher { // Sends a transaction to Ethereum with the same nonce as the previous one to override it. // We use the maximum number of retries here because this transaction must be included. pub async fn cancel_create_new_task_tx(&self, old_task_gas_price: U256) { - info!("Canceling created task"); + info!("Cancelling createNewTask transaction..."); let iteration = Arc::new(Mutex::new(0)); let previous_gas_price = Arc::new(Mutex::new(old_task_gas_price)); From c99a58cd9e78e625d168f230462e412f4653940b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Avila=20Gast=C3=B3n?= <72628438+avilagaston9@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:13:25 -0300 Subject: [PATCH 67/90] Update batcher/aligned-batcher/src/lib.rs Co-authored-by: Mariano A. Nicolini --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 7ed4a97f82..34351be4f8 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1241,7 +1241,7 @@ impl Batcher { ) .await { - error!("Could not cancel created task: {e}"); + error!("Could not cancel createNewTask transaction: {e}"); return; }; From 9b394c71f26fb29a16478f1ae9fad36eedaecac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Avila=20Gast=C3=B3n?= <72628438+avilagaston9@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:13:42 -0300 Subject: [PATCH 68/90] Update batcher/aligned-batcher/src/lib.rs Co-authored-by: Mariano A. Nicolini --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 34351be4f8..f38cd586f3 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1245,7 +1245,7 @@ impl Batcher { return; }; - info!("Task successfully canceled"); + info!("createNewTask transaction successfully canceled"); } /// Only relevant for testing and for users to easily use Aligned From ad932ab3357a037f4090bf3f9542e091dae611e0 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 12:17:10 -0300 Subject: [PATCH 69/90] refactor: rename old_task_gas_price variable --- batcher/aligned-batcher/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index f38cd586f3..b1da654314 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1221,10 +1221,10 @@ impl Batcher { // Sends a transaction to Ethereum with the same nonce as the previous one to override it. // We use the maximum number of retries here because this transaction must be included. - pub async fn cancel_create_new_task_tx(&self, old_task_gas_price: U256) { + pub async fn cancel_create_new_task_tx(&self, old_tx_gas_price: U256) { info!("Cancelling createNewTask transaction..."); let iteration = Arc::new(Mutex::new(0)); - let previous_gas_price = Arc::new(Mutex::new(old_task_gas_price)); + let previous_gas_price = Arc::new(Mutex::new(old_tx_gas_price)); if let Err(e) = retry_function( || { From 2b99c5e1538fa3eb1df6d6a4070ccfcffa494424 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 18:40:21 -0300 Subject: [PATCH 70/90] refactor: move ArcMutexs out from the retryable function --- batcher/aligned-batcher/src/lib.rs | 27 +++++++++++++++--- .../src/retry/batcher_retryables.rs | 28 ++++--------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index b1da654314..2c61be5b5b 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -3,7 +3,7 @@ use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use eth::service_manager::ServiceManager; -use eth::utils::{get_batcher_signer, get_gas_price}; +use eth::utils::{get_batcher_signer, get_bumped_gas_price, get_gas_price}; use ethers::contract::ContractError; use ethers::signers::Signer; use retry::batcher_retryables::{ @@ -1227,13 +1227,32 @@ impl Batcher { let previous_gas_price = Arc::new(Mutex::new(old_tx_gas_price)); if let Err(e) = retry_function( - || { + || async { + let mut iteration = iteration.lock().await; + let mut previous_gas_price = previous_gas_price.lock().await; + + let current_gas_price = match get_gas_price( + self.batcher_signer.provider(), + self.batcher_signer_fallback.provider(), + ) + .await + { + Ok(gas_price) => gas_price, + Err(e) => return Err(RetryError::Transient(format!("{:?}", e))), + }; + + let bumped_gas_price = + get_bumped_gas_price(*previous_gas_price, current_gas_price, *iteration); + + *iteration += 1; + *previous_gas_price = bumped_gas_price; + cancel_create_new_task_retryable( &self.batcher_signer, &self.batcher_signer_fallback, - iteration.clone(), - previous_gas_price.clone(), + bumped_gas_price, ) + .await }, DEFAULT_MIN_RETRY_DELAY, DEFAULT_BACKOFF_FACTOR, diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index c18a9bc993..de7014c5bf 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -1,14 +1,14 @@ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use aligned_sdk::core::constants::{BATCH_INCLUSION_DELAY, TRANSACTIONS_INCLUSION_DELAY}; use ethers::prelude::*; use log::{info, warn}; -use tokio::{sync::Mutex, time::timeout}; +use tokio::time::timeout; use crate::{ eth::{ payment_service::{BatcherPaymentService, CreateNewTaskFeeParams, SignerMiddlewareT}, - utils::{get_bumped_gas_price, get_current_nonce, get_gas_price}, + utils::get_current_nonce, }, retry::RetryError, types::errors::BatcherError, @@ -175,11 +175,8 @@ pub async fn create_new_task_retryable( pub async fn cancel_create_new_task_retryable( batcher_signer: &SignerMiddlewareT, batcher_signer_fallback: &SignerMiddlewareT, - iteration: Arc>, - previous_gas_price: Arc>, + bumped_gas_price: U256, ) -> Result> { - let mut iteration = iteration.lock().await; - let mut previous_gas_price = previous_gas_price.lock().await; let batcher_addr = batcher_signer.address(); let current_nonce = get_current_nonce( @@ -190,15 +187,6 @@ pub async fn cancel_create_new_task_retryable( .await .map_err(|e| RetryError::Transient(e.to_string()))?; - let current_gas_price = get_gas_price( - batcher_signer.provider(), - batcher_signer_fallback.provider(), - ) - .await - .map_err(|e| RetryError::Transient(format!("{:?}", e)))?; - - let bumped_gas_price = get_bumped_gas_price(*previous_gas_price, current_gas_price, *iteration); - let tx = TransactionRequest::new() .to(batcher_addr) .value(U256::zero()) @@ -220,8 +208,6 @@ pub async fn cancel_create_new_task_retryable( ) .await .map_err(|e| { - *iteration += 1; - *previous_gas_price = bumped_gas_price; warn!("Timeout while waiting for transaction inclusion: {e}"); RetryError::Transient(e.to_string()) })? @@ -229,11 +215,7 @@ pub async fn cancel_create_new_task_retryable( warn!("Error while waiting for tx inclusion: {e}"); RetryError::Transient(e.to_string()) })? - .ok_or({ - *iteration += 1; - *previous_gas_price = bumped_gas_price; - RetryError::Transient("Receipt not found".to_string()) - }) + .ok_or(RetryError::Transient("Receipt not found".to_string())) } #[cfg(test)] From bab60d707be67e5f439c5ad447a543bb6bd3a34d Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 19:16:53 -0300 Subject: [PATCH 71/90] refactor: create eth_http_providers only once --- batcher/aligned-batcher/src/lib.rs | 32 ++++++++++-------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index a153d7e48b..1748c91d23 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -114,28 +114,22 @@ impl Batcher { let metrics = metrics::BatcherMetrics::start(config.batcher.metrics_port) .expect("Failed to start metrics server"); - let eth_rpc_provider = + let eth_http_provider = eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get provider"); - let eth_rpc_provider_fallback = eth::get_provider(config.eth_rpc_url_fallback.clone()) + let eth_http_provider_fallback = eth::get_provider(config.eth_rpc_url_fallback.clone()) .expect("Failed to get fallback provider"); - let eth_rpc_provider_service_manager = - eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get provider"); - - let eth_rpc_provider_service_manager_fallback = - eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get provider"); - // FIXME(marian): We are getting just the last block number right now, but we should really // have the last submitted batch block registered and query it when the batcher is initialized. - let last_uploaded_batch_block = match eth_rpc_provider.get_block_number().await { + let last_uploaded_batch_block = match eth_http_provider.get_block_number().await { Ok(block_num) => block_num, Err(e) => { warn!( "Failed to get block number with main rpc, trying with fallback rpc. Err: {:?}", e ); - eth_rpc_provider_fallback + eth_http_provider_fallback .get_block_number() .await .expect("Failed to get block number with fallback rpc") @@ -144,11 +138,11 @@ impl Batcher { let last_uploaded_batch_block = last_uploaded_batch_block.as_u64(); - let chain_id = match eth_rpc_provider.get_chainid().await { + let chain_id = match eth_http_provider.get_chainid().await { Ok(chain_id) => chain_id, Err(e) => { warn!("Failed to get chain id with main rpc: {}", e); - eth_rpc_provider_fallback + eth_http_provider_fallback .get_chainid() .await .expect("Failed to get chain id with fallback rpc") @@ -156,7 +150,7 @@ impl Batcher { }; let payment_service = eth::payment_service::get_batcher_payment_service( - eth_rpc_provider, + eth_http_provider.clone(), config.ecdsa.clone(), deployment_output.addresses.batcher_payment_service.clone(), ) @@ -164,7 +158,7 @@ impl Batcher { .expect("Failed to get Batcher Payment Service contract"); let payment_service_fallback = eth::payment_service::get_batcher_payment_service( - eth_rpc_provider_fallback, + eth_http_provider_fallback.clone(), config.ecdsa.clone(), deployment_output.addresses.batcher_payment_service, ) @@ -172,7 +166,7 @@ impl Batcher { .expect("Failed to get fallback Batcher Payment Service contract"); let service_manager = eth::service_manager::get_service_manager( - eth_rpc_provider_service_manager, + eth_http_provider.clone(), config.ecdsa.clone(), deployment_output.addresses.service_manager.clone(), ) @@ -180,7 +174,7 @@ impl Batcher { .expect("Failed to get Service Manager contract"); let service_manager_fallback = eth::service_manager::get_service_manager( - eth_rpc_provider_service_manager_fallback, + eth_http_provider_fallback.clone(), config.ecdsa, deployment_output.addresses.service_manager, ) @@ -218,12 +212,6 @@ impl Batcher { } .expect("Failed to get disabled verifiers"); - let eth_http_provider = - eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get http provider"); - - let eth_http_provider_fallback = eth::get_provider(config.eth_rpc_url_fallback.clone()) - .expect("Failed to get fallback http provider"); - Self { s3_client, s3_bucket_name, From 56388b1fb5dfec1b06ebb3d7a05922606741e50a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 19:18:32 -0300 Subject: [PATCH 72/90] refactor: improve comments in tests --- batcher/aligned-batcher/src/retry/batcher_retryables.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index ff527f1730..d963398a95 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -142,7 +142,7 @@ mod test { .unwrap(); assert_eq!(balance, U256::zero()); - // Kill anvil + // Anvil is killed when the scope is left } let result = @@ -178,7 +178,7 @@ mod test { .unwrap(); assert_eq!(unlocked, false); - // Kill Anvil + // Anvil is killed when the scope is left } let result = user_balance_is_unlocked_retryable( @@ -220,7 +220,7 @@ mod test { .unwrap(); assert_eq!(nonce, U256::zero()); - // Kill Anvil + // Anvil is killed when the scope is left } let result = get_user_nonce_from_ethereum_retryable( @@ -256,7 +256,7 @@ mod test { let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; assert!(result.is_ok()); - // kill Anvil + // Anvil is killed when the scope is left } let result = get_gas_price_retryable(ð_rpc_provider, ð_rpc_provider).await; assert!(matches!(result, Err(RetryError::Transient(_)))); From 7da5d3ec90031ef8a45af7a3ae13bd08cd2b0105 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 28 Oct 2024 19:21:45 -0300 Subject: [PATCH 73/90] docs: add reference to backon --- batcher/aligned-batcher/src/retry/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/batcher/aligned-batcher/src/retry/mod.rs b/batcher/aligned-batcher/src/retry/mod.rs index 0af5c3bab3..10d552f365 100644 --- a/batcher/aligned-batcher/src/retry/mod.rs +++ b/batcher/aligned-batcher/src/retry/mod.rs @@ -30,6 +30,7 @@ impl RetryError { impl std::error::Error for RetryError where E: std::fmt::Debug {} +// Supports retries only on async functions. See: https://docs.rs/backon/latest/backon/#retry-an-async-function pub async fn retry_function( function: FutureFn, min_delay: u64, From b8edd49805b32277e25dac796be349d168c23332 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 29 Oct 2024 00:09:59 -0300 Subject: [PATCH 74/90] refactor: use anyhow properly --- batcher/aligned-batcher/src/eth/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index dba67c3d7e..a72e041094 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -24,7 +24,7 @@ pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error pub async fn get_batcher_signer( provider: Provider, ecdsa_config: ECDSAConfig, -) -> Result, anyhow::Error> { +) -> anyhow::Result> { let chain_id = provider.get_chainid().await?; // get private key from keystore From f6df0f3f9864e469e88e25f4f1f5f63f720b1b57 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 29 Oct 2024 00:14:41 -0300 Subject: [PATCH 75/90] refactor: improve utils crate imports --- batcher/aligned-batcher/src/eth/utils.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index a72e041094..2a22e7cbe9 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -1,10 +1,14 @@ use std::str::FromStr; use std::sync::Arc; -use crate::config::ECDSAConfig; -use crate::retry::batcher_retryables::{get_current_nonce_retryable, get_gas_price_retryable}; -use crate::retry::retry_function; -use crate::types::errors::BatcherError; +use crate::{ + config::ECDSAConfig, + retry::{ + batcher_retryables::{get_current_nonce_retryable, get_gas_price_retryable}, + retry_function, + }, + types::errors::BatcherError, +}; use aligned_sdk::core::constants::{ DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, OVERRIDE_GAS_PRICE_MULTIPLIER, PERCENTAGE_DIVIDER, From 8acddefb09a535a5eb50c06f788195139524a98b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 29 Oct 2024 09:42:32 -0300 Subject: [PATCH 76/90] refactor: rename constants --- batcher/aligned-batcher/src/eth/utils.rs | 6 ++-- .../src/retry/batcher_retryables.rs | 29 +++++++++---------- batcher/aligned-sdk/src/core/constants.rs | 4 +-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 2a22e7cbe9..74fdb5e25a 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -11,7 +11,8 @@ use crate::{ }; use aligned_sdk::core::constants::{ DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, - OVERRIDE_GAS_PRICE_MULTIPLIER, PERCENTAGE_DIVIDER, + GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION, OVERRIDE_GAS_PRICE_MULTIPLIER, + PERCENTAGE_DIVIDER, }; use ethers::prelude::*; use ethers::providers::{Http, Provider}; @@ -49,7 +50,8 @@ pub fn get_bumped_gas_price( current_gas_price: U256, iteration: usize, ) -> U256 { - let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) + (5 * iteration); + let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) + + (GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION * iteration); let bumped_previous_gas_price = previous_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 0ea2b124f8..f4ee5a112e 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use aligned_sdk::core::constants::{BATCH_INCLUSION_DELAY, TRANSACTIONS_INCLUSION_DELAY}; +use aligned_sdk::core::constants::TRANSACTION_WAIT_TIMEOUT; use ethers::prelude::*; use log::{info, warn}; use tokio::time::timeout; @@ -159,7 +159,7 @@ pub async fn create_new_task_retryable( }; // timeout to prevent a deadlock while waiting for the transaction to be included in a block. - timeout(Duration::from_millis(BATCH_INCLUSION_DELAY), pending_tx) + timeout(Duration::from_millis(TRANSACTION_WAIT_TIMEOUT), pending_tx) .await .map_err(|e| { warn!("Error while waiting for batch inclusion: {e}"); @@ -202,20 +202,17 @@ pub async fn cancel_create_new_task_retryable( }; // timeout to prevent a deadlock while waiting for the transaction to be included in a block. - timeout( - Duration::from_millis(TRANSACTIONS_INCLUSION_DELAY), - pending_tx, - ) - .await - .map_err(|e| { - warn!("Timeout while waiting for transaction inclusion: {e}"); - RetryError::Transient(e.to_string()) - })? - .map_err(|e| { - warn!("Error while waiting for tx inclusion: {e}"); - RetryError::Transient(e.to_string()) - })? - .ok_or(RetryError::Transient("Receipt not found".to_string())) + timeout(Duration::from_millis(TRANSACTION_WAIT_TIMEOUT), pending_tx) + .await + .map_err(|e| { + warn!("Timeout while waiting for transaction inclusion: {e}"); + RetryError::Transient(e.to_string()) + })? + .map_err(|e| { + warn!("Error while waiting for tx inclusion: {e}"); + RetryError::Transient(e.to_string()) + })? + .ok_or(RetryError::Transient("Receipt not found".to_string())) } #[cfg(test)] diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index d9ce840bce..0c1d24bfac 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -1,6 +1,6 @@ /// Batcher /// -pub const BATCH_INCLUSION_DELAY: u64 = 36_000; // 3 blocks -pub const TRANSACTIONS_INCLUSION_DELAY: u64 = 14_000; // 1 block + 2 secs +pub const TRANSACTION_WAIT_TIMEOUT: u64 = 36_000; // 3 blocks +pub const GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION: usize = 5; pub const AGGREGATOR_GAS_COST: u128 = 400_000; pub const BATCHER_SUBMISSION_BASE_GAS_COST: u128 = 125_000; pub const ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF: u128 = 13_000; From 593bfb30abdaae1f8c323515412e83ecdbbee9a8 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 29 Oct 2024 09:54:25 -0300 Subject: [PATCH 77/90] refactor: use match instead of if let --- batcher/aligned-batcher/src/eth/utils.rs | 4 +- .../src/retry/batcher_retryables.rs | 41 ++++++++----------- batcher/aligned-sdk/src/core/constants.rs | 2 +- .../aligned/aggregator_batcher.json | 2 +- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 74fdb5e25a..283685512d 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -11,7 +11,7 @@ use crate::{ }; use aligned_sdk::core::constants::{ DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, - GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION, OVERRIDE_GAS_PRICE_MULTIPLIER, + GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION, OVERRIDE_GAS_PRICE_PORCENTAGE_MULTIPLIER, PERCENTAGE_DIVIDER, }; use ethers::prelude::*; @@ -50,7 +50,7 @@ pub fn get_bumped_gas_price( current_gas_price: U256, iteration: usize, ) -> U256 { - let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_MULTIPLIER) + let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_PORCENTAGE_MULTIPLIER) + (GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION * iteration); let bumped_previous_gas_price = previous_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index f4ee5a112e..6b2a3a26da 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -56,16 +56,16 @@ pub async fn get_current_nonce_retryable( eth_http_provider_fallback: &Provider, addr: Address, ) -> Result> { - if let Ok(current_nonce) = eth_http_provider.get_transaction_count(addr, None).await { - return Ok(current_nonce); + match eth_http_provider.get_transaction_count(addr, None).await { + Ok(current_nonce) => Ok(current_nonce), + Err(_) => eth_http_provider_fallback + .get_transaction_count(addr, None) + .await + .map_err(|e| { + warn!("Error getting user nonce: {e}"); + RetryError::Transient(e.to_string()) + }), } - eth_http_provider_fallback - .get_transaction_count(addr, None) - .await - .map_err(|e| { - warn!("Error getting user nonce: {e}"); - RetryError::Transient(e.to_string()) - }) } pub async fn user_balance_is_unlocked_retryable( @@ -91,21 +91,16 @@ pub async fn get_gas_price_retryable( eth_http_provider: &Provider, eth_http_provider_fallback: &Provider, ) -> Result> { - if let Ok(gas_price) = eth_http_provider - .get_gas_price() - .await - .inspect_err(|e| warn!("Failed to get gas price. Trying with fallback: {e:?}")) - { - return Ok(gas_price); + match eth_http_provider.get_gas_price().await { + Ok(gas_price) => Ok(gas_price), + Err(_) => eth_http_provider_fallback + .get_gas_price() + .await + .map_err(|e| { + warn!("Failed to get fallback gas price: {e:?}"); + RetryError::Transient(e.to_string()) + }), } - - eth_http_provider_fallback - .get_gas_price() - .await - .map_err(|e| { - warn!("Failed to get fallback gas price: {e:?}"); - RetryError::Transient(e.to_string()) - }) } pub async fn create_new_task_retryable( diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index 0c1d24bfac..a12e44f4c1 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -15,7 +15,7 @@ pub const MIN_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 1 pub const RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER: u128 = 250; // fee_for_aggregator -> respondToTaskFeeLimit modifier pub const DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER: u128 = 150; // feeForAggregator modifier pub const GAS_PRICE_PERCENTAGE_MULTIPLIER: u128 = 110; // gasPrice modifier -pub const OVERRIDE_GAS_PRICE_MULTIPLIER: u128 = 120; // gasPrice modifier to override previous transactions +pub const OVERRIDE_GAS_PRICE_PORCENTAGE_MULTIPLIER: u128 = 120; // gasPrice modifier to override previous transactions pub const PERCENTAGE_DIVIDER: u128 = 100; /// SDK /// diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index edd8df1dca..a735609ab5 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -1515,4 +1515,4 @@ "uid": "aggregator", "version": 2, "weekStart": "" -} \ No newline at end of file +} From 1fa1873336502905992eaa66d72e1ea7befb9ff9 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 29 Oct 2024 10:12:15 -0300 Subject: [PATCH 78/90] refactor: use ProviderErrors instead of Strings --- batcher/aligned-batcher/src/eth/utils.rs | 9 ++++---- batcher/aligned-batcher/src/lib.rs | 4 ++-- .../src/retry/batcher_retryables.rs | 22 ++++++++++--------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 283685512d..15d138362f 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -7,7 +7,6 @@ use crate::{ batcher_retryables::{get_current_nonce_retryable, get_gas_price_retryable}, retry_function, }, - types::errors::BatcherError, }; use aligned_sdk::core::constants::{ DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, @@ -64,7 +63,7 @@ pub async fn get_current_nonce( eth_http_provider: &Provider, eth_http_provider_fallback: &Provider, addr: H160, -) -> Result { +) -> Result { retry_function( || get_current_nonce_retryable(eth_http_provider, eth_http_provider_fallback, addr), DEFAULT_MIN_RETRY_DELAY, @@ -74,7 +73,7 @@ pub async fn get_current_nonce( .await .map_err(|e| { error!("Could't get nonce: {:?}", e); - e.to_string() + e.inner() }) } @@ -82,7 +81,7 @@ pub async fn get_current_nonce( pub async fn get_gas_price( eth_http_provider: &Provider, eth_http_provider_fallback: &Provider, -) -> Result { +) -> Result { retry_function( || get_gas_price_retryable(eth_http_provider, eth_http_provider_fallback), DEFAULT_MIN_RETRY_DELAY, @@ -92,7 +91,7 @@ pub async fn get_gas_price( .await .map_err(|e| { error!("Could't get gas price: {e}"); - BatcherError::GasPriceError + e.inner() }) } #[cfg(test)] diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index bb931418ce..24a0231fe7 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1075,7 +1075,7 @@ impl Batcher { let (gas_price, disable_verifiers) = tokio::join!(gas_price_future, disabled_verifiers_future); - let gas_price = gas_price?; + let gas_price = gas_price.map_err(|_| BatcherError::GasPriceError)?; { let new_disable_verifiers = disable_verifiers @@ -1232,7 +1232,7 @@ impl Batcher { .await { Ok(gas_price) => gas_price, - Err(e) => return Err(RetryError::Transient(format!("{:?}", e))), + Err(e) => return Err(RetryError::Transient(e)), }; let bumped_gas_price = diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 6b2a3a26da..6ba8d97bb3 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -55,7 +55,7 @@ pub async fn get_current_nonce_retryable( eth_http_provider: &Provider, eth_http_provider_fallback: &Provider, addr: Address, -) -> Result> { +) -> Result> { match eth_http_provider.get_transaction_count(addr, None).await { Ok(current_nonce) => Ok(current_nonce), Err(_) => eth_http_provider_fallback @@ -63,7 +63,7 @@ pub async fn get_current_nonce_retryable( .await .map_err(|e| { warn!("Error getting user nonce: {e}"); - RetryError::Transient(e.to_string()) + RetryError::Transient(e) }), } } @@ -90,7 +90,7 @@ pub async fn user_balance_is_unlocked_retryable( pub async fn get_gas_price_retryable( eth_http_provider: &Provider, eth_http_provider_fallback: &Provider, -) -> Result> { +) -> Result> { match eth_http_provider.get_gas_price().await { Ok(gas_price) => Ok(gas_price), Err(_) => eth_http_provider_fallback @@ -98,7 +98,7 @@ pub async fn get_gas_price_retryable( .await .map_err(|e| { warn!("Failed to get fallback gas price: {e:?}"); - RetryError::Transient(e.to_string()) + RetryError::Transient(e) }), } } @@ -171,7 +171,7 @@ pub async fn cancel_create_new_task_retryable( batcher_signer: &SignerMiddlewareT, batcher_signer_fallback: &SignerMiddlewareT, bumped_gas_price: U256, -) -> Result> { +) -> Result> { let batcher_addr = batcher_signer.address(); let current_nonce = get_current_nonce( @@ -180,7 +180,7 @@ pub async fn cancel_create_new_task_retryable( batcher_addr, ) .await - .map_err(|e| RetryError::Transient(e.to_string()))?; + .map_err(|e| RetryError::Transient(e))?; let tx = TransactionRequest::new() .to(batcher_addr) @@ -193,7 +193,7 @@ pub async fn cancel_create_new_task_retryable( Err(_) => batcher_signer_fallback .send_transaction(tx.clone(), None) .await - .map_err(|e| RetryError::Transient(e.to_string()))?, + .map_err(|e| RetryError::Transient(ProviderError::CustomError(e.to_string())))?, }; // timeout to prevent a deadlock while waiting for the transaction to be included in a block. @@ -201,13 +201,15 @@ pub async fn cancel_create_new_task_retryable( .await .map_err(|e| { warn!("Timeout while waiting for transaction inclusion: {e}"); - RetryError::Transient(e.to_string()) + RetryError::Transient(ProviderError::CustomError("Receipt not found".to_string())) })? .map_err(|e| { warn!("Error while waiting for tx inclusion: {e}"); - RetryError::Transient(e.to_string()) + RetryError::Transient(e) })? - .ok_or(RetryError::Transient("Receipt not found".to_string())) + .ok_or(RetryError::Transient(ProviderError::CustomError( + "Receipt not found".to_string(), + ))) } #[cfg(test)] From 5acedb88a02e60310e1ec2feca818e453679051c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 29 Oct 2024 10:18:29 -0300 Subject: [PATCH 79/90] fix: clippy --- batcher/aligned-batcher/src/retry/batcher_retryables.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 6ba8d97bb3..0085517651 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -180,7 +180,7 @@ pub async fn cancel_create_new_task_retryable( batcher_addr, ) .await - .map_err(|e| RetryError::Transient(e))?; + .map_err(RetryError::Transient)?; let tx = TransactionRequest::new() .to(batcher_addr) From c7f023ee991ce6e3b17e90c9bcad2d47881f50b6 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Sat, 2 Nov 2024 22:22:32 -0300 Subject: [PATCH 80/90] refactor: add transaction_wait_timeout to the batcher config file --- batcher/aligned-batcher/src/config/mod.rs | 1 + batcher/aligned-batcher/src/lib.rs | 4 ++++ batcher/aligned-batcher/src/retry/batcher_retryables.rs | 7 ++++--- batcher/aligned-sdk/src/core/constants.rs | 1 - config-files/config-batcher.yaml | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/batcher/aligned-batcher/src/config/mod.rs b/batcher/aligned-batcher/src/config/mod.rs index f453a5cfdf..f7289cddf7 100644 --- a/batcher/aligned-batcher/src/config/mod.rs +++ b/batcher/aligned-batcher/src/config/mod.rs @@ -38,6 +38,7 @@ impl NonPayingConfig { #[derive(Debug, Deserialize)] pub struct BatcherConfigFromYaml { pub block_interval: u64, + pub transaction_wait_timeout: u64, pub max_proof_size: usize, pub max_batch_size: usize, pub pre_verification_is_enabled: bool, diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 24a0231fe7..c54c124245 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -77,6 +77,7 @@ pub struct Batcher { service_manager_fallback: ServiceManager, batch_state: Mutex, max_block_interval: u64, + transaction_wait_timeout: u64, max_proof_size: usize, max_batch_size: usize, last_uploaded_batch_block: Mutex, @@ -232,6 +233,7 @@ impl Batcher { service_manager, service_manager_fallback, max_block_interval: config.batcher.block_interval, + transaction_wait_timeout: config.batcher.transaction_wait_timeout, max_proof_size: config.batcher.max_proof_size, max_batch_size: config.batcher.max_batch_size, last_uploaded_batch_block: Mutex::new(last_uploaded_batch_block), @@ -1193,6 +1195,7 @@ impl Batcher { batch_data_pointer.clone(), proof_submitters.clone(), fee_params.clone(), + self.transaction_wait_timeout, &self.payment_service, &self.payment_service_fallback, ) @@ -1245,6 +1248,7 @@ impl Batcher { &self.batcher_signer, &self.batcher_signer_fallback, bumped_gas_price, + self.transaction_wait_timeout, ) .await }, diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 0085517651..20656b52f0 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -1,6 +1,5 @@ use std::time::Duration; -use aligned_sdk::core::constants::TRANSACTION_WAIT_TIMEOUT; use ethers::prelude::*; use log::{info, warn}; use tokio::time::timeout; @@ -108,6 +107,7 @@ pub async fn create_new_task_retryable( batch_data_pointer: String, proofs_submitters: Vec
, fee_params: CreateNewTaskFeeParams, + transaction_wait_timeout: u64, payment_service: &BatcherPaymentService, payment_service_fallback: &BatcherPaymentService, ) -> Result> { @@ -154,7 +154,7 @@ pub async fn create_new_task_retryable( }; // timeout to prevent a deadlock while waiting for the transaction to be included in a block. - timeout(Duration::from_millis(TRANSACTION_WAIT_TIMEOUT), pending_tx) + timeout(Duration::from_millis(transaction_wait_timeout), pending_tx) .await .map_err(|e| { warn!("Error while waiting for batch inclusion: {e}"); @@ -171,6 +171,7 @@ pub async fn cancel_create_new_task_retryable( batcher_signer: &SignerMiddlewareT, batcher_signer_fallback: &SignerMiddlewareT, bumped_gas_price: U256, + transaction_wait_timeout: u64, ) -> Result> { let batcher_addr = batcher_signer.address(); @@ -197,7 +198,7 @@ pub async fn cancel_create_new_task_retryable( }; // timeout to prevent a deadlock while waiting for the transaction to be included in a block. - timeout(Duration::from_millis(TRANSACTION_WAIT_TIMEOUT), pending_tx) + timeout(Duration::from_millis(transaction_wait_timeout), pending_tx) .await .map_err(|e| { warn!("Timeout while waiting for transaction inclusion: {e}"); diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index a12e44f4c1..fe2e46c446 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -1,5 +1,4 @@ /// Batcher /// -pub const TRANSACTION_WAIT_TIMEOUT: u64 = 36_000; // 3 blocks pub const GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION: usize = 5; pub const AGGREGATOR_GAS_COST: u128 = 400_000; pub const BATCHER_SUBMISSION_BASE_GAS_COST: u128 = 125_000; diff --git a/config-files/config-batcher.yaml b/config-files/config-batcher.yaml index 7d0d96160b..7b41f5d4dd 100644 --- a/config-files/config-batcher.yaml +++ b/config-files/config-batcher.yaml @@ -18,6 +18,7 @@ ecdsa: batcher: block_interval: 3 batch_size_interval: 10 + transaction_wait_timeout: 36000 # 3 blocks max_proof_size: 67108864 # 64 MiB max_batch_size: 268435456 # 256 MiB pre_verification_is_enabled: true From 2a262d71e3a2304cc9d7b39732e93acf9f28ca2f Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 4 Nov 2024 11:27:06 -0300 Subject: [PATCH 81/90] refactor: improve comments --- batcher/aligned-batcher/src/eth/utils.rs | 6 ++++-- batcher/aligned-sdk/src/core/constants.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 15d138362f..15aeeed0dc 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -10,7 +10,7 @@ use crate::{ }; use aligned_sdk::core::constants::{ DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, - GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION, OVERRIDE_GAS_PRICE_PORCENTAGE_MULTIPLIER, + GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION, OVERRIDE_GAS_PRICE_PERCENTAGE_MULTIPLIER, PERCENTAGE_DIVIDER, }; use ethers::prelude::*; @@ -49,13 +49,15 @@ pub fn get_bumped_gas_price( current_gas_price: U256, iteration: usize, ) -> U256 { - let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_PORCENTAGE_MULTIPLIER) + let override_gas_multiplier = U256::from(OVERRIDE_GAS_PRICE_PERCENTAGE_MULTIPLIER) + (GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION * iteration); let bumped_previous_gas_price = previous_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); let bumped_current_gas_price = current_gas_price * override_gas_multiplier / U256::from(PERCENTAGE_DIVIDER); + // Return the maximum of the previous and current gas prices + // to avoid sending a transaction with a gas price lower than the previous one. bumped_current_gas_price.max(bumped_previous_gas_price) } diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index fe2e46c446..f6570045f7 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -14,7 +14,7 @@ pub const MIN_FEE_PER_PROOF: u128 = ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF * 1 pub const RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER: u128 = 250; // fee_for_aggregator -> respondToTaskFeeLimit modifier pub const DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER: u128 = 150; // feeForAggregator modifier pub const GAS_PRICE_PERCENTAGE_MULTIPLIER: u128 = 110; // gasPrice modifier -pub const OVERRIDE_GAS_PRICE_PORCENTAGE_MULTIPLIER: u128 = 120; // gasPrice modifier to override previous transactions +pub const OVERRIDE_GAS_PRICE_PERCENTAGE_MULTIPLIER: u128 = 120; // gasPrice modifier to override previous transactions pub const PERCENTAGE_DIVIDER: u128 = 100; /// SDK /// From 05fe8fe714e9ec632995c342107255955efeb606 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 4 Nov 2024 12:15:23 -0300 Subject: [PATCH 82/90] fix: merge conflicts --- .../src/eth/service_manager.rs | 1 - batcher/aligned-batcher/src/eth/utils.rs | 1 - batcher/aligned-batcher/src/lib.rs | 73 ------------------- .../src/retry/batcher_retryables.rs | 4 - 4 files changed, 79 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/service_manager.rs b/batcher/aligned-batcher/src/eth/service_manager.rs index 30962eb4fd..db610ca9d1 100644 --- a/batcher/aligned-batcher/src/eth/service_manager.rs +++ b/batcher/aligned-batcher/src/eth/service_manager.rs @@ -16,7 +16,6 @@ pub type SignerMiddlewareT = SignerMiddleware, Wallet pub type ServiceManager = AlignedLayerServiceManagerContract; pub async fn get_service_manager( - provider: Provider, provider: Provider, ecdsa_config: ECDSAConfig, contract_address: String, diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 5495a2a510..15aeeed0dc 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -19,7 +19,6 @@ use log::error; use super::payment_service::SignerMiddlewareT; -pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error> { pub fn get_provider(eth_rpc_url: String) -> Result, anyhow::Error> { let provider = Http::from_str(eth_rpc_url.as_str()) .map_err(|e| anyhow::Error::msg(format!("Failed to create provider: {}", e)))?; diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index a26924bcf1..2a7bf66d5a 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -23,8 +23,6 @@ use aligned_sdk::core::constants::{ ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CONSTANT_GAS_COST, DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_FEE_PER_PROOF, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, - DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_BACKOFF_FACTOR, - DEFAULT_MAX_FEE_PER_PROOF, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, PERCENTAGE_DIVIDER, RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, }; @@ -56,7 +54,6 @@ mod eth; pub mod gnark; pub mod metrics; pub mod retry; -pub mod retry; pub mod risc_zero; pub mod s3; pub mod sp1; @@ -65,8 +62,6 @@ mod zk_utils; pub const LISTEN_NEW_BLOCKS_MAX_TIMES: usize = usize::MAX; -pub const LISTEN_NEW_BLOCKS_MAX_TIMES: usize = usize::MAX; - pub struct Batcher { s3_client: S3Client, s3_bucket_name: String, @@ -119,17 +114,14 @@ impl Batcher { let metrics = metrics::BatcherMetrics::start(config.batcher.metrics_port) .expect("Failed to start metrics server"); - let eth_http_provider = let eth_http_provider = eth::get_provider(config.eth_rpc_url.clone()).expect("Failed to get provider"); - let eth_http_provider_fallback = eth::get_provider(config.eth_rpc_url_fallback.clone()) let eth_http_provider_fallback = eth::get_provider(config.eth_rpc_url_fallback.clone()) .expect("Failed to get fallback provider"); // FIXME(marian): We are getting just the last block number right now, but we should really // have the last submitted batch block registered and query it when the batcher is initialized. - let last_uploaded_batch_block = match eth_http_provider.get_block_number().await { let last_uploaded_batch_block = match eth_http_provider.get_block_number().await { Ok(block_num) => block_num, Err(e) => { @@ -137,7 +129,6 @@ impl Batcher { "Failed to get block number with main rpc, trying with fallback rpc. Err: {:?}", e ); - eth_http_provider_fallback eth_http_provider_fallback .get_block_number() .await @@ -147,12 +138,10 @@ impl Batcher { let last_uploaded_batch_block = last_uploaded_batch_block.as_u64(); - let chain_id = match eth_http_provider.get_chainid().await { let chain_id = match eth_http_provider.get_chainid().await { Ok(chain_id) => chain_id, Err(e) => { warn!("Failed to get chain id with main rpc: {}", e); - eth_http_provider_fallback eth_http_provider_fallback .get_chainid() .await @@ -184,7 +173,6 @@ impl Batcher { .expect("Failed to get fallback Batcher Payment Service contract"); let service_manager = eth::service_manager::get_service_manager( - eth_http_provider.clone(), eth_http_provider.clone(), config.ecdsa.clone(), deployment_output.addresses.service_manager.clone(), @@ -193,7 +181,6 @@ impl Batcher { .expect("Failed to get Service Manager contract"); let service_manager_fallback = eth::service_manager::get_service_manager( - eth_http_provider_fallback.clone(), eth_http_provider_fallback.clone(), config.ecdsa, deployment_output.addresses.service_manager, @@ -289,26 +276,6 @@ impl Batcher { .map_err(|e| e.inner()) } - pub async fn listen_new_blocks_retryable( - self: Arc, - ) -> Result<(), RetryError> { - let eth_ws_provider = Provider::connect(&self.eth_ws_url).await.map_err(|e| { - warn!("Failed to instantiate Ethereum websocket provider"); - RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) - })?; - retry_function( - || { - let app = self.clone(); - async move { app.listen_new_blocks_retryable().await } - }, - DEFAULT_MIN_RETRY_DELAY, - DEFAULT_BACKOFF_FACTOR, - LISTEN_NEW_BLOCKS_MAX_TIMES, - ) - .await - .map_err(|e| e.inner()) - } - pub async fn listen_new_blocks_retryable( self: Arc, ) -> Result<(), RetryError> { @@ -330,27 +297,6 @@ impl Batcher { RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) })?; - let mut stream_fallback = - eth_ws_provider_fallback - .subscribe_blocks() - .await - .map_err(|e| { - warn!("Error subscribing to blocks."); - RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) - })?; - let eth_ws_provider_fallback = - Provider::connect(&self.eth_ws_url_fallback) - .await - .map_err(|e| { - warn!("Failed to instantiate fallback Ethereum websocket provider"); - RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) - })?; - - let mut stream = eth_ws_provider.subscribe_blocks().await.map_err(|e| { - warn!("Error subscribing to blocks."); - RetryError::Transient(BatcherError::EthereumSubscriptionError(e.to_string())) - })?; - let mut stream_fallback = eth_ws_provider_fallback .subscribe_blocks() @@ -1420,20 +1366,6 @@ impl Batcher { ) .await .ok() - retry_function( - || { - get_user_balance_retryable( - &self.payment_service, - &self.payment_service_fallback, - addr, - ) - }, - DEFAULT_MIN_RETRY_DELAY, - DEFAULT_BACKOFF_FACTOR, - DEFAULT_MAX_RETRIES, - ) - .await - .ok() } /// Checks if the user's balance is unlocked for a given address using exponential backoff. @@ -1497,10 +1429,5 @@ impl Batcher { RetryError::Transient(e.to_string()) })?; Ok(()) - .map_err(|e| { - warn!("Error uploading batch to s3 {e}"); - RetryError::Transient(e.to_string()) - })?; - Ok(()) } } diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 20656b52f0..2a3c630981 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -268,7 +268,6 @@ mod test { (anvil, payment_service) } - #[ignore] #[tokio::test] async fn test_get_user_balance_retryable() { let payment_service; @@ -301,7 +300,6 @@ mod test { assert_eq!(balance, U256::zero()); } - #[ignore] #[tokio::test] async fn test_user_balance_is_unlocked_retryable() { let payment_service; @@ -344,7 +342,6 @@ mod test { assert_eq!(unlocked, false); } - #[ignore] #[tokio::test] async fn test_get_user_nonce_retryable() { let payment_service; @@ -387,7 +384,6 @@ mod test { assert_eq!(nonce, U256::zero()); } - #[ignore] #[tokio::test] async fn test_get_gas_price_retryable() { let eth_rpc_provider; From 007b93abacbe9f49511e87fb4433bbe6e96ef2dd Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 4 Nov 2024 12:46:11 -0300 Subject: [PATCH 83/90] refactor: minor improvements --- batcher/aligned-batcher/src/lib.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 2a7bf66d5a..95b171ded9 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -68,11 +68,11 @@ pub struct Batcher { download_endpoint: String, eth_ws_url: String, eth_ws_url_fallback: String, + batcher_signer: Arc, + batcher_signer_fallback: Arc, chain_id: U256, payment_service: BatcherPaymentService, payment_service_fallback: BatcherPaymentService, - batcher_signer: Arc, - batcher_signer_fallback: Arc, service_manager: ServiceManager, service_manager_fallback: ServiceManager, batch_state: Mutex, @@ -225,11 +225,11 @@ impl Batcher { download_endpoint, eth_ws_url: config.eth_ws_url, eth_ws_url_fallback: config.eth_ws_url_fallback, + batcher_signer, + batcher_signer_fallback, chain_id, payment_service, payment_service_fallback, - batcher_signer, - batcher_signer_fallback, service_manager, service_manager_fallback, max_block_interval: config.batcher.block_interval, @@ -1347,7 +1347,6 @@ impl Batcher { Ok(()) } - /// Gets the balance of user with address `addr` from Ethereum using exponential backoff. /// Gets the balance of user with address `addr` from Ethereum using exponential backoff. /// Returns `None` if the balance couldn't be returned /// FIXME: This should return a `Result` instead. @@ -1368,8 +1367,6 @@ impl Batcher { .ok() } - /// Checks if the user's balance is unlocked for a given address using exponential backoff. - /// Returns `false` if an error occurs during the retries. /// Checks if the user's balance is unlocked for a given address using exponential backoff. /// Returns `false` if an error occurs during the retries. async fn user_balance_is_unlocked(&self, addr: &Address) -> bool { From 9fac61111530525db711120bac00749718a6e074 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 4 Nov 2024 14:52:33 -0300 Subject: [PATCH 84/90] refactor: improve error messages --- batcher/aligned-batcher/src/retry/batcher_retryables.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/retry/batcher_retryables.rs b/batcher/aligned-batcher/src/retry/batcher_retryables.rs index 2a3c630981..14dddbaaa9 100644 --- a/batcher/aligned-batcher/src/retry/batcher_retryables.rs +++ b/batcher/aligned-batcher/src/retry/batcher_retryables.rs @@ -202,7 +202,9 @@ pub async fn cancel_create_new_task_retryable( .await .map_err(|e| { warn!("Timeout while waiting for transaction inclusion: {e}"); - RetryError::Transient(ProviderError::CustomError("Receipt not found".to_string())) + RetryError::Transient(ProviderError::CustomError(format!( + "Timeout while waiting for transaction inclusion: {e}" + ))) })? .map_err(|e| { warn!("Error while waiting for tx inclusion: {e}"); From 25b2aa32f31b10e49f51533a5b17da96a309e1f4 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 4 Nov 2024 14:53:33 -0300 Subject: [PATCH 85/90] refactor: improve error messages --- config-files/config-batcher-docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-files/config-batcher-docker.yaml b/config-files/config-batcher-docker.yaml index 216cb8eee7..2340211c51 100644 --- a/config-files/config-batcher-docker.yaml +++ b/config-files/config-batcher-docker.yaml @@ -18,6 +18,7 @@ ecdsa: batcher: block_interval: 3 batch_size_interval: 10 + transaction_wait_timeout: 36000 # 3 blocks max_proof_size: 67108864 # 64 MiB max_batch_size: 268435456 # 256 MiB eth_ws_reconnects: 99999999999999 From e3f0eac87536a08464f99d25d35e187753d21514 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 5 Nov 2024 15:37:59 -0300 Subject: [PATCH 86/90] docs: add a comment to revisit gas price limit in the future --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index f9429b9623..4a85d81f41 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1298,7 +1298,7 @@ impl Batcher { DEFAULT_MIN_RETRY_DELAY, DEFAULT_BACKOFF_FACTOR, usize::MAX, - ) + ) // TODO (#1380): Set a gas price limit when canceling a created task .await { error!("Could not cancel createNewTask transaction: {e}"); From 7d6c2e26f125dcbcfc35bb6a5890899b0286e43d Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 5 Nov 2024 15:51:33 -0300 Subject: [PATCH 87/90] refactor: improve comments --- batcher/aligned-batcher/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 4a85d81f41..9dab18c7e9 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1260,7 +1260,8 @@ impl Batcher { } // Sends a transaction to Ethereum with the same nonce as the previous one to override it. - // We use the maximum number of retries here because this transaction must be included. + // Uses exponential backoff with the maximum number of retries because this transaction must be included. + // Bumps the gas price used and waits for 3 blocks on each attepmt to get the Receipt of the cancel transaction. pub async fn cancel_create_new_task_tx(&self, old_tx_gas_price: U256) { info!("Cancelling createNewTask transaction..."); let iteration = Arc::new(Mutex::new(0)); From c7beeb740d6884e9cb4739bb266eb180fd9ae129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Avila=20Gast=C3=B3n?= <72628438+avilagaston9@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:45:28 -0300 Subject: [PATCH 88/90] Update batcher/aligned-batcher/src/lib.rs Co-authored-by: Mauro Toscano <12560266+MauroToscano@users.noreply.github.com> --- batcher/aligned-batcher/src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 9dab18c7e9..c2d1fb98cb 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1259,9 +1259,11 @@ impl Batcher { } } - // Sends a transaction to Ethereum with the same nonce as the previous one to override it. - // Uses exponential backoff with the maximum number of retries because this transaction must be included. - // Bumps the gas price used and waits for 3 blocks on each attepmt to get the Receipt of the cancel transaction. + /// Sends a transaction to Ethereum with the same nonce as the previous one to override it. + /// In case of a recoverable error, it will retry with an exponential backoff forever + /// A tx not included in 3 blocks will be considered an error, and will trigger a bump of the fee, with the rules on ```get_bumped_gas_price``` + /// This will do 5 bumps every 3 blocks, and then the exponential backoff will dominate, doing bumps at 8,13,24,45,89 and so on + /// Errors on get_gas_price are considered transient, so this function will keep going pub async fn cancel_create_new_task_tx(&self, old_tx_gas_price: U256) { info!("Cancelling createNewTask transaction..."); let iteration = Arc::new(Mutex::new(0)); From eacc20bb00a856c9307aba3181983a02ef232df0 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 5 Nov 2024 18:56:59 -0300 Subject: [PATCH 89/90] refactor: rename get_bumped_gas_price to calculate_bumped_gas_price --- batcher/aligned-batcher/src/eth/utils.rs | 8 ++++---- batcher/aligned-batcher/src/lib.rs | 16 ++++++++-------- batcher/aligned-sdk/src/core/constants.rs | 1 + 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/batcher/aligned-batcher/src/eth/utils.rs b/batcher/aligned-batcher/src/eth/utils.rs index 15aeeed0dc..657e271cdf 100644 --- a/batcher/aligned-batcher/src/eth/utils.rs +++ b/batcher/aligned-batcher/src/eth/utils.rs @@ -44,7 +44,7 @@ pub async fn get_batcher_signer( /// Calculates an increased gas price for retrying a transaction override. /// The gas price rises with each retry by applying a multiplier based on the iteration count. -pub fn get_bumped_gas_price( +pub fn calculate_bumped_gas_price( previous_gas_price: U256, current_gas_price: U256, iteration: usize, @@ -109,7 +109,7 @@ mod tests { let expected = U256::from(1440); // (1200 * (120 + 0)) / 100 assert_eq!( - get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), + calculate_bumped_gas_price(previous_gas_price, current_gas_price, iteration), expected ); } @@ -123,7 +123,7 @@ mod tests { let expected = U256::from(1560); // (1200 * (120 + 10) / 100 assert_eq!( - get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), + calculate_bumped_gas_price(previous_gas_price, current_gas_price, iteration), expected ); } @@ -137,7 +137,7 @@ mod tests { let expected = U256::from(1875); // (1500 * (120 + 5) / 100 assert_eq!( - get_bumped_gas_price(previous_gas_price, current_gas_price, iteration), + calculate_bumped_gas_price(previous_gas_price, current_gas_price, iteration), expected ); } diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index c2d1fb98cb..4f1138a606 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -3,7 +3,7 @@ use config::NonPayingConfig; use connection::{send_message, WsMessageSink}; use dotenvy::dotenv; use eth::service_manager::ServiceManager; -use eth::utils::{get_batcher_signer, get_bumped_gas_price, get_gas_price}; +use eth::utils::{calculate_bumped_gas_price, get_batcher_signer, get_gas_price}; use ethers::contract::ContractError; use ethers::signers::Signer; use retry::batcher_retryables::{ @@ -20,8 +20,8 @@ use std::net::SocketAddr; use std::sync::Arc; use aligned_sdk::core::constants::{ - ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CONSTANT_GAS_COST, - DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_BACKOFF_FACTOR, + ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF, AGGREGATOR_GAS_COST, CANCEL_TRANSACTION_MAX_RETRIES, + CONSTANT_GAS_COST, DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER, DEFAULT_BACKOFF_FACTOR, DEFAULT_MAX_FEE_PER_PROOF, DEFAULT_MAX_RETRIES, DEFAULT_MIN_RETRY_DELAY, GAS_PRICE_PERCENTAGE_MULTIPLIER, MIN_FEE_PER_PROOF, PERCENTAGE_DIVIDER, RESPOND_TO_TASK_FEE_LIMIT_PERCENTAGE_MULTIPLIER, @@ -1260,10 +1260,10 @@ impl Batcher { } /// Sends a transaction to Ethereum with the same nonce as the previous one to override it. - /// In case of a recoverable error, it will retry with an exponential backoff forever - /// A tx not included in 3 blocks will be considered an error, and will trigger a bump of the fee, with the rules on ```get_bumped_gas_price``` + /// In case of a recoverable error, it will retry with an exponential backoff up to CANCEL_TRANSACTION_MAX_RETRIES times. + /// A tx not included in 3 blocks will be considered an error, and will trigger a bump of the fee, with the rules on ```calculate_bumped_gas_price``` /// This will do 5 bumps every 3 blocks, and then the exponential backoff will dominate, doing bumps at 8,13,24,45,89 and so on - /// Errors on get_gas_price are considered transient, so this function will keep going + /// Errors on get_gas_price are considered transient, so this function will keep going pub async fn cancel_create_new_task_tx(&self, old_tx_gas_price: U256) { info!("Cancelling createNewTask transaction..."); let iteration = Arc::new(Mutex::new(0)); @@ -1285,7 +1285,7 @@ impl Batcher { }; let bumped_gas_price = - get_bumped_gas_price(*previous_gas_price, current_gas_price, *iteration); + calculate_bumped_gas_price(*previous_gas_price, current_gas_price, *iteration); *iteration += 1; *previous_gas_price = bumped_gas_price; @@ -1300,7 +1300,7 @@ impl Batcher { }, DEFAULT_MIN_RETRY_DELAY, DEFAULT_BACKOFF_FACTOR, - usize::MAX, + CANCEL_TRANSACTION_MAX_RETRIES, ) // TODO (#1380): Set a gas price limit when canceling a created task .await { diff --git a/batcher/aligned-sdk/src/core/constants.rs b/batcher/aligned-sdk/src/core/constants.rs index f6570045f7..00536deff3 100644 --- a/batcher/aligned-sdk/src/core/constants.rs +++ b/batcher/aligned-sdk/src/core/constants.rs @@ -1,5 +1,6 @@ /// Batcher /// pub const GAS_PRICE_INCREMENT_PERCENTAGE_PER_ITERATION: usize = 5; +pub const CANCEL_TRANSACTION_MAX_RETRIES: usize = 20; pub const AGGREGATOR_GAS_COST: u128 = 400_000; pub const BATCHER_SUBMISSION_BASE_GAS_COST: u128 = 125_000; pub const ADDITIONAL_SUBMISSION_GAS_COST_PER_PROOF: u128 = 13_000; From 1b7fce8b83c21ae4a4f962d35300b8fa3fe115f8 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 5 Nov 2024 19:19:02 -0300 Subject: [PATCH 90/90] refactor: improve docs on cancel_create_new_task --- batcher/aligned-batcher/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 4f1138a606..c941bb94c8 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1262,8 +1262,9 @@ impl Batcher { /// Sends a transaction to Ethereum with the same nonce as the previous one to override it. /// In case of a recoverable error, it will retry with an exponential backoff up to CANCEL_TRANSACTION_MAX_RETRIES times. /// A tx not included in 3 blocks will be considered an error, and will trigger a bump of the fee, with the rules on ```calculate_bumped_gas_price``` - /// This will do 5 bumps every 3 blocks, and then the exponential backoff will dominate, doing bumps at 8,13,24,45,89 and so on - /// Errors on get_gas_price are considered transient, so this function will keep going + /// This will do 5 bumps every 3 blocks, and then the exponential backoff will dominate, doing bumps at 8,13,24,45,89 and so on. + /// Errors on ```get_gas_price``` calls inside this function are considered transient, + /// so they won't stop the retries. pub async fn cancel_create_new_task_tx(&self, old_tx_gas_price: U256) { info!("Cancelling createNewTask transaction..."); let iteration = Arc::new(Mutex::new(0)); @@ -1301,7 +1302,7 @@ impl Batcher { DEFAULT_MIN_RETRY_DELAY, DEFAULT_BACKOFF_FACTOR, CANCEL_TRANSACTION_MAX_RETRIES, - ) // TODO (#1380): Set a gas price limit when canceling a created task + ) .await { error!("Could not cancel createNewTask transaction: {e}");