From 9f706efda503e366423820bb0786b28c4a003f81 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 12 Nov 2024 20:23:54 -0300 Subject: [PATCH 1/6] feat: batch cant have more proofs than max_batch_len --- batcher/aligned-batcher/src/config/mod.rs | 1 + batcher/aligned-batcher/src/lib.rs | 4 +- .../aligned-batcher/src/types/batch_queue.rs | 120 +++++++++++++++++- config-files/config-batcher.yaml | 1 + 4 files changed, 120 insertions(+), 6 deletions(-) diff --git a/batcher/aligned-batcher/src/config/mod.rs b/batcher/aligned-batcher/src/config/mod.rs index b4f6ba0ada..d35365680d 100644 --- a/batcher/aligned-batcher/src/config/mod.rs +++ b/batcher/aligned-batcher/src/config/mod.rs @@ -41,6 +41,7 @@ pub struct BatcherConfigFromYaml { pub transaction_wait_timeout: u64, pub max_proof_size: usize, pub max_batch_size: usize, + pub max_batch_len: usize, pub pre_verification_is_enabled: bool, pub metrics_port: u16, pub telemetry_ip_port_address: String, diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index f27ca723e3..6093a74e23 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -82,6 +82,7 @@ pub struct Batcher { transaction_wait_timeout: u64, max_proof_size: usize, max_batch_size: usize, + max_batch_len: usize, last_uploaded_batch_block: Mutex, pre_verification_is_enabled: bool, non_paying_config: Option, @@ -244,6 +245,7 @@ impl Batcher { transaction_wait_timeout: config.batcher.transaction_wait_timeout, max_proof_size: config.batcher.max_proof_size, max_batch_size: config.batcher.max_batch_size, + max_batch_len: config.batcher.max_batch_len, last_uploaded_batch_block: Mutex::new(last_uploaded_batch_block), pre_verification_is_enabled: config.batcher.pre_verification_is_enabled, non_paying_config, @@ -1048,7 +1050,7 @@ impl Batcher { *batch_posting = true; let batch_queue_copy = batch_state_lock.batch_queue.clone(); let (resulting_batch_queue, finalized_batch) = - batch_queue::try_build_batch(batch_queue_copy, gas_price, self.max_batch_size) + batch_queue::try_build_batch(batch_queue_copy, gas_price, self.max_batch_size, self.max_batch_len) .inspect_err(|e| { *batch_posting = false; match e { diff --git a/batcher/aligned-batcher/src/types/batch_queue.rs b/batcher/aligned-batcher/src/types/batch_queue.rs index c90219d237..3a6c52e858 100644 --- a/batcher/aligned-batcher/src/types/batch_queue.rs +++ b/batcher/aligned-batcher/src/types/batch_queue.rs @@ -147,6 +147,7 @@ pub(crate) fn try_build_batch( batch_queue: BatchQueue, gas_price: U256, max_batch_size: usize, + max_batch_len: usize, ) -> Result<(BatchQueue, Vec), BatcherError> { let mut batch_queue = batch_queue; let mut batch_size = calculate_batch_size(&batch_queue)?; @@ -156,7 +157,7 @@ pub(crate) fn try_build_batch( let batch_len = batch_queue.len(); let fee_per_proof = calculate_fee_per_proof(batch_len, gas_price); - if batch_size > max_batch_size || fee_per_proof > entry.nonced_verification_data.max_fee { + if batch_size > max_batch_size || fee_per_proof > entry.nonced_verification_data.max_fee || batch_len > max_batch_len { // Update the state for the next iteration: // * Subtract this entry size to the size of the batch size. // * Push the current entry to the resulting batch queue. @@ -298,7 +299,7 @@ mod test { let gas_price = U256::from(1); let (resulting_batch_queue, batch) = - try_build_batch(batch_queue, gas_price, 5000000).unwrap(); + try_build_batch(batch_queue, gas_price, 5000000, 50).unwrap(); assert!(resulting_batch_queue.is_empty()); @@ -401,7 +402,7 @@ mod test { let gas_price = U256::from(1); let (resulting_batch_queue, finalized_batch) = - try_build_batch(batch_queue, gas_price, 5000000).unwrap(); + try_build_batch(batch_queue, gas_price, 5000000, 50).unwrap(); // The resulting batch queue (entries from the old batch queue that were not willing to pay // in this batch), should be empty and hence, all entries from the batch queue should be in @@ -512,7 +513,7 @@ mod test { let gas_price = U256::from(1); let (resulting_batch_queue, finalized_batch) = - try_build_batch(batch_queue, gas_price, 5000000).unwrap(); + try_build_batch(batch_queue, gas_price, 5000000, 50).unwrap(); // The resulting batch queue (entries from the old batch queue that were not willing to pay // in this batch), should be empty and hence, all entries from the batch queue should be in @@ -529,4 +530,113 @@ mod test { max_fee_1 ); } -} + + #[test] + fn batch_finalization_algorithm_works_not_bigger_than_max_batch_len() { + // The following information will be the same for each entry, it is just some dummy data to see + // algorithm working. + + let proof_generator_addr = Address::random(); + let payment_service_addr = Address::random(); + let sender_addr = Address::random(); + let bytes_for_verification_data = vec![42_u8; 10]; + let dummy_signature = Signature { + r: U256::from(1), + s: U256::from(2), + v: 3, + }; + let verification_data = VerificationData { + proving_system: ProvingSystemId::Risc0, + proof: bytes_for_verification_data.clone(), + pub_input: Some(bytes_for_verification_data.clone()), + verification_key: Some(bytes_for_verification_data.clone()), + vm_program_code: Some(bytes_for_verification_data), + proof_generator_addr, + }; + let chain_id = U256::from(42); + + // Here we create different entries for the batch queue. + // Since we are sending with the same address, the low nonces should have higher max fees. + + // Entry 1 + let nonce_1 = U256::from(1); + let max_fee_1 = U256::from(1_300_000_000_000_002u128); + let nonced_verification_data_1 = NoncedVerificationData::new( + verification_data.clone(), + nonce_1, + max_fee_1, + chain_id, + payment_service_addr, + ); + let vd_commitment_1: VerificationDataCommitment = nonced_verification_data_1.clone().into(); + let entry_1 = BatchQueueEntry::new_for_testing( + nonced_verification_data_1, + vd_commitment_1, + dummy_signature, + sender_addr, + ); + let batch_priority_1 = BatchQueueEntryPriority::new(max_fee_1, nonce_1); + + // Entry 2 + let nonce_2 = U256::from(2); + let max_fee_2 = U256::from(1_300_000_000_000_001u128); + let nonced_verification_data_2 = NoncedVerificationData::new( + verification_data.clone(), + nonce_2, + max_fee_2, + chain_id, + payment_service_addr, + ); + let vd_commitment_2: VerificationDataCommitment = nonced_verification_data_2.clone().into(); + let entry_2 = BatchQueueEntry::new_for_testing( + nonced_verification_data_2, + vd_commitment_2, + dummy_signature, + sender_addr, + ); + let batch_priority_2 = BatchQueueEntryPriority::new(max_fee_2, nonce_2); + + // Entry 3 + let nonce_3 = U256::from(3); + let max_fee_3 = U256::from(1_300_000_000_000_000u128); + let nonced_verification_data_3 = NoncedVerificationData::new( + verification_data.clone(), + nonce_3, + max_fee_3, + chain_id, + payment_service_addr, + ); + let vd_commitment_3: VerificationDataCommitment = nonced_verification_data_3.clone().into(); + let entry_3 = BatchQueueEntry::new_for_testing( + nonced_verification_data_3, + vd_commitment_3, + dummy_signature, + sender_addr, + ); + let batch_priority_3 = BatchQueueEntryPriority::new(max_fee_3, nonce_3); + + let mut batch_queue = BatchQueue::new(); + batch_queue.push(entry_1, batch_priority_1); + batch_queue.push(entry_2, batch_priority_2); + batch_queue.push(entry_3, batch_priority_3); + + let gas_price = U256::from(1); + + let max_batch_len = 2; + + let (resulting_batch_queue, finalized_batch) = + try_build_batch(batch_queue, gas_price, 5000000, max_batch_len).unwrap(); + + assert_eq!(resulting_batch_queue.len(), 1); + assert_eq!(finalized_batch.len(), 2); + assert_eq!( + finalized_batch[0].nonced_verification_data.max_fee, + max_fee_2 + ); + assert_eq!( + finalized_batch[1].nonced_verification_data.max_fee, + max_fee_1 + ); + } + +} \ No newline at end of file diff --git a/config-files/config-batcher.yaml b/config-files/config-batcher.yaml index 8dd5f5b8a7..92d0bdc30e 100644 --- a/config-files/config-batcher.yaml +++ b/config-files/config-batcher.yaml @@ -21,6 +21,7 @@ batcher: transaction_wait_timeout: 36000 # 3 blocks max_proof_size: 67108864 # 64 MiB max_batch_size: 268435456 # 256 MiB + max_batch_len: 3000 # 3000 proofs in a batch pre_verification_is_enabled: true metrics_port: 9093 telemetry_ip_port_address: localhost:4001 From adffc81163ca1b1907d016b394699f5114cf04f7 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 12 Nov 2024 20:24:36 -0300 Subject: [PATCH 2/6] chore: add comment in test --- batcher/aligned-batcher/src/types/batch_queue.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/batcher/aligned-batcher/src/types/batch_queue.rs b/batcher/aligned-batcher/src/types/batch_queue.rs index 3a6c52e858..768a9a152f 100644 --- a/batcher/aligned-batcher/src/types/batch_queue.rs +++ b/batcher/aligned-batcher/src/types/batch_queue.rs @@ -622,6 +622,7 @@ mod test { let gas_price = U256::from(1); + // The max batch len is 2, so the algorithm should stop at the second entry. let max_batch_len = 2; let (resulting_batch_queue, finalized_batch) = From 134f195493b500fab23657c34597feb6397b7031 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 12 Nov 2024 20:30:40 -0300 Subject: [PATCH 3/6] fix: detail --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9a8ed77208..6a70b54ab7 100644 --- a/Makefile +++ b/Makefile @@ -397,7 +397,7 @@ batcher_send_plonk_bn254_burst: batcher/target/release/aligned --vk ../../scripts/test_files/gnark_plonk_bn254_script/plonk.vk \ --proof_generator_addr 0x66f9664f97F2b50F62D13eA064982f936dE76657 \ --rpc_url $(RPC_URL) \ - --repetitions 4 \ + --repetitions $(BURST_SIZE) \ --network $(NETWORK) batcher_send_plonk_bls12_381_task: batcher/target/release/aligned From da505812f3901b188cffa7a00579c437fe3c9549 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 12 Nov 2024 20:45:26 -0300 Subject: [PATCH 4/6] chore: cargo fmt --- batcher/aligned-batcher/src/lib.rs | 34 +++++++++++-------- .../aligned-batcher/src/types/batch_queue.rs | 8 +++-- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 6093a74e23..e3477233f1 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1049,21 +1049,25 @@ impl Batcher { // Set the batch posting flag to true *batch_posting = true; let batch_queue_copy = batch_state_lock.batch_queue.clone(); - let (resulting_batch_queue, finalized_batch) = - batch_queue::try_build_batch(batch_queue_copy, gas_price, self.max_batch_size, self.max_batch_len) - .inspect_err(|e| { - *batch_posting = false; - match e { - // We can't post a batch since users are not willing to pay the needed fee, wait for more proofs - BatcherError::BatchCostTooHigh => { - info!("No working batch found. Waiting for more proofs") - } - // FIXME: We should refactor this code and instead of returning None, return an error. - // See issue https://github.com/yetanotherco/aligned_layer/issues/1046. - e => error!("Unexpected error: {:?}", e), - } - }) - .ok()?; + let (resulting_batch_queue, finalized_batch) = batch_queue::try_build_batch( + batch_queue_copy, + gas_price, + self.max_batch_size, + self.max_batch_len, + ) + .inspect_err(|e| { + *batch_posting = false; + match e { + // We can't post a batch since users are not willing to pay the needed fee, wait for more proofs + BatcherError::BatchCostTooHigh => { + info!("No working batch found. Waiting for more proofs") + } + // FIXME: We should refactor this code and instead of returning None, return an error. + // See issue https://github.com/yetanotherco/aligned_layer/issues/1046. + e => error!("Unexpected error: {:?}", e), + } + }) + .ok()?; batch_state_lock.batch_queue = resulting_batch_queue; let updated_user_proof_count_and_min_fee = diff --git a/batcher/aligned-batcher/src/types/batch_queue.rs b/batcher/aligned-batcher/src/types/batch_queue.rs index 768a9a152f..1e3c774b13 100644 --- a/batcher/aligned-batcher/src/types/batch_queue.rs +++ b/batcher/aligned-batcher/src/types/batch_queue.rs @@ -157,7 +157,10 @@ pub(crate) fn try_build_batch( let batch_len = batch_queue.len(); let fee_per_proof = calculate_fee_per_proof(batch_len, gas_price); - if batch_size > max_batch_size || fee_per_proof > entry.nonced_verification_data.max_fee || batch_len > max_batch_len { + if batch_size > max_batch_size + || fee_per_proof > entry.nonced_verification_data.max_fee + || batch_len > max_batch_len + { // Update the state for the next iteration: // * Subtract this entry size to the size of the batch size. // * Push the current entry to the resulting batch queue. @@ -639,5 +642,4 @@ mod test { max_fee_1 ); } - -} \ No newline at end of file +} From df0fcce089e8b5b5a2b6063b7c460542923db2d9 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 13 Nov 2024 19:25:50 -0300 Subject: [PATCH 5/6] chore: add max_batch_len to docker batcher config --- config-files/config-batcher-docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-files/config-batcher-docker.yaml b/config-files/config-batcher-docker.yaml index ab847b2780..9ba391bddf 100644 --- a/config-files/config-batcher-docker.yaml +++ b/config-files/config-batcher-docker.yaml @@ -21,6 +21,7 @@ batcher: transaction_wait_timeout: 36000 # 3 blocks max_proof_size: 67108864 # 64 MiB max_batch_size: 268435456 # 256 MiB + max_batch_len: 3000 # 3000 proofs in a batch eth_ws_reconnects: 99999999999999 pre_verification_is_enabled: true metrics_port: 9093 From 256124dd8dc1adf21dbf3fed96e882d1fe9dab0c Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:34:49 -0300 Subject: [PATCH 6/6] refactor: batch_size and batch_len names refactor --- batcher/aligned-batcher/src/config/mod.rs | 4 ++-- batcher/aligned-batcher/src/lib.rs | 12 ++++++------ batcher/aligned-batcher/src/types/batch_queue.rs | 14 +++++++------- config-files/config-batcher-docker.yaml | 4 ++-- config-files/config-batcher.yaml | 4 ++-- config-files/config-docker.yaml | 4 ++-- config-files/config.yaml | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/batcher/aligned-batcher/src/config/mod.rs b/batcher/aligned-batcher/src/config/mod.rs index d35365680d..3a6f01e349 100644 --- a/batcher/aligned-batcher/src/config/mod.rs +++ b/batcher/aligned-batcher/src/config/mod.rs @@ -40,8 +40,8 @@ pub struct BatcherConfigFromYaml { pub block_interval: u64, pub transaction_wait_timeout: u64, pub max_proof_size: usize, - pub max_batch_size: usize, - pub max_batch_len: usize, + pub max_batch_byte_size: usize, + pub max_batch_proof_qty: usize, pub pre_verification_is_enabled: bool, pub metrics_port: u16, pub telemetry_ip_port_address: String, diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index f990c4c97f..50c08a863f 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -82,8 +82,8 @@ pub struct Batcher { max_block_interval: u64, transaction_wait_timeout: u64, max_proof_size: usize, - max_batch_size: usize, - max_batch_len: usize, + max_batch_byte_size: usize, + max_batch_proof_qty: usize, last_uploaded_batch_block: Mutex, pre_verification_is_enabled: bool, non_paying_config: Option, @@ -245,8 +245,8 @@ impl Batcher { max_block_interval: config.batcher.block_interval, transaction_wait_timeout: config.batcher.transaction_wait_timeout, max_proof_size: config.batcher.max_proof_size, - max_batch_size: config.batcher.max_batch_size, - max_batch_len: config.batcher.max_batch_len, + max_batch_byte_size: config.batcher.max_batch_byte_size, + max_batch_proof_qty: config.batcher.max_batch_proof_qty, last_uploaded_batch_block: Mutex::new(last_uploaded_batch_block), pre_verification_is_enabled: config.batcher.pre_verification_is_enabled, non_paying_config, @@ -1060,8 +1060,8 @@ impl Batcher { let (resulting_batch_queue, finalized_batch) = batch_queue::try_build_batch( batch_queue_copy, gas_price, - self.max_batch_size, - self.max_batch_len, + self.max_batch_byte_size, + self.max_batch_proof_qty, ) .inspect_err(|e| { *batch_posting = false; diff --git a/batcher/aligned-batcher/src/types/batch_queue.rs b/batcher/aligned-batcher/src/types/batch_queue.rs index 1e3c774b13..17918e7a21 100644 --- a/batcher/aligned-batcher/src/types/batch_queue.rs +++ b/batcher/aligned-batcher/src/types/batch_queue.rs @@ -146,8 +146,8 @@ pub(crate) fn calculate_batch_size(batch_queue: &BatchQueue) -> Result Result<(BatchQueue, Vec), BatcherError> { let mut batch_queue = batch_queue; let mut batch_size = calculate_batch_size(&batch_queue)?; @@ -157,9 +157,9 @@ pub(crate) fn try_build_batch( let batch_len = batch_queue.len(); let fee_per_proof = calculate_fee_per_proof(batch_len, gas_price); - if batch_size > max_batch_size + if batch_size > max_batch_byte_size || fee_per_proof > entry.nonced_verification_data.max_fee - || batch_len > max_batch_len + || batch_len > max_batch_proof_qty { // Update the state for the next iteration: // * Subtract this entry size to the size of the batch size. @@ -535,7 +535,7 @@ mod test { } #[test] - fn batch_finalization_algorithm_works_not_bigger_than_max_batch_len() { + fn batch_finalization_algorithm_works_not_bigger_than_max_batch_proof_qty() { // The following information will be the same for each entry, it is just some dummy data to see // algorithm working. @@ -626,10 +626,10 @@ mod test { let gas_price = U256::from(1); // The max batch len is 2, so the algorithm should stop at the second entry. - let max_batch_len = 2; + let max_batch_proof_qty = 2; let (resulting_batch_queue, finalized_batch) = - try_build_batch(batch_queue, gas_price, 5000000, max_batch_len).unwrap(); + try_build_batch(batch_queue, gas_price, 5000000, max_batch_proof_qty).unwrap(); assert_eq!(resulting_batch_queue.len(), 1); assert_eq!(finalized_batch.len(), 2); diff --git a/config-files/config-batcher-docker.yaml b/config-files/config-batcher-docker.yaml index 9ba391bddf..e9c8f7d5ca 100644 --- a/config-files/config-batcher-docker.yaml +++ b/config-files/config-batcher-docker.yaml @@ -20,8 +20,8 @@ batcher: batch_size_interval: 10 transaction_wait_timeout: 36000 # 3 blocks max_proof_size: 67108864 # 64 MiB - max_batch_size: 268435456 # 256 MiB - max_batch_len: 3000 # 3000 proofs in a batch + max_batch_byte_size: 268435456 # 256 MiB + max_batch_proof_qty: 3000 # 3000 proofs in a batch eth_ws_reconnects: 99999999999999 pre_verification_is_enabled: true metrics_port: 9093 diff --git a/config-files/config-batcher.yaml b/config-files/config-batcher.yaml index 92d0bdc30e..6d47bd7d5c 100644 --- a/config-files/config-batcher.yaml +++ b/config-files/config-batcher.yaml @@ -20,8 +20,8 @@ batcher: batch_size_interval: 10 transaction_wait_timeout: 36000 # 3 blocks max_proof_size: 67108864 # 64 MiB - max_batch_size: 268435456 # 256 MiB - max_batch_len: 3000 # 3000 proofs in a batch + max_batch_byte_size: 268435456 # 256 MiB + max_batch_proof_qty: 3000 # 3000 proofs in a batch pre_verification_is_enabled: true metrics_port: 9093 telemetry_ip_port_address: localhost:4001 diff --git a/config-files/config-docker.yaml b/config-files/config-docker.yaml index 712edf11ce..18c7be892f 100644 --- a/config-files/config-docker.yaml +++ b/config-files/config-docker.yaml @@ -24,7 +24,7 @@ batcher: block_interval: 3 batch_size_interval: 10 max_proof_size: 67108864 # 64 MiB - max_batch_size: 268435456 # 256 MiB + max_batch_byte_size: 268435456 # 256 MiB eth_ws_reconnects: 99999999999999 pre_verification_is_enabled: true @@ -47,7 +47,7 @@ operator: metadata_url: "https://yetanotherco.github.io/operator_metadata/metadata.json" enable_metrics: true metrics_ip_port_address: localhost:9092 - max_batch_size: 268435456 # 256 MiB + max_batch_byte_size: 268435456 # 256 MiB last_processed_batch_filepath: config-files/operator.last_processed_batch.json # Operators variables needed for register it in EigenLayer el_delegation_manager_address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" diff --git a/config-files/config.yaml b/config-files/config.yaml index d11584da06..3f53d6617a 100644 --- a/config-files/config.yaml +++ b/config-files/config.yaml @@ -24,7 +24,7 @@ batcher: block_interval: 3 batch_size_interval: 10 max_proof_size: 67108864 # 64 MiB - max_batch_size: 268435456 # 256 MiB + max_batch_byte_size: 268435456 # 256 MiB pre_verification_is_enabled: true metrics_port: 9093