From be0fb91d78bb32acb364e27531845d5dc9a2d082 Mon Sep 17 00:00:00 2001 From: Sergey Timoshin Date: Fri, 13 Dec 2024 12:01:35 +0000 Subject: [PATCH 1/2] refactor: add support for batched updates Update src/dao/generated/accounts.rs Co-authored-by: ananas-block <58553958+ananas-block@users.noreply.github.com> Update src/ingester/parser/batch_event_parser.rs Co-authored-by: ananas-block <58553958+ananas-block@users.noreply.github.com> refactor: remove getSubtrees method and related API documentation Update src/ingester/parser/mod.rs Co-authored-by: ananas-block <58553958+ananas-block@users.noreply.github.com> Refactor struct fields to use camelCase naming convention replace the `calculate_two_inputs_hash_chain` function with the `create_two_inputs_hash_chain` method from the `light_compressed_account` crate. Refactor error handling in `parse_public_transaction_event_v2` format Remove obsolete and commented-out account update code Move `node_index_to_leaf_index` function to the appropriate location and remove dead code Add comments to clarify nullifier field usage Add comments to clarify tx_hash field usage in account struct Add comments to clarify seq field usage in account struct Add comments to clarify nullifier_queue_index field usage in account struct Refactor get_compressed_accounts_by_owner module and add common utilities for account filtering add validity proof v2 Add get_validity_proof_v2 and update API specifications remove unnecessary logging fix: mock tests fix for getValidityProof (v1) refactor: remove unused tree height parameters and add getValidityProofV2 method fix: update method name from address_from_bytes to state_from_bytes in mod.rs consistency. fix indexed_accounts query fix get_compressed_account_by_owner v1 refactor: simplify account handling and improve code consistency in transaction processing refactor: remove queue_position handling and update related queries and indexes upd .gitignore cargo fmt and fixed tests --- .gitignore | 5 +++- Cargo.lock | 1 + Cargo.toml | 3 +++ src/api/method/get_queue_elements.rs | 4 +++ src/ingester/parser/state_update.rs | 2 ++ src/ingester/persist/mod.rs | 3 +++ src/monitor/mod.rs | 1 + tests/integration_tests/mock_tests.rs | 38 ++++++++++----------------- 8 files changed, 32 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 55636461..449ca0d1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ test-ledger/ /target /.cargo -.DS_Store \ No newline at end of file +.DS_Store + +minio +test.db diff --git a/Cargo.lock b/Cargo.lock index 29066873..524d1656 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5033,6 +5033,7 @@ dependencies = [ "sea-orm-migration", "serde", "serde_json", + "serde_yaml", "serial_test", "solana-client", "solana-program", diff --git a/Cargo.toml b/Cargo.toml index 92baf8b0..c035878f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,9 @@ rust-s3 = "0.34.0" lru = "0.12.0" light-client = "0.9.1" +serde_yaml = "0.9" + + [dev-dependencies] function_name = "0.3.0" serial_test = "2.0.0" diff --git a/src/api/method/get_queue_elements.rs b/src/api/method/get_queue_elements.rs index c1e1f7f9..c2a071c5 100644 --- a/src/api/method/get_queue_elements.rs +++ b/src/api/method/get_queue_elements.rs @@ -1,4 +1,8 @@ use light_merkle_tree_metadata::queue::QueueType; +<<<<<<< HEAD +======= +use log::info; +>>>>>>> d7e9eeb (refactor: add support for batched updates) use sea_orm::{ ConnectionTrait, DatabaseBackend, DatabaseConnection, FromQueryResult, Statement, TransactionTrait, diff --git a/src/ingester/parser/state_update.rs b/src/ingester/parser/state_update.rs index b75ca56a..67fea0f5 100644 --- a/src/ingester/parser/state_update.rs +++ b/src/ingester/parser/state_update.rs @@ -92,6 +92,8 @@ impl StateUpdate { pub fn merge_updates(updates: Vec) -> StateUpdate { let mut merged = StateUpdate::default(); + // TODO: remove assert after tx_hash and in_seq_numbers are associated with in_accounts + // assert!(updates.iter().filter(|update| update.tx_hash != Hash::default()).count() <= 1); for update in updates { // legacy merged.in_seq_numbers.extend(update.in_seq_numbers); diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 9e45d156..2853bb45 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -187,6 +187,9 @@ pub async fn persist_state_update( })) .collect(); + for leaf_node in &leaf_nodes_with_signatures { + debug!("Leaf node: {:?}, signature: {:?}", leaf_node.0, leaf_node.1); + } leaf_nodes_with_signatures.sort_by_key(|x| x.0.seq); debug!("Persisting state nodes..."); diff --git a/src/monitor/mod.rs b/src/monitor/mod.rs index d8ca004c..59de29e5 100644 --- a/src/monitor/mod.rs +++ b/src/monitor/mod.rs @@ -77,6 +77,7 @@ pub fn continously_monitor_photon( } } else { let tree_roots = load_db_tree_roots_with_infinite_retry(db.as_ref()).await; + println!("Tree roots (continously_monitor_photon): {:?}", tree_roots); validate_tree_roots(rpc_client.as_ref(), tree_roots).await; } sleep(Duration::from_millis(5000)).await; diff --git a/tests/integration_tests/mock_tests.rs b/tests/integration_tests/mock_tests.rs index 37470da5..063383b3 100644 --- a/tests/integration_tests/mock_tests.rs +++ b/tests/integration_tests/mock_tests.rs @@ -742,10 +742,8 @@ async fn test_persisted_state_trees( }) .collect(); let txn = setup.db_conn.as_ref().begin().await.unwrap(); - let tree_height = 5; - persist_leaf_nodes(&txn, leaf_nodes.clone(), tree_height) - .await - .unwrap(); + let tree_height = 33; // prev. 5 + persist_leaf_nodes(&txn, leaf_nodes.clone()).await.unwrap(); txn.commit().await.unwrap(); let proofs = get_multiple_compressed_leaf_proofs( @@ -779,9 +777,7 @@ async fn test_persisted_state_trees( }) .collect(); let txn = setup.db_conn.as_ref().begin().await.unwrap(); - persist_leaf_nodes(&txn, leaf_nodes.clone(), tree_height) - .await - .unwrap(); + persist_leaf_nodes(&txn, leaf_nodes.clone()).await.unwrap(); txn.commit().await.unwrap(); let leaves = leaf_nodes @@ -823,9 +819,9 @@ async fn test_indexed_merkle_trees( .unwrap(); let values = (0..num_nodes).map(|i| vec![i * 4 + 1]).collect(); - let tree_height = 4; + let tree_height = 33; // prev. 4 - multi_append(&txn, values, tree.to_bytes_vec(), tree_height) + multi_append(&txn, values, tree.to_bytes_vec()) .await .unwrap(); @@ -857,7 +853,7 @@ async fn test_indexed_merkle_trees( let values = vec![vec![3]]; - multi_append(&txn, values, tree.to_bytes_vec(), tree_height) + multi_append(&txn, values, tree.to_bytes_vec()) .await .unwrap(); @@ -1148,12 +1144,10 @@ async fn test_persisted_state_trees_bug_with_latter_smaller_seq_values( ]; let leaf_node_chunks = vec![leaf_nodes_1, leaf_nodes_2]; - let tree_height = 3; + let tree_height = 33; // prev. 3 for chunk in leaf_node_chunks { let txn = setup.db_conn.as_ref().begin().await.unwrap(); - persist_leaf_nodes(&txn, chunk.clone(), tree_height) - .await - .unwrap(); + persist_leaf_nodes(&txn, chunk.clone()).await.unwrap(); txn.commit().await.unwrap(); let proof_address = "12prJNGB6sfTMrZM1Udv2Aamv9fLzpm5YfMqssTmGrWy"; @@ -1273,7 +1267,7 @@ async fn test_persisted_state_trees_multiple_cases( ) { let name = trim_test_name(function_name!()); let tree = SerializablePubkey::new_unique(); - let tree_height = 10; + let tree_height = 33; // prev. 10 info!("Test case 1: Sequential leaf nodes"); let leaf_nodes_1 = create_leaf_nodes(tree, 0..5, |i| i); @@ -1295,7 +1289,7 @@ async fn test_persisted_state_trees_multiple_cases( test_persist_and_verify(name.clone(), db_backend, tree, leaf_nodes_4, tree_height).await; info!("Test case 7: Very large tree"); - let large_tree_height = 20; + let large_tree_height = 33; // prev. 20 let leaf_nodes_7 = create_leaf_nodes(tree, 0..20, |i| i); test_persist_and_verify( name.clone(), @@ -1366,14 +1360,10 @@ async fn test_persist_and_verify( let txn = setup.db_conn.as_ref().begin().await.unwrap(); if one_at_a_time { for leaf_node in leaf_nodes.clone() { - persist_leaf_nodes(&txn, vec![leaf_node], tree_height) - .await - .unwrap(); + persist_leaf_nodes(&txn, vec![leaf_node]).await.unwrap(); } } else { - persist_leaf_nodes(&txn, leaf_nodes.clone(), tree_height) - .await - .unwrap(); + persist_leaf_nodes(&txn, leaf_nodes.clone()).await.unwrap(); } txn.commit().await.unwrap(); @@ -1463,7 +1453,7 @@ async fn test_update_indexed_merkle_tree( let name = trim_test_name(function_name!()); let setup = setup(name.clone(), db_backend).await; let tree = Pubkey::new_unique(); - let tree_height = 10; + let tree_height = 33; // prev. 10 let index = 1; let value = [1; 32]; let index_element_1 = RawIndexedElement { @@ -1492,7 +1482,7 @@ async fn test_update_indexed_merkle_tree( seq: *seq as u64, }, ); - update_indexed_tree_leaves(&txn, indexed_leaf_updates, tree_height) + update_indexed_tree_leaves(&txn, indexed_leaf_updates) .await .unwrap(); } From 6c3c80d250f737a2c0b1e1b11a1e92a1ea599051 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Thu, 20 Feb 2025 16:55:32 +0000 Subject: [PATCH 2/2] fix mock test --- Cargo.lock | 1 - Cargo.toml | 2 -- src/api/method/get_queue_elements.rs | 4 ---- src/ingester/parser/state_update.rs | 3 +-- src/ingester/persist/mod.rs | 3 --- src/monitor/mod.rs | 1 - 6 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 524d1656..29066873 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5033,7 +5033,6 @@ dependencies = [ "sea-orm-migration", "serde", "serde_json", - "serde_yaml", "serial_test", "solana-client", "solana-program", diff --git a/Cargo.toml b/Cargo.toml index c035878f..8ea286d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,8 +124,6 @@ rust-s3 = "0.34.0" lru = "0.12.0" light-client = "0.9.1" -serde_yaml = "0.9" - [dev-dependencies] function_name = "0.3.0" diff --git a/src/api/method/get_queue_elements.rs b/src/api/method/get_queue_elements.rs index c2a071c5..c1e1f7f9 100644 --- a/src/api/method/get_queue_elements.rs +++ b/src/api/method/get_queue_elements.rs @@ -1,8 +1,4 @@ use light_merkle_tree_metadata::queue::QueueType; -<<<<<<< HEAD -======= -use log::info; ->>>>>>> d7e9eeb (refactor: add support for batched updates) use sea_orm::{ ConnectionTrait, DatabaseBackend, DatabaseConnection, FromQueryResult, Statement, TransactionTrait, diff --git a/src/ingester/parser/state_update.rs b/src/ingester/parser/state_update.rs index 67fea0f5..4611cb6a 100644 --- a/src/ingester/parser/state_update.rs +++ b/src/ingester/parser/state_update.rs @@ -92,8 +92,7 @@ impl StateUpdate { pub fn merge_updates(updates: Vec) -> StateUpdate { let mut merged = StateUpdate::default(); - // TODO: remove assert after tx_hash and in_seq_numbers are associated with in_accounts - // assert!(updates.iter().filter(|update| update.tx_hash != Hash::default()).count() <= 1); + for update in updates { // legacy merged.in_seq_numbers.extend(update.in_seq_numbers); diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 2853bb45..9e45d156 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -187,9 +187,6 @@ pub async fn persist_state_update( })) .collect(); - for leaf_node in &leaf_nodes_with_signatures { - debug!("Leaf node: {:?}, signature: {:?}", leaf_node.0, leaf_node.1); - } leaf_nodes_with_signatures.sort_by_key(|x| x.0.seq); debug!("Persisting state nodes..."); diff --git a/src/monitor/mod.rs b/src/monitor/mod.rs index 59de29e5..d8ca004c 100644 --- a/src/monitor/mod.rs +++ b/src/monitor/mod.rs @@ -77,7 +77,6 @@ pub fn continously_monitor_photon( } } else { let tree_roots = load_db_tree_roots_with_infinite_retry(db.as_ref()).await; - println!("Tree roots (continously_monitor_photon): {:?}", tree_roots); validate_tree_roots(rpc_client.as_ref(), tree_roots).await; } sleep(Duration::from_millis(5000)).await;