diff --git a/cli/src/utils/constants.ts b/cli/src/utils/constants.ts index 81bd82b1b3..d86f588678 100644 --- a/cli/src/utils/constants.ts +++ b/cli/src/utils/constants.ts @@ -19,12 +19,12 @@ export const SOLANA_VALIDATOR_PROCESS_NAME = "solana-test-validator"; export const LIGHT_PROVER_PROCESS_NAME = "light-prover"; export const INDEXER_PROCESS_NAME = "photon"; -export const PHOTON_VERSION = "0.51.0"; +export const PHOTON_VERSION = "0.51.2"; // Set these to override Photon requirements with a specific git commit: export const USE_PHOTON_FROM_GIT = true; // If true, will show git install command instead of crates.io. export const PHOTON_GIT_REPO = "https://github.com/lightprotocol/photon.git"; -export const PHOTON_GIT_COMMIT = "1a785036de52896b68d06413e3b0231122d6aa4a"; // If empty, will use main branch. +export const PHOTON_GIT_COMMIT = "711c47b20330c6bb78feb0a2c15e8292fcd0a7b0"; // If empty, will use main branch. export const LIGHT_PROTOCOL_PROGRAMS_DIR_ENV = "LIGHT_PROTOCOL_PROGRAMS_DIR"; export const BASE_PATH = "../../bin/"; diff --git a/forester-utils/src/instructions/address_batch_update.rs b/forester-utils/src/instructions/address_batch_update.rs index ca1bc8d18a..a70903280a 100644 --- a/forester-utils/src/instructions/address_batch_update.rs +++ b/forester-utils/src/instructions/address_batch_update.rs @@ -6,7 +6,10 @@ use futures::stream::Stream; use light_batched_merkle_tree::{ constants::DEFAULT_BATCH_ADDRESS_TREE_HEIGHT, merkle_tree::InstructionDataAddressAppendInputs, }; -use light_client::{indexer::Indexer, rpc::Rpc}; +use light_client::{ + indexer::{AddressQueueData, Indexer, QueueElementsV2Options}, + rpc::Rpc, +}; use light_compressed_account::{ hash_chain::create_hash_chain_from_slice, instruction_data::compressed_proof::CompressedProof, }; @@ -71,40 +74,49 @@ async fn stream_instruction_data<'a, R: Rpc>( } } - let indexer_update_info = { + let address_queue = { let mut connection = rpc_pool.get_connection().await?; let indexer = connection.indexer_mut()?; debug!( "Requesting {} addresses from Photon for chunk {} with start_queue_index={:?}", elements_for_chunk, chunk_idx, next_queue_index ); + let options = QueueElementsV2Options::default() + .with_address_queue(next_queue_index, Some(elements_for_chunk as u16)); match indexer - .get_address_queue_with_proofs( - &merkle_tree_pubkey, - elements_for_chunk as u16, - next_queue_index, - None, - ) - .await { - Ok(info) => info, + .get_queue_elements(merkle_tree_pubkey.to_bytes(), options, None) + .await + { + Ok(response) => match response.value.address_queue { + Some(queue) => queue, + None => { + yield Err(ForesterUtilsError::Indexer( + "No address queue data in response".into(), + )); + return; + } + }, Err(e) => { - yield Err(ForesterUtilsError::Indexer(format!("Failed to get address queue with proofs: {}", e))); + yield Err(ForesterUtilsError::Indexer(format!( + "Failed to get queue elements: {}", + e + ))); return; } } }; debug!( - "Photon response for chunk {}: received {} addresses, batch_start_index={}, first_queue_index={:?}, last_queue_index={:?}", + "Photon response for chunk {}: received {} addresses, start_index={}, first_queue_index={:?}, last_queue_index={:?}", chunk_idx, - indexer_update_info.value.addresses.len(), - indexer_update_info.value.batch_start_index, - indexer_update_info.value.addresses.first().map(|a| a.queue_index), - indexer_update_info.value.addresses.last().map(|a| a.queue_index) + address_queue.addresses.len(), + address_queue.start_index, + address_queue.queue_indices.first(), + address_queue.queue_indices.last() ); - if let Some(last_address) = indexer_update_info.value.addresses.last() { - next_queue_index = Some(last_address.queue_index + 1); + if let Some(last_queue_index) = address_queue.queue_indices.last() { + next_queue_index = Some(last_queue_index + 1); debug!( "Setting next_queue_index={} for chunk {}", next_queue_index.unwrap(), @@ -113,21 +125,24 @@ async fn stream_instruction_data<'a, R: Rpc>( } if chunk_idx == 0 { - if let Some(first_proof) = indexer_update_info.value.non_inclusion_proofs.first() { - if first_proof.root != current_root { - warn!("Indexer root does not match on-chain root"); - yield Err(ForesterUtilsError::Indexer("Indexer root does not match on-chain root".into())); - return; - } - } else { - yield Err(ForesterUtilsError::Indexer("No non-inclusion proofs found in indexer response".into())); + if address_queue.addresses.is_empty() { + yield Err(ForesterUtilsError::Indexer( + "No addresses found in indexer response".into(), + )); + return; + } + if address_queue.initial_root != current_root { + warn!("Indexer root does not match on-chain root"); + yield Err(ForesterUtilsError::Indexer( + "Indexer root does not match on-chain root".into(), + )); return; } } let (all_inputs, new_current_root) = match get_all_circuit_inputs_for_chunk( chunk_hash_chains, - &indexer_update_info, + &address_queue, zkp_batch_size, chunk_start, start_index, @@ -197,9 +212,7 @@ fn calculate_max_zkp_batches_per_call(batch_size: u16) -> usize { fn get_all_circuit_inputs_for_chunk( chunk_hash_chains: &[[u8; 32]], - indexer_update_info: &light_client::indexer::Response< - light_client::indexer::BatchAddressUpdateIndexerResponse, - >, + address_queue: &AddressQueueData, batch_size: u16, chunk_start_idx: usize, global_start_index: u64, @@ -212,14 +225,9 @@ fn get_all_circuit_inputs_for_chunk( ForesterUtilsError, > { let subtrees_array: [[u8; 32]; DEFAULT_BATCH_ADDRESS_TREE_HEIGHT as usize] = - indexer_update_info - .value - .subtrees - .clone() - .try_into() - .map_err(|_| { - ForesterUtilsError::Prover("Failed to convert subtrees to array".into()) - })?; + address_queue.subtrees.clone().try_into().map_err(|_| { + ForesterUtilsError::Prover("Failed to convert subtrees to array".into()) + })?; let mut sparse_merkle_tree = SparseMerkleTree::::new( @@ -235,7 +243,7 @@ fn get_all_circuit_inputs_for_chunk( let start_idx = batch_idx * batch_size as usize; let end_idx = start_idx + batch_size as usize; - let addresses_len = indexer_update_info.value.addresses.len(); + let addresses_len = address_queue.addresses.len(); if start_idx >= addresses_len { return Err(ForesterUtilsError::Indexer(format!( "Insufficient addresses: batch {} requires start_idx {} but only {} addresses available", @@ -250,41 +258,41 @@ fn get_all_circuit_inputs_for_chunk( ))); } - let batch_addresses: Vec<[u8; 32]> = indexer_update_info.value.addresses - [start_idx..safe_end_idx] - .iter() - .map(|x| x.address) - .collect(); + let batch_addresses: Vec<[u8; 32]> = + address_queue.addresses[start_idx..safe_end_idx].to_vec(); - let proofs_len = indexer_update_info.value.non_inclusion_proofs.len(); - if start_idx >= proofs_len { + // Check that we have enough low element data + let low_elements_len = address_queue.low_element_values.len(); + if start_idx >= low_elements_len { return Err(ForesterUtilsError::Indexer(format!( - "Insufficient non-inclusion proofs: batch {} requires start_idx {} but only {} proofs available", - batch_idx, start_idx, proofs_len + "Insufficient low element data: batch {} requires start_idx {} but only {} elements available", + batch_idx, start_idx, low_elements_len ))); } - let safe_proofs_end_idx = std::cmp::min(end_idx, proofs_len); - if safe_proofs_end_idx - start_idx != batch_size as usize { + let safe_low_end_idx = std::cmp::min(end_idx, low_elements_len); + if safe_low_end_idx - start_idx != batch_size as usize { return Err(ForesterUtilsError::Indexer(format!( - "Insufficient non-inclusion proofs: batch {} requires {} proofs (indices {}..{}) but only {} available", - batch_idx, batch_size, start_idx, end_idx, safe_proofs_end_idx - start_idx + "Insufficient low element data: batch {} requires {} elements (indices {}..{}) but only {} available", + batch_idx, batch_size, start_idx, end_idx, safe_low_end_idx - start_idx ))); } - let mut low_element_values = Vec::new(); - let mut low_element_next_values = Vec::new(); - let mut low_element_indices = Vec::new(); - let mut low_element_next_indices = Vec::new(); - let mut low_element_proofs = Vec::new(); - - for proof in &indexer_update_info.value.non_inclusion_proofs[start_idx..safe_proofs_end_idx] - { - low_element_values.push(proof.low_address_value); - low_element_indices.push(proof.low_address_index as usize); - low_element_next_indices.push(proof.low_address_next_index as usize); - low_element_next_values.push(proof.low_address_next_value); - low_element_proofs.push(proof.low_address_proof.to_vec()); - } + let low_element_values: Vec<[u8; 32]> = + address_queue.low_element_values[start_idx..safe_low_end_idx].to_vec(); + let low_element_next_values: Vec<[u8; 32]> = + address_queue.low_element_next_values[start_idx..safe_low_end_idx].to_vec(); + let low_element_indices: Vec = address_queue.low_element_indices + [start_idx..safe_low_end_idx] + .iter() + .map(|&x| x as usize) + .collect(); + let low_element_next_indices: Vec = address_queue.low_element_next_indices + [start_idx..safe_low_end_idx] + .iter() + .map(|&x| x as usize) + .collect(); + let low_element_proofs: Vec> = + address_queue.low_element_proofs[start_idx..safe_low_end_idx].to_vec(); let computed_hash_chain = create_hash_chain_from_slice(&batch_addresses)?; if computed_hash_chain != *leaves_hash_chain { diff --git a/forester/src/processor/v2/state/helpers.rs b/forester/src/processor/v2/state/helpers.rs index 26a911cff2..49ee2a07a1 100644 --- a/forester/src/processor/v2/state/helpers.rs +++ b/forester/src/processor/v2/state/helpers.rs @@ -37,7 +37,7 @@ pub async fn fetch_batches( input_start_index: Option, fetch_len: u64, zkp_batch_size: u64, -) -> crate::Result> { +) -> crate::Result> { let fetch_len_u16: u16 = match fetch_len.try_into() { Ok(v) => v, Err(_) => { @@ -70,7 +70,7 @@ pub async fn fetch_batches( .with_input_queue_batch_size(Some(zkp_batch_size_u16)); let res = indexer - .get_queue_elements_v2(context.merkle_tree.to_bytes(), options, None) + .get_queue_elements(context.merkle_tree.to_bytes(), options, None) .await?; Ok(res.value.state_queue) diff --git a/forester/src/processor/v2/state/supervisor.rs b/forester/src/processor/v2/state/supervisor.rs index 5b7c6f38a6..a7d7d19eb3 100644 --- a/forester/src/processor/v2/state/supervisor.rs +++ b/forester/src/processor/v2/state/supervisor.rs @@ -431,7 +431,7 @@ impl StateSupervisor { async fn build_append_job( &mut self, batch_idx: usize, - state_queue: &light_client::indexer::StateQueueDataV2, + state_queue: &light_client::indexer::StateQueueData, start: usize, result_tx: mpsc::Sender, ) -> crate::Result> { @@ -483,7 +483,7 @@ impl StateSupervisor { async fn build_nullify_job( &mut self, batch_idx: usize, - state_queue: &light_client::indexer::StateQueueDataV2, + state_queue: &light_client::indexer::StateQueueData, start: usize, result_tx: mpsc::Sender, ) -> crate::Result> { diff --git a/forester/tests/legacy/batched_address_test.rs b/forester/tests/legacy/batched_address_test.rs index 7b6db499d7..4e104c2a61 100644 --- a/forester/tests/legacy/batched_address_test.rs +++ b/forester/tests/legacy/batched_address_test.rs @@ -7,7 +7,9 @@ use light_batched_merkle_tree::{ merkle_tree::BatchedMerkleTreeAccount, }; use light_client::{ - indexer::{photon_indexer::PhotonIndexer, AddressMerkleTreeAccounts, Indexer}, + indexer::{ + photon_indexer::PhotonIndexer, AddressMerkleTreeAccounts, Indexer, QueueElementsV2Options, + }, local_test_validator::LightValidatorConfig, rpc::{client::RpcUrl, LightClient, LightClientConfig, Rpc}, }; @@ -167,24 +169,25 @@ async fn test_address_batched() { ) .unwrap(); - let photon_address_queue_with_proofs = photon_indexer - .get_address_queue_with_proofs(&address_merkle_tree_pubkey, 10, None, None) + let options = QueueElementsV2Options::default().with_address_queue(None, Some(10)); + let photon_address_queue = photon_indexer + .get_queue_elements(address_merkle_tree_pubkey.to_bytes(), options.clone(), None) .await .unwrap(); - let test_indexer_address_queue_with_proofs = env + let test_indexer_address_queue = env .indexer - .get_address_queue_with_proofs(&address_merkle_tree_pubkey, 10, None, None) + .get_queue_elements(address_merkle_tree_pubkey.to_bytes(), options.clone(), None) .await .unwrap(); println!( "photon_indexer_update_info {}: {:#?}", - 0, photon_address_queue_with_proofs + 0, photon_address_queue ); println!( "test_indexer_update_info {}: {:#?}", - 0, test_indexer_address_queue_with_proofs + 0, test_indexer_address_queue ); for i in 0..merkle_tree.queue_batches.batch_size { @@ -202,26 +205,26 @@ async fn test_address_batched() { sleep(Duration::from_millis(100)).await; if (i + 1) % 10 == 0 { - let photon_address_queue_with_proofs = photon_indexer - .get_address_queue_with_proofs(&address_merkle_tree_pubkey, 10, None, None) + let photon_address_queue = photon_indexer + .get_queue_elements(address_merkle_tree_pubkey.to_bytes(), options.clone(), None) .await .unwrap(); - let test_indexer_address_queue_with_proofs = env + let test_indexer_address_queue = env .indexer - .get_address_queue_with_proofs(&address_merkle_tree_pubkey, 10, None, None) + .get_queue_elements(address_merkle_tree_pubkey.to_bytes(), options.clone(), None) .await .unwrap(); println!( "photon_indexer_update_info {}: {:#?}", i + 1, - photon_address_queue_with_proofs + photon_address_queue ); println!( "test_indexer_update_info {}: {:#?}", i + 1, - test_indexer_address_queue_with_proofs + test_indexer_address_queue ); } } diff --git a/program-tests/utils/src/e2e_test_env.rs b/program-tests/utils/src/e2e_test_env.rs index 1b9837bf42..481f103914 100644 --- a/program-tests/utils/src/e2e_test_env.rs +++ b/program-tests/utils/src/e2e_test_env.rs @@ -86,7 +86,10 @@ use light_batched_merkle_tree::{ }; use light_client::{ fee::{FeeConfig, TransactionParams}, - indexer::{AddressMerkleTreeAccounts, AddressWithTree, Indexer, StateMerkleTreeAccounts}, + indexer::{ + AddressMerkleTreeAccounts, AddressWithTree, Indexer, QueueElementsV2Options, + StateMerkleTreeAccounts, + }, rpc::{errors::RpcError, merkle_tree::MerkleTreeExt, Rpc}, }; // TODO: implement traits for context object and indexer that we can implement with an rpc as well @@ -741,20 +744,18 @@ where [full_batch_index as usize] [zkp_batch_index as usize]; - let addresses = self + let options = QueueElementsV2Options::default() + .with_address_queue(None, Some(batch.batch_size as u16)); + let result = self .indexer - .get_queue_elements( - merkle_tree_pubkey.to_bytes(), - None, - Some(batch.batch_size as u16), - None, - None, - None, - ) + .get_queue_elements(merkle_tree_pubkey.to_bytes(), options, None) .await .unwrap(); - let addresses = - addresses.value.output_queue_elements.unwrap_or_default().iter().map(|x| x.account_hash).collect::>(); + let addresses = result + .value + .address_queue + .map(|aq| aq.addresses) + .unwrap_or_default(); // // local_leaves_hash_chain is only used for a test assertion. // let local_nullifier_hash_chain = create_hash_chain_from_array(&addresses); // assert_eq!(leaves_hash_chain, local_nullifier_hash_chain); diff --git a/program-tests/utils/src/test_batch_forester.rs b/program-tests/utils/src/test_batch_forester.rs index 0a9fc2ec2b..0fcc2441b6 100644 --- a/program-tests/utils/src/test_batch_forester.rs +++ b/program-tests/utils/src/test_batch_forester.rs @@ -321,7 +321,7 @@ pub async fn get_batched_nullify_ix_data( use forester_utils::{ account_zero_copy::AccountZeroCopy, instructions::create_account::create_account_instruction, }; -use light_client::indexer::Indexer; +use light_client::indexer::{Indexer, QueueElementsV2Options}; use light_program_test::indexer::state_tree::StateMerkleTreeBundle; use light_sparse_merkle_tree::SparseMerkleTree; @@ -650,24 +650,17 @@ pub async fn create_batch_update_address_tree_instruction_data_with_proof>(); + .address_queue + .map(|aq| aq.addresses) + .unwrap_or_default(); // // local_leaves_hash_chain is only used for a test assertion. // let local_nullifier_hash_chain = create_hash_chain_from_slice(addresses.as_slice()).unwrap(); // assert_eq!(leaves_hash_chain, local_nullifier_hash_chain); diff --git a/scripts/devenv/versions.sh b/scripts/devenv/versions.sh index 62deb0c745..14a1c310f2 100755 --- a/scripts/devenv/versions.sh +++ b/scripts/devenv/versions.sh @@ -12,8 +12,8 @@ export NODE_VERSION="22.16.0" export SOLANA_VERSION="2.2.15" export ANCHOR_VERSION="0.31.1" export JQ_VERSION="1.8.0" -export PHOTON_VERSION="0.51.0" -export PHOTON_COMMIT="5e5b52a14323997d4433f687ea77f1f480e124ad" +export PHOTON_VERSION="0.51.2" +export PHOTON_COMMIT="711c47b20330c6bb78feb0a2c15e8292fcd0a7b0" export REDIS_VERSION="8.0.1" export ANCHOR_TAG="anchor-v${ANCHOR_VERSION}" diff --git a/sdk-libs/client/src/indexer/indexer_trait.rs b/sdk-libs/client/src/indexer/indexer_trait.rs index 9d37b064bf..51066c07b6 100644 --- a/sdk-libs/client/src/indexer/indexer_trait.rs +++ b/sdk-libs/client/src/indexer/indexer_trait.rs @@ -7,10 +7,9 @@ use super::{ CompressedAccount, CompressedTokenAccount, OwnerBalance, QueueElementsResult, QueueInfoResult, SignatureWithMetadata, TokenBalance, ValidityProofWithContext, }, - Address, AddressWithTree, BatchAddressUpdateIndexerResponse, - GetCompressedAccountsByOwnerConfig, GetCompressedTokenAccountsByOwnerOrDelegateOptions, Hash, - IndexerError, IndexerRpcConfig, MerkleProof, NewAddressProofWithContext, PaginatedOptions, - RetryConfig, + Address, AddressWithTree, GetCompressedAccountsByOwnerConfig, + GetCompressedTokenAccountsByOwnerOrDelegateOptions, Hash, IndexerError, IndexerRpcConfig, + MerkleProof, NewAddressProofWithContext, PaginatedOptions, QueueElementsV2Options, RetryConfig, }; // TODO: remove all references in input types. #[async_trait] @@ -172,32 +171,16 @@ pub trait Indexer: std::marker::Send + std::marker::Sync { config: Option, ) -> Result, IndexerError>; - // TODO: in different pr: - // replace zkp_batch_size with PaginatedOptions - // - return type should be ItemsWithCursor - async fn get_address_queue_with_proofs( - &mut self, - merkle_tree_pubkey: &Pubkey, - zkp_batch_size: u16, - start_offset: Option, - config: Option, - ) -> Result, IndexerError>; - - // TODO: in different pr: - // replace num_elements & start_queue_index with PaginatedOptions - // - return type should be ItemsWithCursor /// Returns queue elements from the queue with the given merkle tree pubkey. - /// Can fetch from output queue (append), input queue (nullify), or both atomically. + /// Can fetch from output queue (append), input queue (nullify), address queue, or combinations. /// For input queues account compression program does not store queue elements in the /// account data but only emits these in the public transaction event. The /// indexer needs the queue elements to create batch update proofs. + /// Returns queue elements with deduplicated nodes for efficient staging tree construction. async fn get_queue_elements( &mut self, merkle_tree_pubkey: [u8; 32], - output_queue_start_index: Option, - output_queue_limit: Option, - input_queue_start_index: Option, - input_queue_limit: Option, + options: QueueElementsV2Options, config: Option, ) -> Result, IndexerError>; @@ -208,14 +191,6 @@ pub trait Indexer: std::marker::Send + std::marker::Sync { config: Option, ) -> Result, IndexerError>; - /// V2: Returns queue elements with deduplicated nodes for efficient staging tree construction. - /// Supports output queue, input queue, and address queue. - async fn get_queue_elements_v2( - &mut self, - merkle_tree_pubkey: [u8; 32], - options: super::QueueElementsV2Options, - config: Option, - ) -> Result, IndexerError>; async fn get_subtrees( &self, merkle_tree_pubkey: [u8; 32], diff --git a/sdk-libs/client/src/indexer/mod.rs b/sdk-libs/client/src/indexer/mod.rs index 6b86a41559..fa03606dfe 100644 --- a/sdk-libs/client/src/indexer/mod.rs +++ b/sdk-libs/client/src/indexer/mod.rs @@ -14,12 +14,11 @@ pub use error::IndexerError; pub use indexer_trait::Indexer; pub use response::{Context, Items, ItemsWithCursor, Response}; pub use types::{ - AccountProofInputs, Address, AddressMerkleTreeAccounts, AddressProofInputs, AddressQueueDataV2, - AddressQueueIndex, AddressWithTree, BatchAddressUpdateIndexerResponse, CompressedAccount, - CompressedTokenAccount, Hash, InputQueueDataV2, MerkleProof, MerkleProofWithContext, - NewAddressProofWithContext, NextTreeInfo, OutputQueueDataV2, OwnerBalance, ProofOfLeaf, - QueueElementsResult, QueueElementsV2Result, QueueInfo, QueueInfoResult, RootIndex, - SignatureWithMetadata, StateMerkleTreeAccounts, StateQueueDataV2, TokenBalance, TreeInfo, + AccountProofInputs, Address, AddressMerkleTreeAccounts, AddressProofInputs, AddressQueueData, + AddressWithTree, CompressedAccount, CompressedTokenAccount, Hash, InputQueueData, MerkleProof, + MerkleProofWithContext, NewAddressProofWithContext, NextTreeInfo, OutputQueueData, + OwnerBalance, ProofOfLeaf, QueueElementsResult, QueueInfo, QueueInfoResult, RootIndex, + SignatureWithMetadata, StateMerkleTreeAccounts, StateQueueData, TokenBalance, TreeInfo, ValidityProofWithContext, }; mod options; diff --git a/sdk-libs/client/src/indexer/photon_indexer.rs b/sdk-libs/client/src/indexer/photon_indexer.rs index d98fff212d..a75a14b40c 100644 --- a/sdk-libs/client/src/indexer/photon_indexer.rs +++ b/sdk-libs/client/src/indexer/photon_indexer.rs @@ -9,12 +9,8 @@ use photon_api::{ use solana_pubkey::Pubkey; use tracing::{error, trace, warn}; -use super::{ - types::{ - CompressedAccount, CompressedTokenAccount, OwnerBalance, QueueElementsResult, - SignatureWithMetadata, TokenBalance, - }, - BatchAddressUpdateIndexerResponse, +use super::types::{ + CompressedAccount, CompressedTokenAccount, OwnerBalance, SignatureWithMetadata, TokenBalance, }; use crate::indexer::{ base58::Base58Conversions, @@ -1469,274 +1465,6 @@ impl Indexer for PhotonIndexer { .await } - async fn get_address_queue_with_proofs( - &mut self, - _merkle_tree_pubkey: &Pubkey, - _zkp_batch_size: u16, - _start_offset: Option, - _config: Option, - ) -> Result, IndexerError> { - #[cfg(not(feature = "v2"))] - unimplemented!("get_address_queue_with_proofs"); - #[cfg(feature = "v2")] - { - let merkle_tree_pubkey = _merkle_tree_pubkey; - let limit = _zkp_batch_size; - let start_queue_index = _start_offset; - let config = _config.unwrap_or_default(); - self.retry(config.retry_config, || async { - let merkle_tree = Hash::from_bytes(merkle_tree_pubkey.to_bytes().as_ref())?; - let request = photon_api::models::GetBatchAddressUpdateInfoPostRequest { - params: Box::new( - photon_api::models::GetBatchAddressUpdateInfoPostRequestParams { - limit, - start_queue_index, - tree: merkle_tree.to_base58(), - }, - ), - ..Default::default() - }; - - let result = photon_api::apis::default_api::get_batch_address_update_info_post( - &self.configuration, - request, - ) - .await?; - - let api_response = Self::extract_result_with_error_check( - "get_batch_address_update_info", - result.error, - result.result.map(|r| *r), - )?; - if api_response.context.slot < config.slot { - return Err(IndexerError::IndexerNotSyncedToSlot); - } - - let addresses = api_response - .addresses - .iter() - .map(|x| crate::indexer::AddressQueueIndex { - address: Hash::from_base58(x.address.clone().as_ref()).unwrap(), - queue_index: x.queue_index, - }) - .collect(); - - let mut proofs: Vec = vec![]; - for proof in api_response.non_inclusion_proofs { - let proof = NewAddressProofWithContext { - merkle_tree: *merkle_tree_pubkey, - low_address_index: proof.low_element_leaf_index, - low_address_value: Hash::from_base58( - proof.lower_range_address.clone().as_ref(), - ) - .unwrap(), - low_address_next_index: proof.next_index, - low_address_next_value: Hash::from_base58( - proof.higher_range_address.clone().as_ref(), - ) - .unwrap(), - low_address_proof: proof - .proof - .iter() - .map(|x| Hash::from_base58(x.clone().as_ref()).unwrap()) - .collect(), - root: Hash::from_base58(proof.root.clone().as_ref()).unwrap(), - root_seq: proof.root_seq, - - new_low_element: None, - new_element: None, - new_element_next_value: None, - }; - proofs.push(proof); - } - - let subtrees = api_response - .subtrees - .iter() - .map(|x| { - let mut arr = [0u8; 32]; - arr.copy_from_slice(x.as_slice()); - arr - }) - .collect::>(); - - let result = BatchAddressUpdateIndexerResponse { - batch_start_index: api_response.start_index, - addresses, - non_inclusion_proofs: proofs, - subtrees, - }; - Ok(Response { - context: Context { - slot: api_response.context.slot, - }, - value: result, - }) - }) - .await - } - } - - async fn get_queue_elements( - &mut self, - _pubkey: [u8; 32], - _output_queue_start_index: Option, - _output_queue_limit: Option, - _input_queue_start_index: Option, - _input_queue_limit: Option, - _config: Option, - ) -> Result, IndexerError> { - #[cfg(not(feature = "v2"))] - unimplemented!("get_queue_elements"); - #[cfg(feature = "v2")] - { - use super::MerkleProofWithContext; - let pubkey = _pubkey; - let output_queue_start_index = _output_queue_start_index; - let output_queue_limit = _output_queue_limit; - let input_queue_start_index = _input_queue_start_index; - let input_queue_limit = _input_queue_limit; - let config = _config.unwrap_or_default(); - self.retry(config.retry_config, || async { - let request: photon_api::models::GetQueueElementsPostRequest = - photon_api::models::GetQueueElementsPostRequest { - params: Box::from(photon_api::models::GetQueueElementsPostRequestParams { - tree: bs58::encode(pubkey).into_string(), - output_queue_start_index, - output_queue_limit, - input_queue_start_index, - input_queue_limit, - }), - ..Default::default() - }; - - let result = photon_api::apis::default_api::get_queue_elements_post( - &self.configuration, - request, - ) - .await; - let result: Result, IndexerError> = match result { - Ok(api_response) => match api_response.result { - Some(api_result) => { - if api_result.context.slot < config.slot { - return Err(IndexerError::IndexerNotSyncedToSlot); - } - - // Parse output queue elements - let output_queue_elements = api_result - .output_queue_elements - .map(|elements| { - elements - .iter() - .map(|x| -> Result<_, IndexerError> { - let proof: Vec = x - .proof - .iter() - .map(|p| Hash::from_base58(p)) - .collect::, _>>()?; - let root = Hash::from_base58(&x.root)?; - let leaf = Hash::from_base58(&x.leaf)?; - let merkle_tree = Hash::from_base58(&x.tree)?; - let tx_hash = x - .tx_hash - .as_ref() - .map(|h| Hash::from_base58(h)) - .transpose()?; - let account_hash = Hash::from_base58(&x.account_hash)?; - - Ok(MerkleProofWithContext { - proof, - root, - leaf_index: x.leaf_index, - leaf, - merkle_tree, - root_seq: x.root_seq, - tx_hash, - account_hash, - }) - }) - .collect::, _>>() - }) - .transpose()?; - - // Parse input queue elements - let input_queue_elements = api_result - .input_queue_elements - .map(|elements| { - elements - .iter() - .map(|x| -> Result<_, IndexerError> { - let proof: Vec = x - .proof - .iter() - .map(|p| Hash::from_base58(p)) - .collect::, _>>()?; - let root = Hash::from_base58(&x.root)?; - let leaf = Hash::from_base58(&x.leaf)?; - let merkle_tree = Hash::from_base58(&x.tree)?; - let tx_hash = x - .tx_hash - .as_ref() - .map(|h| Hash::from_base58(h)) - .transpose()?; - let account_hash = Hash::from_base58(&x.account_hash)?; - - Ok(MerkleProofWithContext { - proof, - root, - leaf_index: x.leaf_index, - leaf, - merkle_tree, - root_seq: x.root_seq, - tx_hash, - account_hash, - }) - }) - .collect::, _>>() - }) - .transpose()?; - - Ok(Response { - context: Context { - slot: api_result.context.slot, - }, - value: QueueElementsResult { - output_queue_elements, - output_queue_index: api_result.output_queue_index, - input_queue_elements, - input_queue_index: api_result.input_queue_index, - }, - }) - } - None => { - let error = - api_response - .error - .ok_or_else(|| IndexerError::PhotonError { - context: "get_queue_elements".to_string(), - message: "No error details provided".to_string(), - })?; - - Err(IndexerError::PhotonError { - context: "get_queue_elements".to_string(), - message: error - .message - .unwrap_or_else(|| "Unknown error".to_string()), - }) - } - }, - Err(e) => Err(IndexerError::PhotonError { - context: "get_queue_elements".to_string(), - message: e.to_string(), - }), - }; - - result - }) - .await - } - } - async fn get_queue_info( &self, config: Option, @@ -1797,197 +1525,245 @@ impl Indexer for PhotonIndexer { .await } - async fn get_queue_elements_v2( + async fn get_queue_elements( &mut self, - merkle_tree_pubkey: [u8; 32], - options: super::QueueElementsV2Options, - config: Option, - ) -> Result, IndexerError> { - let config = config.unwrap_or_default(); - self.retry(config.retry_config, || async { - let params = photon_api::models::GetQueueElementsV2PostRequestParams { - tree: bs58::encode(merkle_tree_pubkey).into_string(), - output_queue_start_index: options.output_queue_start_index, - output_queue_limit: options.output_queue_limit, - output_queue_zkp_batch_size: options.output_queue_zkp_batch_size, - input_queue_start_index: options.input_queue_start_index, - input_queue_limit: options.input_queue_limit, - input_queue_zkp_batch_size: options.input_queue_zkp_batch_size, - address_queue_start_index: options.address_queue_start_index, - address_queue_limit: options.address_queue_limit, - address_queue_zkp_batch_size: options.address_queue_zkp_batch_size, - }; + _merkle_tree_pubkey: [u8; 32], + _options: super::QueueElementsV2Options, + _config: Option, + ) -> Result, IndexerError> { + #[cfg(not(feature = "v2"))] + unimplemented!(); - let request = photon_api::models::GetQueueElementsV2PostRequest { - params: Box::new(params), - ..Default::default() - }; + #[cfg(feature = "v2")] + { + use crate::indexer::OutputQueueData; + let merkle_tree_pubkey = _merkle_tree_pubkey; + let options = _options; + let config = _config.unwrap_or_default(); + self.retry(config.retry_config, || async { + // Build nested QueueRequest objects for the new API format + let output_queue = options.output_queue_limit.map(|limit| { + let mut req = photon_api::models::QueueRequest::new(limit); + req.start_index = options.output_queue_start_index; + req.zkp_batch_size = options.output_queue_zkp_batch_size; + req + }); - let result = photon_api::apis::default_api::get_queue_elements_v2_post( - &self.configuration, - request, - ) - .await?; + let input_queue = options.input_queue_limit.map(|limit| { + let mut req = photon_api::models::QueueRequest::new(limit); + req.start_index = options.input_queue_start_index; + req.zkp_batch_size = options.input_queue_zkp_batch_size; + req + }); - let api_response = Self::extract_result_with_error_check( - "get_queue_elements_v2", - result.error, - result.result.map(|r| *r), - )?; + let address_queue = options.address_queue_limit.map(|limit| { + let mut req = photon_api::models::QueueRequest::new(limit); + req.start_index = options.address_queue_start_index; + req.zkp_batch_size = options.address_queue_zkp_batch_size; + req + }); - if api_response.context.slot < config.slot { - return Err(IndexerError::IndexerNotSyncedToSlot); - } + let mut params = photon_api::models::GetQueueElementsPostRequestParams::new( + bs58::encode(merkle_tree_pubkey).into_string(), + ); + params.output_queue = output_queue; + params.input_queue = input_queue; + params.address_queue = address_queue; - let state_queue = if let Some(state) = api_response.state_queue { - let node_hashes: Result, IndexerError> = state - .node_hashes - .iter() - .map(|h| Hash::from_base58(h)) - .collect(); - let initial_root = Hash::from_base58(&state.initial_root)?; + let request = photon_api::models::GetQueueElementsPostRequest { + params: Box::new(params), + ..Default::default() + }; - let output_queue = if let Some(output) = state.output_queue { - let account_hashes: Result, IndexerError> = output - .account_hashes - .iter() - .map(|h| Hash::from_base58(h)) - .collect(); - let old_leaves: Result, IndexerError> = - output.leaves.iter().map(|h| Hash::from_base58(h)).collect(); - let leaves_hash_chains: Result, IndexerError> = output - .leaves_hash_chains + let result = photon_api::apis::default_api::get_queue_elements_post( + &self.configuration, + request, + ) + .await?; + + let api_response = Self::extract_result_with_error_check( + "get_queue_elements", + result.error, + result.result.map(|r| *r), + )?; + + if api_response.context.slot < config.slot { + return Err(IndexerError::IndexerNotSyncedToSlot); + } + + let state_queue = if let Some(state) = api_response.state_queue { + // Extract nodes and node_hashes from combined Node objects + let nodes: Vec = state.nodes.iter().map(|n| n.index).collect(); + let node_hashes: Result, IndexerError> = state + .nodes .iter() - .map(|h| Hash::from_base58(h)) + .map(|n| Hash::from_base58(&n.hash)) .collect(); + let initial_root = Hash::from_base58(&state.initial_root)?; - Some(super::OutputQueueDataV2 { - leaf_indices: output.leaf_indices, - account_hashes: account_hashes?, - old_leaves: old_leaves?, - first_queue_index: output.first_queue_index, - next_index: output.next_index, - leaves_hash_chains: leaves_hash_chains?, + let output_queue = if let Some(output) = state.output_queue { + let account_hashes: Result, IndexerError> = output + .account_hashes + .iter() + .map(|h| Hash::from_base58(h)) + .collect(); + let old_leaves: Result, IndexerError> = + output.leaves.iter().map(|h| Hash::from_base58(h)).collect(); + let leaves_hash_chains: Result, IndexerError> = output + .leaves_hash_chains + .iter() + .map(|h| Hash::from_base58(h)) + .collect(); + + Some(OutputQueueData { + leaf_indices: output.leaf_indices, + account_hashes: account_hashes?, + old_leaves: old_leaves?, + first_queue_index: output.first_queue_index, + next_index: output.next_index, + leaves_hash_chains: leaves_hash_chains?, + }) + } else { + None + }; + + let input_queue = if let Some(input) = state.input_queue { + let account_hashes: Result, IndexerError> = input + .account_hashes + .iter() + .map(|h| Hash::from_base58(h)) + .collect(); + let current_leaves: Result, IndexerError> = + input.leaves.iter().map(|h| Hash::from_base58(h)).collect(); + let tx_hashes: Result, IndexerError> = input + .tx_hashes + .iter() + .map(|h| Hash::from_base58(h)) + .collect(); + let nullifiers: Result, IndexerError> = input + .nullifiers + .iter() + .map(|h| Hash::from_base58(h)) + .collect(); + let leaves_hash_chains: Result, IndexerError> = input + .leaves_hash_chains + .iter() + .map(|h| Hash::from_base58(h)) + .collect(); + + Some(super::InputQueueData { + leaf_indices: input.leaf_indices, + account_hashes: account_hashes?, + current_leaves: current_leaves?, + tx_hashes: tx_hashes?, + nullifiers: nullifiers?, + first_queue_index: input.first_queue_index, + leaves_hash_chains: leaves_hash_chains?, + }) + } else { + None + }; + + Some(super::StateQueueData { + nodes, + node_hashes: node_hashes?, + initial_root, + root_seq: state.root_seq, + output_queue, + input_queue, }) } else { None }; - let input_queue = if let Some(input) = state.input_queue { - let account_hashes: Result, IndexerError> = input - .account_hashes + // Transform AddressQueueDataV2 + let address_queue = if let Some(address) = api_response.address_queue { + let addresses: Result, IndexerError> = address + .addresses .iter() .map(|h| Hash::from_base58(h)) .collect(); - let current_leaves: Result, IndexerError> = - input.leaves.iter().map(|h| Hash::from_base58(h)).collect(); - let tx_hashes: Result, IndexerError> = input - .tx_hashes + + let low_element_values: Result, IndexerError> = address + .low_element_values .iter() .map(|h| Hash::from_base58(h)) .collect(); - let nullifiers: Result, IndexerError> = input - .nullifiers + + let low_element_next_values: Result, IndexerError> = address + .low_element_next_values .iter() .map(|h| Hash::from_base58(h)) .collect(); - let leaves_hash_chains: Result, IndexerError> = input + + // Extract nodes and node_hashes from combined Node objects + // Proofs are reconstructed from nodes using AddressQueueDataV2::reconstruct_proof() + let nodes: Vec = address.nodes.iter().map(|n| n.index).collect(); + let node_hashes: Result, IndexerError> = address + .nodes + .iter() + .map(|n| Hash::from_base58(&n.hash)) + .collect(); + + let initial_root = Hash::from_base58(&address.initial_root)?; + + let leaves_hash_chains: Result, IndexerError> = address .leaves_hash_chains .iter() .map(|h| Hash::from_base58(h)) .collect(); - Some(super::InputQueueDataV2 { - leaf_indices: input.leaf_indices, - account_hashes: account_hashes?, - current_leaves: current_leaves?, - tx_hashes: tx_hashes?, - nullifiers: nullifiers?, - first_queue_index: input.first_queue_index, + let subtrees: Result, IndexerError> = address + .subtrees + .iter() + .map(|h| Hash::from_base58(h)) + .collect(); + + // Parse low_element_proofs for debugging/validation + let low_element_proofs: Result>, IndexerError> = address + .low_element_proofs + .iter() + .map(|proof| { + proof.iter().map(|h| Hash::from_base58(h)).collect::, + IndexerError, + >>( + ) + }) + .collect(); + + Some(super::AddressQueueData { + addresses: addresses?, + queue_indices: address.queue_indices, + low_element_values: low_element_values?, + low_element_next_values: low_element_next_values?, + low_element_indices: address.low_element_indices, + low_element_next_indices: address.low_element_next_indices, + nodes, + node_hashes: node_hashes?, + initial_root, leaves_hash_chains: leaves_hash_chains?, + subtrees: subtrees?, + start_index: address.start_index, + root_seq: address.root_seq, + low_element_proofs: low_element_proofs?, }) } else { None }; - Some(super::StateQueueDataV2 { - nodes: state.nodes, - node_hashes: node_hashes?, - initial_root, - root_seq: state.root_seq, - output_queue, - input_queue, - }) - } else { - None - }; - - // Transform AddressQueueDataV2 - let address_queue = if let Some(address) = api_response.address_queue { - let addresses: Result, IndexerError> = address - .addresses - .iter() - .map(|h| Hash::from_base58(h)) - .collect(); - - let low_element_values: Result, IndexerError> = address - .low_element_values - .iter() - .map(|h| Hash::from_base58(h)) - .collect(); - - let low_element_next_values: Result, IndexerError> = address - .low_element_next_values - .iter() - .map(|h| Hash::from_base58(h)) - .collect(); - - let low_element_proofs: Result>, IndexerError> = address - .low_element_proofs - .iter() - .map(|proof_vec| { - proof_vec - .iter() - .map(|h| Hash::from_base58(h)) - .collect::, IndexerError>>() - }) - .collect(); - - let node_hashes: Result, IndexerError> = address - .node_hashes - .iter() - .map(|h| Hash::from_base58(h)) - .collect(); - - let initial_root = Hash::from_base58(&address.initial_root)?; - - Some(super::AddressQueueDataV2 { - addresses: addresses?, - low_element_values: low_element_values?, - low_element_next_values: low_element_next_values?, - low_element_indices: address.low_element_indices, - low_element_next_indices: address.low_element_next_indices, - low_element_proofs: low_element_proofs?, - nodes: address.nodes, - node_hashes: node_hashes?, - initial_root, - first_queue_index: address.start_index, + Ok(Response { + context: Context { + slot: api_response.context.slot, + }, + value: super::QueueElementsResult { + state_queue, + address_queue, + }, }) - } else { - None - }; - - Ok(Response { - context: Context { - slot: api_response.context.slot, - }, - value: super::QueueElementsV2Result { - state_queue, - address_queue, - }, }) - }) - .await + .await + } } async fn get_subtrees( diff --git a/sdk-libs/client/src/indexer/types.rs b/sdk-libs/client/src/indexer/types.rs index 1ff7dd1e2d..877e99ac16 100644 --- a/sdk-libs/client/src/indexer/types.rs +++ b/sdk-libs/client/src/indexer/types.rs @@ -44,16 +44,7 @@ pub struct QueueInfoResult { } #[derive(Debug, Clone, PartialEq, Default)] -pub struct QueueElementsResult { - pub output_queue_elements: Option>, - pub output_queue_index: Option, - pub input_queue_elements: Option>, - pub input_queue_index: Option, -} - -/// V2 Output Queue Data -#[derive(Debug, Clone, PartialEq, Default)] -pub struct OutputQueueDataV2 { +pub struct OutputQueueData { pub leaf_indices: Vec, pub account_hashes: Vec<[u8; 32]>, pub old_leaves: Vec<[u8; 32]>, @@ -66,7 +57,7 @@ pub struct OutputQueueDataV2 { /// V2 Input Queue Data #[derive(Debug, Clone, PartialEq, Default)] -pub struct InputQueueDataV2 { +pub struct InputQueueData { pub leaf_indices: Vec, pub account_hashes: Vec<[u8; 32]>, pub current_leaves: Vec<[u8; 32]>, @@ -80,7 +71,7 @@ pub struct InputQueueDataV2 { /// State queue data with shared tree nodes for output and input queues #[derive(Debug, Clone, PartialEq, Default)] -pub struct StateQueueDataV2 { +pub struct StateQueueData { /// Shared deduplicated tree nodes for state queues (output + input) /// node_index encoding: (level << 56) | position pub nodes: Vec, @@ -90,31 +81,93 @@ pub struct StateQueueDataV2 { /// Sequence number of the root pub root_seq: u64, /// Output queue data (if requested) - pub output_queue: Option, + pub output_queue: Option, /// Input queue data (if requested) - pub input_queue: Option, + pub input_queue: Option, } /// V2 Address Queue Data with deduplicated nodes +/// Proofs are reconstructed from `nodes`/`node_hashes` using `low_element_indices` #[derive(Debug, Clone, PartialEq, Default)] -pub struct AddressQueueDataV2 { +pub struct AddressQueueData { pub addresses: Vec<[u8; 32]>, + pub queue_indices: Vec, pub low_element_values: Vec<[u8; 32]>, pub low_element_next_values: Vec<[u8; 32]>, pub low_element_indices: Vec, pub low_element_next_indices: Vec, - pub low_element_proofs: Vec>, + /// Deduplicated node indices - encoding: (level << 56) | position pub nodes: Vec, + /// Hashes corresponding to each node index pub node_hashes: Vec<[u8; 32]>, pub initial_root: [u8; 32], - pub first_queue_index: u64, + pub leaves_hash_chains: Vec<[u8; 32]>, + pub subtrees: Vec<[u8; 32]>, + pub start_index: u64, + pub root_seq: u64, + /// Original low element proofs from indexer (for debugging/validation) + pub low_element_proofs: Vec>, +} + +impl AddressQueueData { + /// Reconstruct a merkle proof for a given low_element_index from the deduplicated nodes. + /// The tree_height is needed to know how many levels to traverse. + pub fn reconstruct_proof( + &self, + address_idx: usize, + tree_height: u8, + ) -> Result, IndexerError> { + let leaf_index = self.low_element_indices[address_idx]; + let mut proof = Vec::with_capacity(tree_height as usize); + let mut pos = leaf_index; + + for level in 0..tree_height { + let sibling_pos = if pos.is_multiple_of(2) { + pos + 1 + } else { + pos - 1 + }; + let sibling_idx = Self::encode_node_index(level, sibling_pos); + + if let Some(hash_idx) = self.nodes.iter().position(|&n| n == sibling_idx) { + proof.push(self.node_hashes[hash_idx]); + } else { + return Err(IndexerError::MissingResult { + context: "reconstruct_proof".to_string(), + message: format!( + "Missing proof node at level {} position {} (encoded: {})", + level, sibling_pos, sibling_idx + ), + }); + } + pos /= 2; + } + + Ok(proof) + } + + /// Reconstruct all proofs for all addresses + pub fn reconstruct_all_proofs( + &self, + tree_height: u8, + ) -> Result>, IndexerError> { + (0..self.addresses.len()) + .map(|i| self.reconstruct_proof(i, tree_height)) + .collect() + } + + /// Encode node index: (level << 56) | position + #[inline] + fn encode_node_index(level: u8, position: u64) -> u64 { + ((level as u64) << 56) | position + } } /// V2 Queue Elements Result with deduplicated node data #[derive(Debug, Clone, PartialEq, Default)] -pub struct QueueElementsV2Result { - pub state_queue: Option, - pub address_queue: Option, +pub struct QueueElementsResult { + pub state_queue: Option, + pub address_queue: Option, } #[derive(Debug, Clone, PartialEq, Default)] @@ -769,20 +822,6 @@ impl TryFrom<&photon_api::models::Account> for CompressedAccount { } } -#[derive(Debug, Clone, PartialEq, Default)] -pub struct AddressQueueIndex { - pub address: [u8; 32], - pub queue_index: u64, -} - -#[derive(Debug, Clone, PartialEq, Default)] -pub struct BatchAddressUpdateIndexerResponse { - pub batch_start_index: u64, - pub addresses: Vec, - pub non_inclusion_proofs: Vec, - pub subtrees: Vec<[u8; 32]>, -} - #[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq)] pub struct StateMerkleTreeAccounts { pub merkle_tree: Pubkey, diff --git a/sdk-libs/client/src/rpc/indexer.rs b/sdk-libs/client/src/rpc/indexer.rs index f30ef2b0c4..55c6b069e0 100644 --- a/sdk-libs/client/src/rpc/indexer.rs +++ b/sdk-libs/client/src/rpc/indexer.rs @@ -3,12 +3,12 @@ use solana_pubkey::Pubkey; use super::LightClient; use crate::indexer::{ - Address, AddressWithTree, BatchAddressUpdateIndexerResponse, CompressedAccount, - CompressedTokenAccount, GetCompressedAccountsByOwnerConfig, - GetCompressedTokenAccountsByOwnerOrDelegateOptions, Hash, Indexer, IndexerError, - IndexerRpcConfig, Items, ItemsWithCursor, MerkleProof, NewAddressProofWithContext, - OwnerBalance, PaginatedOptions, QueueElementsResult, QueueInfoResult, Response, RetryConfig, - SignatureWithMetadata, TokenBalance, ValidityProofWithContext, + Address, AddressWithTree, CompressedAccount, CompressedTokenAccount, + GetCompressedAccountsByOwnerConfig, GetCompressedTokenAccountsByOwnerOrDelegateOptions, Hash, + Indexer, IndexerError, IndexerRpcConfig, Items, ItemsWithCursor, MerkleProof, + NewAddressProofWithContext, OwnerBalance, PaginatedOptions, QueueElementsResult, + QueueElementsV2Options, QueueInfoResult, Response, RetryConfig, SignatureWithMetadata, + TokenBalance, ValidityProofWithContext, }; #[async_trait] @@ -186,42 +186,17 @@ impl Indexer for LightClient { .await?) } - async fn get_address_queue_with_proofs( - &mut self, - merkle_tree_pubkey: &Pubkey, - zkp_batch_size: u16, - start_offset: Option, - config: Option, - ) -> Result, IndexerError> { - Ok(self - .indexer - .as_mut() - .ok_or(IndexerError::NotInitialized)? - .get_address_queue_with_proofs(merkle_tree_pubkey, zkp_batch_size, start_offset, config) - .await?) - } - async fn get_queue_elements( &mut self, merkle_tree_pubkey: [u8; 32], - output_queue_start_index: Option, - output_queue_limit: Option, - input_queue_start_index: Option, - input_queue_limit: Option, + options: QueueElementsV2Options, config: Option, ) -> Result, IndexerError> { Ok(self .indexer .as_mut() .ok_or(IndexerError::NotInitialized)? - .get_queue_elements( - merkle_tree_pubkey, - output_queue_start_index, - output_queue_limit, - input_queue_start_index, - input_queue_limit, - config, - ) + .get_queue_elements(merkle_tree_pubkey, options, config) .await?) } @@ -237,19 +212,6 @@ impl Indexer for LightClient { .await?) } - async fn get_queue_elements_v2( - &mut self, - merkle_tree_pubkey: [u8; 32], - options: crate::indexer::QueueElementsV2Options, - config: Option, - ) -> Result, IndexerError> { - Ok(self - .indexer - .as_mut() - .ok_or(IndexerError::NotInitialized)? - .get_queue_elements_v2(merkle_tree_pubkey, options, config) - .await?) - } async fn get_subtrees( &self, merkle_tree_pubkey: [u8; 32], diff --git a/sdk-libs/photon-api/src/apis/default_api.rs b/sdk-libs/photon-api/src/apis/default_api.rs index b8db34f8a9..d0dd52fa51 100644 --- a/sdk-libs/photon-api/src/apis/default_api.rs +++ b/sdk-libs/photon-api/src/apis/default_api.rs @@ -1997,55 +1997,6 @@ pub async fn get_validity_proof_v2_post( } } -/// struct for typed errors of method [`get_queue_elements_v2_post`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum GetQueueElementsV2PostError { - Status429(models::GetBatchAddressUpdateInfoPost429Response), - Status500(models::GetBatchAddressUpdateInfoPost429Response), - UnknownValue(serde_json::Value), -} - -/// V2: Get queue elements with deduplicated nodes -pub async fn get_queue_elements_v2_post( - configuration: &configuration::Configuration, - get_queue_elements_v2_post_request: models::GetQueueElementsV2PostRequest, -) -> Result> { - let local_var_configuration = configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!("{}/getQueueElementsV2", local_var_configuration.base_path); - let local_var_uri_str = append_api_key(local_var_configuration, &local_var_uri_str); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = - local_var_req_builder.header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - local_var_req_builder = local_var_req_builder.json(&get_queue_elements_v2_post_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - serde_json::from_str(&local_var_content).map_err(Error::from) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } -} - fn append_api_key(configuration: &Configuration, uri_str: &str) -> String { let mut uri_str = uri_str.to_string(); if let Some(ref api_key) = configuration.api_key { diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs index b37d9cda17..6b43e56c0c 100644 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs +++ b/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs @@ -14,44 +14,18 @@ use crate::models; pub struct GetQueueElementsPost200ResponseResult { #[serde(rename = "context")] pub context: Box, - - #[serde( - rename = "outputQueueElements", - default, - skip_serializing_if = "Option::is_none" - )] - pub output_queue_elements: Option>, - - #[serde( - rename = "outputQueueIndex", - default, - skip_serializing_if = "Option::is_none" - )] - pub output_queue_index: Option, - - #[serde( - rename = "inputQueueElements", - default, - skip_serializing_if = "Option::is_none" - )] - pub input_queue_elements: Option>, - - #[serde( - rename = "inputQueueIndex", - default, - skip_serializing_if = "Option::is_none" - )] - pub input_queue_index: Option, + #[serde(rename = "stateQueue", skip_serializing_if = "Option::is_none")] + pub state_queue: Option>, + #[serde(rename = "addressQueue", skip_serializing_if = "Option::is_none")] + pub address_queue: Option>, } impl GetQueueElementsPost200ResponseResult { pub fn new(context: models::Context) -> GetQueueElementsPost200ResponseResult { GetQueueElementsPost200ResponseResult { context: Box::new(context), - output_queue_elements: None, - output_queue_index: None, - input_queue_elements: None, - input_queue_index: None, + state_queue: None, + address_queue: None, } } } diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs index 64e34e74ef..b2998b6fc0 100644 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs +++ b/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs @@ -11,48 +11,28 @@ use crate::models; #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct GetQueueElementsPostRequestParams { - /// A 32-byte hash represented as a base58 string. - #[serde(rename = "tree")] + /// The merkle tree public key pub tree: String, - #[serde( - rename = "outputQueueStartIndex", - default, - skip_serializing_if = "Option::is_none" - )] - pub output_queue_start_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_queue: Option, - #[serde( - rename = "outputQueueLimit", - default, - skip_serializing_if = "Option::is_none" - )] - pub output_queue_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_queue: Option, - #[serde( - rename = "inputQueueStartIndex", - default, - skip_serializing_if = "Option::is_none" - )] - pub input_queue_start_index: Option, - - #[serde( - rename = "inputQueueLimit", - default, - skip_serializing_if = "Option::is_none" - )] - pub input_queue_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub address_queue: Option, } impl GetQueueElementsPostRequestParams { pub fn new(tree: String) -> GetQueueElementsPostRequestParams { GetQueueElementsPostRequestParams { tree, - output_queue_start_index: None, - output_queue_limit: None, - input_queue_start_index: None, - input_queue_limit: None, + output_queue: None, + input_queue: None, + address_queue: None, } } } diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_200_response.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_200_response.rs deleted file mode 100644 index 13441587c3..0000000000 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_200_response.rs +++ /dev/null @@ -1,60 +0,0 @@ -/* - * photon-indexer - * - * Solana indexer for general compression - * - * The version of the OpenAPI document: 0.50.0 - * - * Generated by: https://openapi-generator.tech - */ - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct GetQueueElementsV2Post200Response { - #[serde(rename = "error", skip_serializing_if = "Option::is_none")] - pub error: Option>, - /// An ID to identify the response. - #[serde(rename = "id")] - pub id: Id, - /// The version of the JSON-RPC protocol. - #[serde(rename = "jsonrpc")] - pub jsonrpc: Jsonrpc, - #[serde(rename = "result", skip_serializing_if = "Option::is_none")] - pub result: Option>, -} - -impl GetQueueElementsV2Post200Response { - pub fn new(id: Id, jsonrpc: Jsonrpc) -> GetQueueElementsV2Post200Response { - GetQueueElementsV2Post200Response { - error: None, - id, - jsonrpc, - result: None, - } - } -} -/// An ID to identify the response. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Id { - #[serde(rename = "test-account")] - TestAccount, -} - -impl Default for Id { - fn default() -> Id { - Self::TestAccount - } -} -/// The version of the JSON-RPC protocol. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Jsonrpc { - #[serde(rename = "2.0")] - Variant2Period0, -} - -impl Default for Jsonrpc { - fn default() -> Jsonrpc { - Self::Variant2Period0 - } -} diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_200_response_result.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_200_response_result.rs deleted file mode 100644 index 34b395e7e8..0000000000 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_200_response_result.rs +++ /dev/null @@ -1,31 +0,0 @@ -/* - * photon-indexer - * - * Solana indexer for general compression - * - * The version of the OpenAPI document: 0.50.0 - * - * Generated by: https://openapi-generator.tech - */ - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct GetQueueElementsV2Post200ResponseResult { - #[serde(rename = "context")] - pub context: Box, - #[serde(rename = "stateQueue", skip_serializing_if = "Option::is_none")] - pub state_queue: Option>, - #[serde(rename = "addressQueue", skip_serializing_if = "Option::is_none")] - pub address_queue: Option>, -} - -impl GetQueueElementsV2Post200ResponseResult { - pub fn new(context: models::Context) -> GetQueueElementsV2Post200ResponseResult { - GetQueueElementsV2Post200ResponseResult { - context: Box::new(context), - state_queue: None, - address_queue: None, - } - } -} diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_request.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_request.rs deleted file mode 100644 index 1d96beffac..0000000000 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_request.rs +++ /dev/null @@ -1,78 +0,0 @@ -/* - * photon-indexer - * - * Solana indexer for general compression - * - * The version of the OpenAPI document: 0.50.0 - * - * Generated by: https://openapi-generator.tech - */ - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct GetQueueElementsV2PostRequest { - /// An ID to identify the request. - #[serde(rename = "id")] - pub id: Id, - /// The version of the JSON-RPC protocol. - #[serde(rename = "jsonrpc")] - pub jsonrpc: Jsonrpc, - /// The name of the method to invoke. - #[serde(rename = "method")] - pub method: Method, - #[serde(rename = "params")] - pub params: Box, -} - -impl GetQueueElementsV2PostRequest { - pub fn new( - id: Id, - jsonrpc: Jsonrpc, - method: Method, - params: models::GetQueueElementsV2PostRequestParams, - ) -> GetQueueElementsV2PostRequest { - GetQueueElementsV2PostRequest { - id, - jsonrpc, - method, - params: Box::new(params), - } - } -} -/// An ID to identify the request. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Id { - #[serde(rename = "test-account")] - TestAccount, -} - -impl Default for Id { - fn default() -> Id { - Self::TestAccount - } -} -/// The version of the JSON-RPC protocol. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Jsonrpc { - #[serde(rename = "2.0")] - Variant2Period0, -} - -impl Default for Jsonrpc { - fn default() -> Jsonrpc { - Self::Variant2Period0 - } -} -/// The name of the method to invoke. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Method { - #[serde(rename = "getQueueElementsV2")] - GetQueueElementsV2, -} - -impl Default for Method { - fn default() -> Method { - Self::GetQueueElementsV2 - } -} diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_request_params.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_request_params.rs deleted file mode 100644 index 3eff127d3d..0000000000 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_v2_post_request_params.rs +++ /dev/null @@ -1,78 +0,0 @@ -/* - * photon-indexer - * - * Solana indexer for general compression - * - * The version of the OpenAPI document: 0.50.0 - * - * Generated by: https://openapi-generator.tech - */ - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct GetQueueElementsV2PostRequestParams { - /// The merkle tree public key - #[serde(rename = "tree")] - pub tree: String, - /// Starting index for the output queue - #[serde( - rename = "outputQueueStartIndex", - skip_serializing_if = "Option::is_none" - )] - pub output_queue_start_index: Option, - /// Limit for the output queue elements - #[serde(rename = "outputQueueLimit", skip_serializing_if = "Option::is_none")] - pub output_queue_limit: Option, - /// Optional override for the output queue ZKP batch size - #[serde( - rename = "outputQueueZkpBatchSize", - skip_serializing_if = "Option::is_none" - )] - pub output_queue_zkp_batch_size: Option, - /// Starting index for the input queue - #[serde( - rename = "inputQueueStartIndex", - skip_serializing_if = "Option::is_none" - )] - pub input_queue_start_index: Option, - /// Limit for the input queue elements - #[serde(rename = "inputQueueLimit", skip_serializing_if = "Option::is_none")] - pub input_queue_limit: Option, - /// Optional override for the input queue ZKP batch size - #[serde( - rename = "inputQueueZkpBatchSize", - skip_serializing_if = "Option::is_none" - )] - pub input_queue_zkp_batch_size: Option, - /// Starting index for the address queue - #[serde( - rename = "addressQueueStartIndex", - skip_serializing_if = "Option::is_none" - )] - pub address_queue_start_index: Option, - /// Limit for the address queue elements - #[serde(rename = "addressQueueLimit", skip_serializing_if = "Option::is_none")] - pub address_queue_limit: Option, - /// Optional override for the address queue ZKP batch size - #[serde( - rename = "addressQueueZkpBatchSize", - skip_serializing_if = "Option::is_none" - )] - pub address_queue_zkp_batch_size: Option, -} - -impl GetQueueElementsV2PostRequestParams { - pub fn new(tree: String) -> GetQueueElementsV2PostRequestParams { - GetQueueElementsV2PostRequestParams { - tree, - output_queue_start_index: None, - output_queue_limit: None, - output_queue_zkp_batch_size: None, - input_queue_start_index: None, - input_queue_limit: None, - input_queue_zkp_batch_size: None, - address_queue_start_index: None, - address_queue_limit: None, - address_queue_zkp_batch_size: None, - } - } -} diff --git a/sdk-libs/photon-api/src/models/address_queue_data_v2.rs b/sdk-libs/photon-api/src/models/address_queue_data_v2.rs index ce8e0b16c8..8c6f6296e5 100644 --- a/sdk-libs/photon-api/src/models/address_queue_data_v2.rs +++ b/sdk-libs/photon-api/src/models/address_queue_data_v2.rs @@ -8,37 +8,31 @@ * Generated by: https://openapi-generator.tech */ +use crate::models; + #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] #[allow(clippy::too_many_arguments)] pub struct AddressQueueDataV2 { - #[serde(rename = "addresses")] pub addresses: Vec, - #[serde(rename = "queueIndices")] pub queue_indices: Vec, /// Deduplicated tree nodes for address tree non-inclusion proofs - /// node_index encoding: (level << 56) | position - #[serde(rename = "nodes")] - pub nodes: Vec, - #[serde(rename = "nodeHashes")] - pub node_hashes: Vec, - #[serde(rename = "lowElementIndices")] + #[serde(default)] + pub nodes: Vec, pub low_element_indices: Vec, - #[serde(rename = "lowElementValues")] pub low_element_values: Vec, - #[serde(rename = "lowElementNextIndices")] pub low_element_next_indices: Vec, - #[serde(rename = "lowElementNextValues")] pub low_element_next_values: Vec, - #[serde(rename = "lowElementProofs", default)] + #[serde(default)] pub low_element_proofs: Vec>, - #[serde(rename = "leavesHashChains", default)] + #[serde(default)] pub leaves_hash_chains: Vec, - #[serde(rename = "initialRoot")] pub initial_root: String, - #[serde(rename = "startIndex")] pub start_index: u64, - #[serde(rename = "subtrees", default)] + #[serde(default)] pub subtrees: Vec, + #[serde(default)] + pub root_seq: u64, } impl AddressQueueDataV2 { @@ -46,8 +40,7 @@ impl AddressQueueDataV2 { pub fn new( addresses: Vec, queue_indices: Vec, - nodes: Vec, - node_hashes: Vec, + nodes: Vec, low_element_indices: Vec, low_element_values: Vec, low_element_next_indices: Vec, @@ -57,12 +50,12 @@ impl AddressQueueDataV2 { initial_root: String, start_index: u64, subtrees: Vec, + root_seq: u64, ) -> AddressQueueDataV2 { AddressQueueDataV2 { addresses, queue_indices, nodes, - node_hashes, low_element_indices, low_element_values, low_element_next_indices, @@ -72,6 +65,7 @@ impl AddressQueueDataV2 { initial_root, start_index, subtrees, + root_seq, } } } diff --git a/sdk-libs/photon-api/src/models/mod.rs b/sdk-libs/photon-api/src/models/mod.rs index 70cd7de750..115861e99e 100644 --- a/sdk-libs/photon-api/src/models/mod.rs +++ b/sdk-libs/photon-api/src/models/mod.rs @@ -332,17 +332,13 @@ pub use self::_get_queue_info_post_200_response_result::{ }; pub mod address_queue_data_v2; pub use self::address_queue_data_v2::AddressQueueDataV2; -pub mod _get_queue_elements_v2_post_200_response; -pub use self::_get_queue_elements_v2_post_200_response::GetQueueElementsV2Post200Response; -pub mod _get_queue_elements_v2_post_200_response_result; -pub use self::_get_queue_elements_v2_post_200_response_result::GetQueueElementsV2Post200ResponseResult; -pub mod _get_queue_elements_v2_post_request; -pub use self::_get_queue_elements_v2_post_request::GetQueueElementsV2PostRequest; -pub mod _get_queue_elements_v2_post_request_params; -pub use self::_get_queue_elements_v2_post_request_params::GetQueueElementsV2PostRequestParams; pub mod input_queue_data_v2; pub use self::input_queue_data_v2::InputQueueDataV2; pub mod output_queue_data_v2; pub use self::output_queue_data_v2::OutputQueueDataV2; pub mod state_queue_data_v2; pub use self::state_queue_data_v2::StateQueueDataV2; +pub mod node; +pub use self::node::Node; +pub mod queue_request; +pub use self::queue_request::QueueRequest; diff --git a/sdk-libs/photon-api/src/models/node.rs b/sdk-libs/photon-api/src/models/node.rs new file mode 100644 index 0000000000..695363db2c --- /dev/null +++ b/sdk-libs/photon-api/src/models/node.rs @@ -0,0 +1,24 @@ +/* + * photon-indexer + * + * Solana indexer for general compression + * + * The version of the OpenAPI document: 0.50.0 + * + * Generated by: https://openapi-generator.tech + */ + +/// A tree node with its encoded index and hash +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Node { + /// Encoded node index: (level << 56) | position + pub index: u64, + pub hash: String, +} + +impl Node { + pub fn new(index: u64, hash: String) -> Node { + Node { index, hash } + } +} diff --git a/sdk-libs/photon-api/src/models/queue_request.rs b/sdk-libs/photon-api/src/models/queue_request.rs new file mode 100644 index 0000000000..55d45d2c80 --- /dev/null +++ b/sdk-libs/photon-api/src/models/queue_request.rs @@ -0,0 +1,40 @@ +/* + * photon-indexer + * + * Solana indexer for general compression + * + * The version of the OpenAPI document: 0.50.0 + * + * Generated by: https://openapi-generator.tech + */ + +/// Parameters for requesting queue elements +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QueueRequest { + pub limit: u16, + #[serde(skip_serializing_if = "Option::is_none")] + pub start_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub zkp_batch_size: Option, +} + +impl QueueRequest { + pub fn new(limit: u16) -> QueueRequest { + QueueRequest { + limit, + start_index: None, + zkp_batch_size: None, + } + } + + pub fn with_start_index(mut self, start_index: u64) -> Self { + self.start_index = Some(start_index); + self + } + + pub fn with_zkp_batch_size(mut self, zkp_batch_size: u16) -> Self { + self.zkp_batch_size = Some(zkp_batch_size); + self + } +} diff --git a/sdk-libs/photon-api/src/models/state_queue_data_v2.rs b/sdk-libs/photon-api/src/models/state_queue_data_v2.rs index 70b52f9c7b..b04e09c600 100644 --- a/sdk-libs/photon-api/src/models/state_queue_data_v2.rs +++ b/sdk-libs/photon-api/src/models/state_queue_data_v2.rs @@ -3,7 +3,7 @@ * * Solana indexer for general compression * - * The version of the OpenAPI document: 0.0.0 + * The version of the OpenAPI document: 0.50.0 * * Generated by: https://openapi-generator.tech */ @@ -12,23 +12,19 @@ use crate::models; /// State queue data with shared tree nodes for output and input queues #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] #[allow(clippy::too_many_arguments)] pub struct StateQueueDataV2 { /// Shared deduplicated tree nodes for state queues (output + input) - /// node_index encoding: (level << 56) | position - #[serde(rename = "nodes", skip_serializing_if = "Vec::is_empty", default)] - pub nodes: Vec, - #[serde(rename = "nodeHashes", skip_serializing_if = "Vec::is_empty", default)] - pub node_hashes: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub nodes: Vec, /// Initial root for the state tree (shared by output and input queues) - #[serde(rename = "initialRoot")] pub initial_root: String, /// Sequence number of the root - #[serde(rename = "rootSeq")] pub root_seq: u64, - #[serde(rename = "outputQueue", skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none")] pub output_queue: Option>, - #[serde(rename = "inputQueue", skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none")] pub input_queue: Option>, } @@ -36,7 +32,6 @@ impl StateQueueDataV2 { pub fn new(initial_root: String, root_seq: u64) -> StateQueueDataV2 { StateQueueDataV2 { nodes: Vec::new(), - node_hashes: Vec::new(), initial_root, root_seq, output_queue: None, diff --git a/sdk-libs/program-test/src/indexer/test_indexer.rs b/sdk-libs/program-test/src/indexer/test_indexer.rs index a8af8bc23e..e338a0428b 100644 --- a/sdk-libs/program-test/src/indexer/test_indexer.rs +++ b/sdk-libs/program-test/src/indexer/test_indexer.rs @@ -17,20 +17,18 @@ use async_trait::async_trait; use borsh::BorshDeserialize; #[cfg(feature = "devenv")] use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeAccount; -#[cfg(feature = "v2")] -use light_client::indexer::MerkleProofWithContext; #[cfg(feature = "devenv")] use light_client::rpc::{Rpc, RpcError}; use light_client::{ fee::FeeConfig, indexer::{ AccountProofInputs, Address, AddressMerkleTreeAccounts, AddressProofInputs, - AddressWithTree, BatchAddressUpdateIndexerResponse, CompressedAccount, - CompressedTokenAccount, Context, GetCompressedAccountsByOwnerConfig, - GetCompressedTokenAccountsByOwnerOrDelegateOptions, Indexer, IndexerError, - IndexerRpcConfig, Items, ItemsWithCursor, MerkleProof, NewAddressProofWithContext, - OwnerBalance, PaginatedOptions, QueueElementsResult, Response, RetryConfig, RootIndex, - SignatureWithMetadata, StateMerkleTreeAccounts, TokenBalance, ValidityProofWithContext, + AddressWithTree, CompressedAccount, CompressedTokenAccount, Context, + GetCompressedAccountsByOwnerConfig, GetCompressedTokenAccountsByOwnerOrDelegateOptions, + Indexer, IndexerError, IndexerRpcConfig, Items, ItemsWithCursor, MerkleProof, + NewAddressProofWithContext, OwnerBalance, PaginatedOptions, QueueElementsResult, + QueueElementsV2Options, Response, RetryConfig, RootIndex, SignatureWithMetadata, + StateMerkleTreeAccounts, TokenBalance, ValidityProofWithContext, }, }; use light_compressed_account::{ @@ -616,58 +614,116 @@ impl Indexer for TestIndexer { async fn get_queue_elements( &mut self, _merkle_tree_pubkey: [u8; 32], - _output_queue_start_index: Option, - _output_queue_limit: Option, - _input_queue_start_index: Option, - _input_queue_limit: Option, + _options: QueueElementsV2Options, _config: Option, ) -> Result, IndexerError> { #[cfg(not(feature = "v2"))] unimplemented!("get_queue_elements"); #[cfg(feature = "v2")] { + use std::collections::HashMap; + + use light_client::indexer::{ + AddressQueueData, InputQueueData, OutputQueueData, StateQueueData, + }; + use light_hasher::bigint::bigint_to_be_bytes_array; + let merkle_tree_pubkey = _merkle_tree_pubkey; - let output_queue_start_index = _output_queue_start_index.unwrap_or(0); - let output_queue_limit = _output_queue_limit; - let input_queue_start_index = _input_queue_start_index.unwrap_or(0); - let input_queue_limit = _input_queue_limit; + let options = _options; let pubkey = Pubkey::new_from_array(merkle_tree_pubkey); + // Helper function to encode node index: (level << 56) | position + fn encode_node_index(level: u8, position: u64) -> u64 { + ((level as u64) << 56) | position + } + + // Helper function to add proof nodes to the deduplicated node map + fn add_proof_to_node_map( + proof: &[[u8; 32]], + leaf_index: u64, + node_map: &mut HashMap, + ) { + let mut pos = leaf_index; + for (level, node_hash) in proof.iter().enumerate() { + let sibling_pos = if pos.is_multiple_of(2) { + pos + 1 + } else { + pos - 1 + }; + let encoded = encode_node_index(level as u8, sibling_pos); + node_map.entry(encoded).or_insert(*node_hash); + pos /= 2; + } + } + // Check if this is an address tree let address_tree_bundle = self .address_merkle_trees .iter() .find(|x| x.accounts.merkle_tree == pubkey); + if let Some(address_tree_bundle) = address_tree_bundle { - // For address trees, return output queue only - let output_queue_elements = if let Some(limit) = output_queue_limit { - let start = output_queue_start_index as usize; + // For address trees, return address queue data if requested + let address_queue = if let Some(limit) = options.address_queue_limit { + let start = options.address_queue_start_index.unwrap_or(0) as usize; let end = std::cmp::min( start + limit as usize, address_tree_bundle.queue_elements.len(), ); - let queue_elements = address_tree_bundle.queue_elements[start..end].to_vec(); - - let merkle_proofs_with_context = queue_elements - .iter() - .map(|element| MerkleProofWithContext { - proof: Vec::new(), - leaf: [0u8; 32], - leaf_index: 0, - merkle_tree: address_tree_bundle.accounts.merkle_tree.to_bytes(), - root: address_tree_bundle.root(), - tx_hash: None, - root_seq: output_queue_start_index, - account_hash: *element, - }) - .collect(); - Some(merkle_proofs_with_context) - } else { - None - }; + let addresses = address_tree_bundle.queue_elements[start..end].to_vec(); + + // Build low element data for each address + let mut low_element_values = Vec::with_capacity(addresses.len()); + let mut low_element_next_values = Vec::with_capacity(addresses.len()); + let mut low_element_indices = Vec::with_capacity(addresses.len()); + let mut low_element_next_indices = Vec::with_capacity(addresses.len()); + let mut low_element_proofs = Vec::with_capacity(addresses.len()); + + // Collect all nodes for deduplication + let mut node_map: HashMap = HashMap::new(); + + for address in &addresses { + let address_biguint = BigUint::from_be_bytes(address.as_slice()); + let (old_low_element, old_low_next_value) = address_tree_bundle + .find_low_element_for_nonexistent(&address_biguint)?; + let proof = + address_tree_bundle.get_proof_of_leaf(old_low_element.index, true)?; + + add_proof_to_node_map(&proof, old_low_element.index as u64, &mut node_map); + + low_element_values + .push(bigint_to_be_bytes_array(&old_low_element.value).unwrap()); + low_element_next_values + .push(bigint_to_be_bytes_array(&old_low_next_value).unwrap()); + low_element_indices.push(old_low_element.index as u64); + low_element_next_indices.push(old_low_element.next_index as u64); + low_element_proofs.push(proof); + } - let output_queue_index = if output_queue_elements.is_some() { - Some(output_queue_start_index) + // Convert node map to sorted vectors + let mut nodes: Vec = node_map.keys().copied().collect(); + nodes.sort(); + let node_hashes: Vec<[u8; 32]> = nodes.iter().map(|k| node_map[k]).collect(); + + let queue_indices: Vec = + (start as u64..(start + addresses.len()) as u64).collect(); + + Some(AddressQueueData { + addresses, + queue_indices, + low_element_values, + low_element_next_values, + low_element_indices, + low_element_next_indices, + nodes, + node_hashes, + initial_root: address_tree_bundle.root(), + leaves_hash_chains: Vec::new(), + subtrees: address_tree_bundle.get_subtrees(), + start_index: start as u64, + root_seq: address_tree_bundle.sequence_number(), + low_element_proofs, + }) } else { None }; @@ -677,10 +733,8 @@ impl Indexer for TestIndexer { slot: self.get_current_slot(), }, value: QueueElementsResult { - output_queue_elements, - output_queue_index, - input_queue_elements: None, - input_queue_index: None, + state_queue: None, + address_queue, }, }); } @@ -689,172 +743,144 @@ impl Indexer for TestIndexer { let state_tree_bundle = self .state_merkle_trees .iter_mut() - .find(|x| x.accounts.merkle_tree == pubkey); + .find(|x| x.accounts.merkle_tree == pubkey || x.accounts.nullifier_queue == pubkey); if let Some(state_tree_bundle) = state_tree_bundle { - // For state trees, return both input and output queues + // Collect nodes for deduplication across both queues + let mut node_map: HashMap = HashMap::new(); - // Build input queue elements if requested - let input_queue_elements = if let Some(limit) = input_queue_limit { - let start = input_queue_start_index as usize; + // Build output queue data if requested + let output_queue = if let Some(limit) = options.output_queue_limit { + let start = options.output_queue_start_index.unwrap_or(0) as usize; let end = std::cmp::min( start + limit as usize, - state_tree_bundle.input_leaf_indices.len(), + state_tree_bundle.output_queue_elements.len(), ); - let queue_elements = state_tree_bundle.input_leaf_indices[start..end].to_vec(); + let queue_elements = + state_tree_bundle.output_queue_elements[start..end].to_vec(); - let merkle_proofs = queue_elements - .iter() - .map(|leaf_info| { - match state_tree_bundle - .merkle_tree - .get_proof_of_leaf(leaf_info.leaf_index as usize, true) - { - Ok(proof) => proof.to_vec(), - Err(_) => { - let mut next_index = - state_tree_bundle.merkle_tree.get_next_index() as u64; - while next_index < leaf_info.leaf_index as u64 { - state_tree_bundle.merkle_tree.append(&[0u8; 32]).unwrap(); - next_index = - state_tree_bundle.merkle_tree.get_next_index() as u64; - } - state_tree_bundle - .merkle_tree - .get_proof_of_leaf(leaf_info.leaf_index as usize, true) - .unwrap() - .to_vec(); - Vec::new() - } - } - }) - .collect::>(); + let leaf_indices: Vec = + queue_elements.iter().map(|(_, index)| *index).collect(); + let account_hashes: Vec<[u8; 32]> = + queue_elements.iter().map(|(hash, _)| *hash).collect(); - let leaves = queue_elements + // Get old leaves at those indices and collect proof nodes + let old_leaves: Vec<[u8; 32]> = leaf_indices .iter() - .map(|leaf_info| { - state_tree_bundle + .map(|index| { + // Extend merkle tree if needed + while state_tree_bundle.merkle_tree.leaves().len() <= *index as usize { + state_tree_bundle.merkle_tree.append(&[0u8; 32]).unwrap(); + } + let leaf = state_tree_bundle .merkle_tree - .get_leaf(leaf_info.leaf_index as usize) - .unwrap_or_default() - }) - .collect::>(); + .get_leaf(*index as usize) + .unwrap_or_default(); - let merkle_proofs_with_context = merkle_proofs - .iter() - .zip(queue_elements.iter()) - .zip(leaves.iter()) - .map(|((proof, element), leaf)| MerkleProofWithContext { - proof: proof.clone(), - leaf: *leaf, - leaf_index: element.leaf_index as u64, - merkle_tree: state_tree_bundle.accounts.merkle_tree.to_bytes(), - root: state_tree_bundle.merkle_tree.root(), - tx_hash: Some(element.tx_hash), - root_seq: 0, - account_hash: element.leaf, + // Get proof and add to node map + if let Ok(proof) = state_tree_bundle + .merkle_tree + .get_proof_of_leaf(*index as usize, true) + { + add_proof_to_node_map(&proof, *index, &mut node_map); + } + leaf }) .collect(); - Some(merkle_proofs_with_context) + Some(OutputQueueData { + leaf_indices, + account_hashes, + old_leaves, + first_queue_index: start as u64, + next_index: state_tree_bundle.merkle_tree.get_next_index() as u64, + leaves_hash_chains: Vec::new(), + }) } else { None }; - // Build output queue elements if requested - let output_queue_elements = if let Some(limit) = output_queue_limit { - let start = output_queue_start_index as usize; + // Build input queue data if requested + let input_queue = if let Some(limit) = options.input_queue_limit { + let start = options.input_queue_start_index.unwrap_or(0) as usize; let end = std::cmp::min( start + limit as usize, - state_tree_bundle.output_queue_elements.len(), + state_tree_bundle.input_leaf_indices.len(), ); - let queue_elements = - state_tree_bundle.output_queue_elements[start..end].to_vec(); + let queue_elements = state_tree_bundle.input_leaf_indices[start..end].to_vec(); - let indices = queue_elements + let leaf_indices: Vec = queue_elements .iter() - .map(|(_, index)| index) - .collect::>(); + .map(|info| info.leaf_index as u64) + .collect(); + let account_hashes: Vec<[u8; 32]> = + queue_elements.iter().map(|info| info.leaf).collect(); + let tx_hashes: Vec<[u8; 32]> = + queue_elements.iter().map(|info| info.tx_hash).collect(); - let merkle_proofs = indices + // Get current leaves and collect proof nodes + let current_leaves: Vec<[u8; 32]> = leaf_indices .iter() .map(|index| { - match state_tree_bundle - .merkle_tree - .get_proof_of_leaf(**index as usize, true) - { - Ok(proof) => proof.to_vec(), - Err(_) => { - let mut next_index = - state_tree_bundle.merkle_tree.get_next_index() as u64; - while next_index < **index { - state_tree_bundle.merkle_tree.append(&[0u8; 32]).unwrap(); - next_index = - state_tree_bundle.merkle_tree.get_next_index() as u64; - } - state_tree_bundle - .merkle_tree - .get_proof_of_leaf(**index as usize, true) - .unwrap() - .to_vec(); - Vec::new() - } + // Extend merkle tree if needed + while state_tree_bundle.merkle_tree.leaves().len() <= *index as usize { + state_tree_bundle.merkle_tree.append(&[0u8; 32]).unwrap(); } - }) - .collect::>(); - - let leaves = indices - .iter() - .map(|index| { - state_tree_bundle + let leaf = state_tree_bundle .merkle_tree - .get_leaf(**index as usize) - .unwrap_or_default() - }) - .collect::>(); + .get_leaf(*index as usize) + .unwrap_or_default(); - let merkle_proofs_with_context = merkle_proofs - .iter() - .zip(queue_elements.iter()) - .zip(leaves.iter()) - .map(|((proof, (element, index)), leaf)| MerkleProofWithContext { - proof: proof.clone(), - leaf: *leaf, - leaf_index: *index, - merkle_tree: state_tree_bundle.accounts.merkle_tree.to_bytes(), - root: state_tree_bundle.merkle_tree.root(), - tx_hash: None, - root_seq: 0, - account_hash: *element, + // Get proof and add to node map + if let Ok(proof) = state_tree_bundle + .merkle_tree + .get_proof_of_leaf(*index as usize, true) + { + add_proof_to_node_map(&proof, *index, &mut node_map); + } + leaf }) .collect(); - Some(merkle_proofs_with_context) - } else { - None - }; - - let output_queue_index = if output_queue_elements.is_some() { - Some(output_queue_start_index) + Some(InputQueueData { + leaf_indices, + account_hashes, + current_leaves, + tx_hashes, + nullifiers: Vec::new(), + first_queue_index: start as u64, + leaves_hash_chains: Vec::new(), + }) } else { None }; - let input_queue_index = if input_queue_elements.is_some() { - Some(input_queue_start_index) + // Build state queue result if either input or output queue was requested + let state_queue = if output_queue.is_some() || input_queue.is_some() { + // Convert node map to sorted vectors + let mut nodes: Vec = node_map.keys().copied().collect(); + nodes.sort(); + let node_hashes: Vec<[u8; 32]> = nodes.iter().map(|k| node_map[k]).collect(); + + Some(StateQueueData { + nodes, + node_hashes, + initial_root: state_tree_bundle.merkle_tree.root(), + root_seq: state_tree_bundle.merkle_tree.sequence_number as u64, + output_queue, + input_queue, + }) } else { None }; - let slot = self.get_current_slot(); - return Ok(Response { - context: Context { slot }, + context: Context { + slot: self.get_current_slot(), + }, value: QueueElementsResult { - output_queue_elements, - output_queue_index, - input_queue_elements, - input_queue_index, + state_queue, + address_queue: None, }, }); } @@ -918,92 +944,6 @@ impl Indexer for TestIndexer { } } - async fn get_address_queue_with_proofs( - &mut self, - _merkle_tree_pubkey: &Pubkey, - _zkp_batch_size: u16, - _start_offset: Option, - _config: Option, - ) -> Result, IndexerError> { - #[cfg(not(feature = "v2"))] - unimplemented!("get_address_queue_with_proofs"); - #[cfg(feature = "v2")] - { - use light_client::indexer::AddressQueueIndex; - let merkle_tree_pubkey = _merkle_tree_pubkey; - let zkp_batch_size = _zkp_batch_size; - - let batch_start_index = self - .get_address_merkle_trees() - .iter() - .find(|x| x.accounts.merkle_tree == *merkle_tree_pubkey) - .unwrap() - .get_v2_indexed_merkle_tree() - .ok_or(IndexerError::Unknown( - "Failed to get v2 indexed merkle tree".into(), - ))? - .merkle_tree - .rightmost_index; - - let address_proof_items = self - .get_queue_elements( - merkle_tree_pubkey.to_bytes(), - Some(0), - Some(zkp_batch_size), - None, - None, - None, - ) - .await - .map_err(|_| IndexerError::Unknown("Failed to get queue elements".into()))? - .value; - - let output_elements = address_proof_items - .output_queue_elements - .ok_or(IndexerError::Unknown("No output queue elements".into()))?; - - let addresses: Vec = output_elements - .iter() - .enumerate() - .map(|(i, proof)| AddressQueueIndex { - address: proof.account_hash, - queue_index: proof.root_seq + i as u64, - }) - .collect(); - let non_inclusion_proofs = self - .get_multiple_new_address_proofs( - merkle_tree_pubkey.to_bytes(), - output_elements.iter().map(|x| x.account_hash).collect(), - None, - ) - .await - .map_err(|_| { - IndexerError::Unknown( - "Failed to get get_multiple_new_address_proofs_full".into(), - ) - })? - .value; - - let subtrees = self - .get_subtrees(merkle_tree_pubkey.to_bytes(), None) - .await - .map_err(|_| IndexerError::Unknown("Failed to get subtrees".into()))? - .value; - - Ok(Response { - context: Context { - slot: self.get_current_slot(), - }, - value: BatchAddressUpdateIndexerResponse { - batch_start_index: batch_start_index as u64, - addresses, - non_inclusion_proofs: non_inclusion_proofs.items, - subtrees: subtrees.items, - }, - }) - } - } - // New required trait methods async fn get_compressed_balance_by_owner( &self, @@ -1061,15 +1001,6 @@ impl Indexer for TestIndexer { async fn get_indexer_health(&self, _config: Option) -> Result { todo!("get_indexer_health not implemented") } - - async fn get_queue_elements_v2( - &mut self, - _merkle_tree_pubkey: [u8; 32], - _options: light_client::indexer::QueueElementsV2Options, - _config: Option, - ) -> Result, IndexerError> { - unimplemented!("get_queue_elements_v2 not implemented for TestIndexer") - } } #[async_trait] diff --git a/sdk-libs/program-test/src/program_test/indexer.rs b/sdk-libs/program-test/src/program_test/indexer.rs index 0ef0b61b7a..a1a80113ce 100644 --- a/sdk-libs/program-test/src/program_test/indexer.rs +++ b/sdk-libs/program-test/src/program_test/indexer.rs @@ -1,11 +1,11 @@ use async_trait::async_trait; use light_client::indexer::{ - Address, AddressWithTree, BatchAddressUpdateIndexerResponse, CompressedAccount, - CompressedTokenAccount, GetCompressedAccountsByOwnerConfig, - GetCompressedTokenAccountsByOwnerOrDelegateOptions, Hash, Indexer, IndexerError, - IndexerRpcConfig, Items, ItemsWithCursor, MerkleProof, NewAddressProofWithContext, - OwnerBalance, PaginatedOptions, QueueElementsResult, Response, RetryConfig, - SignatureWithMetadata, TokenBalance, ValidityProofWithContext, + Address, AddressWithTree, CompressedAccount, CompressedTokenAccount, + GetCompressedAccountsByOwnerConfig, GetCompressedTokenAccountsByOwnerOrDelegateOptions, Hash, + Indexer, IndexerError, IndexerRpcConfig, Items, ItemsWithCursor, MerkleProof, + NewAddressProofWithContext, OwnerBalance, PaginatedOptions, QueueElementsResult, + QueueElementsV2Options, Response, RetryConfig, SignatureWithMetadata, TokenBalance, + ValidityProofWithContext, }; use solana_sdk::pubkey::Pubkey; @@ -182,42 +182,17 @@ impl Indexer for LightProgramTest { .await?) } - async fn get_address_queue_with_proofs( - &mut self, - merkle_tree_pubkey: &Pubkey, - zkp_batch_size: u16, - start_offset: Option, - config: Option, - ) -> Result, IndexerError> { - Ok(self - .indexer - .as_mut() - .ok_or(IndexerError::NotInitialized)? - .get_address_queue_with_proofs(merkle_tree_pubkey, zkp_batch_size, start_offset, config) - .await?) - } - async fn get_queue_elements( &mut self, merkle_tree_pubkey: [u8; 32], - output_queue_start_index: Option, - output_queue_limit: Option, - input_queue_start_index: Option, - input_queue_limit: Option, + options: QueueElementsV2Options, config: Option, ) -> Result, IndexerError> { Ok(self .indexer .as_mut() .ok_or(IndexerError::NotInitialized)? - .get_queue_elements( - merkle_tree_pubkey, - output_queue_start_index, - output_queue_limit, - input_queue_start_index, - input_queue_limit, - config, - ) + .get_queue_elements(merkle_tree_pubkey, options, config) .await?) } @@ -338,18 +313,4 @@ impl Indexer for LightProgramTest { .get_indexer_health(config) .await?) } - - async fn get_queue_elements_v2( - &mut self, - merkle_tree_pubkey: [u8; 32], - options: light_client::indexer::QueueElementsV2Options, - config: Option, - ) -> Result, IndexerError> { - Ok(self - .indexer - .as_mut() - .ok_or(IndexerError::NotInitialized)? - .get_queue_elements_v2(merkle_tree_pubkey, options, config) - .await?) - } }