diff --git a/cli/src/commands/test-validator/index.ts b/cli/src/commands/test-validator/index.ts index f201ee2e6e..dee831c578 100644 --- a/cli/src/commands/test-validator/index.ts +++ b/cli/src/commands/test-validator/index.ts @@ -69,18 +69,18 @@ class SetupCommand extends Command { default: 8784, exclusive: ["skip-indexer"], }), - "prover-port": Flags.integer({ - description: "Enable Light Prover server on this port.", - required: false, - default: 3001, - exclusive: ["skip-prover"], - }), "grpc-port": Flags.integer({ description: "Enable Photon indexer gRPC on this port.", required: false, default: 50051, exclusive: ["skip-indexer"], }), + "prover-port": Flags.integer({ + description: "Enable Light Prover server on this port.", + required: false, + default: 3001, + exclusive: ["skip-prover"], + }), "limit-ledger-size": Flags.integer({ description: "Keep this amount of shreds in root slots.", required: false, diff --git a/cli/src/utils/initTestEnv.ts b/cli/src/utils/initTestEnv.ts index e28994b107..e298f7c2cc 100644 --- a/cli/src/utils/initTestEnv.ts +++ b/cli/src/utils/initTestEnv.ts @@ -1,4 +1,3 @@ -import { airdropSol } from "@lightprotocol/stateless.js"; import { getConfig, getPayer, setAnchorProvider, setConfig } from "./utils"; import { BASE_PATH, @@ -131,9 +130,9 @@ export async function initTestEnv({ await startIndexer( `http://127.0.0.1:${rpcPort}`, indexerPort, + grpcPort, checkPhotonVersion, photonDatabaseUrl, - grpcPort, ); } diff --git a/cli/src/utils/process.ts b/cli/src/utils/process.ts index 9956b15058..ffdc4003e5 100644 --- a/cli/src/utils/process.ts +++ b/cli/src/utils/process.ts @@ -212,6 +212,10 @@ export function spawnBinary(command: string, args: string[] = []) { stdio: ["ignore", out, err], shell: false, detached: true, + env: { + ...process.env, + RUST_LOG: process.env.RUST_LOG || "debug", + }, }); spawnedProcess.on("close", async (code) => { diff --git a/cli/src/utils/processPhotonIndexer.ts b/cli/src/utils/processPhotonIndexer.ts index aa75bb37ba..2f7b1c33df 100644 --- a/cli/src/utils/processPhotonIndexer.ts +++ b/cli/src/utils/processPhotonIndexer.ts @@ -39,9 +39,9 @@ function getPhotonInstallMessage(): string { export async function startIndexer( rpcUrl: string, indexerPort: number, + grpcPort: number = 50051, checkPhotonVersion: boolean = true, photonDatabaseUrl?: string, - grpcPort: number = 50051, ) { await killIndexer(); const resolvedOrNull = which.sync("photon", { nothrow: true }); diff --git a/forester-utils/src/instructions/state_batch_append.rs b/forester-utils/src/instructions/state_batch_append.rs index 25f2787666..32d72d14ff 100644 --- a/forester-utils/src/instructions/state_batch_append.rs +++ b/forester-utils/src/instructions/state_batch_append.rs @@ -9,7 +9,6 @@ use light_batched_merkle_tree::{ use light_client::{indexer::Indexer, rpc::Rpc}; use light_compressed_account::instruction_data::compressed_proof::CompressedProof; use light_hasher::bigint::bigint_to_be_bytes_array; -use light_merkle_tree_metadata::QueueType; use light_prover_client::{ proof_client::ProofClient, proof_types::batch_append::{get_batch_append_inputs, BatchAppendsCircuitInputs}, @@ -120,9 +119,10 @@ pub async fn get_append_instruction_stream<'a, R: Rpc>( indexer .get_queue_elements( merkle_tree_pubkey.to_bytes(), - QueueType::OutputStateV2, - zkp_batch_size, next_queue_index, + Some(zkp_batch_size), + None, + None, None, ) .await @@ -130,8 +130,8 @@ pub async fn get_append_instruction_stream<'a, R: Rpc>( let (batch_elements, batch_first_queue_idx) = match queue_elements_result { Ok(res) => { - let items = res.value.elements; - let first_idx = res.value.first_value_queue_index; + let items = res.value.output_queue_elements.unwrap_or_default(); + let first_idx = res.value.output_queue_index; if items.len() != zkp_batch_size as usize { warn!( "Got {} elements but expected {}, stopping", @@ -238,6 +238,5 @@ pub async fn get_append_instruction_stream<'a, R: Rpc>( yield Ok(proofs_buffer); } }; - Ok((Box::pin(stream), zkp_batch_size)) } diff --git a/forester-utils/src/instructions/state_batch_nullify.rs b/forester-utils/src/instructions/state_batch_nullify.rs index 35aab74a6e..b7bc5bd4d6 100644 --- a/forester-utils/src/instructions/state_batch_nullify.rs +++ b/forester-utils/src/instructions/state_batch_nullify.rs @@ -9,7 +9,6 @@ use light_batched_merkle_tree::{ use light_client::{indexer::Indexer, rpc::Rpc}; use light_compressed_account::instruction_data::compressed_proof::CompressedProof; use light_hasher::bigint::bigint_to_be_bytes_array; -use light_merkle_tree_metadata::QueueType; use light_prover_client::{ proof_client::ProofClient, proof_types::batch_update::{get_batch_update_inputs, BatchUpdateCircuitInputs}, @@ -113,9 +112,10 @@ pub async fn get_nullify_instruction_stream<'a, R: Rpc>( let indexer = connection.indexer_mut()?; indexer.get_queue_elements( merkle_tree_pubkey.to_bytes(), - QueueType::InputStateV2, - zkp_batch_size, + None, + None, next_queue_index, + Some(zkp_batch_size), None, ) .await @@ -123,8 +123,8 @@ pub async fn get_nullify_instruction_stream<'a, R: Rpc>( let (batch_elements, batch_first_queue_idx) = match queue_elements_result { Ok(res) => { - let items = res.value.elements; - let first_idx = res.value.first_value_queue_index; + let items = res.value.input_queue_elements.unwrap_or_default(); + let first_idx = res.value.input_queue_index; if items.len() != zkp_batch_size as usize { warn!( "Got {} elements but expected {}, stopping", diff --git a/forester/src/config.rs b/forester/src/config.rs index d4369ff31d..9cd2eab8e2 100644 --- a/forester/src/config.rs +++ b/forester/src/config.rs @@ -83,6 +83,8 @@ pub struct GeneralConfig { pub skip_v2_state_trees: bool, pub skip_v2_address_trees: bool, pub tree_id: Option, + pub sleep_after_processing_ms: u64, + pub sleep_when_idle_ms: u64, } impl Default for GeneralConfig { @@ -96,6 +98,8 @@ impl Default for GeneralConfig { skip_v2_state_trees: false, skip_v2_address_trees: false, tree_id: None, + sleep_after_processing_ms: 10_000, + sleep_when_idle_ms: 45_000, } } } @@ -111,6 +115,8 @@ impl GeneralConfig { skip_v2_state_trees: true, skip_v2_address_trees: false, tree_id: None, + sleep_after_processing_ms: 50, + sleep_when_idle_ms: 100, } } @@ -124,6 +130,8 @@ impl GeneralConfig { skip_v2_state_trees: false, skip_v2_address_trees: true, tree_id: None, + sleep_after_processing_ms: 50, + sleep_when_idle_ms: 100, } } } @@ -276,6 +284,8 @@ impl ForesterConfig { .tree_id .as_ref() .and_then(|id| Pubkey::from_str(id).ok()), + sleep_after_processing_ms: 10_000, + sleep_when_idle_ms: 45_000, }, rpc_pool_config: RpcPoolConfig { max_size: args.rpc_pool_size, @@ -332,6 +342,8 @@ impl ForesterConfig { skip_v1_address_trees: false, skip_v2_address_trees: false, tree_id: None, + sleep_after_processing_ms: 10_000, + sleep_when_idle_ms: 45_000, }, rpc_pool_config: RpcPoolConfig { max_size: 10, diff --git a/forester/src/epoch_manager.rs b/forester/src/epoch_manager.rs index 4d2fcaa752..c40b7d00d5 100644 --- a/forester/src/epoch_manager.rs +++ b/forester/src/epoch_manager.rs @@ -1260,11 +1260,6 @@ impl EpochManager { forester_slot_details.start_solana_slot, ) .await?; - info!( - "Slot {} started, beginning processing", - forester_slot_details.slot - ); - let mut estimated_slot = self.slot_tracker.estimated_current_slot(); 'inner_processing_loop: loop { @@ -1334,9 +1329,9 @@ impl EpochManager { estimated_slot = self.slot_tracker.estimated_current_slot(); let sleep_duration_ms = if items_processed_this_iteration > 0 { - 10_000 + self.config.general_config.sleep_after_processing_ms } else { - 45_000 + self.config.general_config.sleep_when_idle_ms }; tokio::time::sleep(Duration::from_millis(sleep_duration_ms)).await; @@ -2184,6 +2179,8 @@ mod tests { skip_v2_state_trees: skip_v2_state, skip_v2_address_trees: skip_v2_address, tree_id: None, + sleep_after_processing_ms: 50, + sleep_when_idle_ms: 100, }, rpc_pool_config: Default::default(), registry_pubkey: Pubkey::default(), diff --git a/forester/tests/e2e_test.rs b/forester/tests/e2e_test.rs index 873b8f4261..9c3b3047d9 100644 --- a/forester/tests/e2e_test.rs +++ b/forester/tests/e2e_test.rs @@ -231,6 +231,8 @@ async fn e2e_test() { skip_v1_address_trees: false, skip_v2_address_trees: false, tree_id: None, + sleep_after_processing_ms: 50, + sleep_when_idle_ms: 100, }, rpc_pool_config: RpcPoolConfig { max_size: 50, diff --git a/forester/tests/legacy/test_utils.rs b/forester/tests/legacy/test_utils.rs index a93a410b64..ad0d7a6666 100644 --- a/forester/tests/legacy/test_utils.rs +++ b/forester/tests/legacy/test_utils.rs @@ -94,6 +94,9 @@ pub fn forester_config() -> ForesterConfig { skip_v2_state_trees: false, skip_v1_address_trees: false, skip_v2_address_trees: false, + tree_id: None, + sleep_after_processing_ms: 50, + sleep_when_idle_ms: 100, }, rpc_pool_config: RpcPoolConfig { max_size: 50, diff --git a/forester/tests/test_utils.rs b/forester/tests/test_utils.rs index 409df4d339..c60030a1f1 100644 --- a/forester/tests/test_utils.rs +++ b/forester/tests/test_utils.rs @@ -109,6 +109,8 @@ pub fn forester_config() -> ForesterConfig { skip_v1_address_trees: false, skip_v2_address_trees: false, tree_id: None, + sleep_after_processing_ms: 50, + sleep_when_idle_ms: 100, }, rpc_pool_config: RpcPoolConfig { max_size: 50, diff --git a/program-tests/utils/src/e2e_test_env.rs b/program-tests/utils/src/e2e_test_env.rs index c912e7e2aa..aca8fc6596 100644 --- a/program-tests/utils/src/e2e_test_env.rs +++ b/program-tests/utils/src/e2e_test_env.rs @@ -121,7 +121,6 @@ use light_hasher::{bigint::bigint_to_be_bytes_array, Poseidon}; use light_indexed_merkle_tree::{ array::IndexedArray, reference::IndexedMerkleTree, HIGHEST_ADDRESS_PLUS_ONE, }; -use light_merkle_tree_metadata::QueueType; use light_program_test::{ accounts::{ state_tree::create_state_merkle_tree_and_queue_account, test_accounts::TestAccounts, @@ -746,15 +745,16 @@ where .indexer .get_queue_elements( merkle_tree_pubkey.to_bytes(), - QueueType::AddressV2, - batch.batch_size as u16, + None, + Some(batch.batch_size as u16), + None, None, None, ) .await .unwrap(); let addresses = - addresses.value.elements.iter().map(|x| x.account_hash).collect::>(); + addresses.value.output_queue_elements.unwrap_or_default().iter().map(|x| x.account_hash).collect::>(); // // local_leaves_hash_chain is only used for a test assertion. // let local_nullifier_hash_chain = create_hash_chain_from_array(&addresses); // assert_eq!(leaves_hash_chain, local_nullifier_hash_chain); diff --git a/program-tests/utils/src/test_batch_forester.rs b/program-tests/utils/src/test_batch_forester.rs index 6bc14569a5..f737207ad9 100644 --- a/program-tests/utils/src/test_batch_forester.rs +++ b/program-tests/utils/src/test_batch_forester.rs @@ -23,7 +23,6 @@ use light_batched_merkle_tree::{ use light_client::rpc::{Rpc, RpcError}; use light_compressed_account::{ hash_chain::create_hash_chain_from_slice, instruction_data::compressed_proof::CompressedProof, - QueueType, }; use light_hasher::{bigint::bigint_to_be_bytes_array, Poseidon}; use light_prover_client::{ @@ -654,8 +653,9 @@ pub async fn create_batch_update_address_tree_instruction_data_with_proof>(); diff --git a/prover/server/main.go b/prover/server/main.go index b0eac6e726..1b7122ffb7 100644 --- a/prover/server/main.go +++ b/prover/server/main.go @@ -21,7 +21,7 @@ import ( "github.com/urfave/cli/v2" ) -const Version = "2.0.0" +const Version = "2.0.6" func main() { runCli() diff --git a/scripts/devenv/versions.sh b/scripts/devenv/versions.sh index b38cc9c033..37dc48a1ad 100755 --- a/scripts/devenv/versions.sh +++ b/scripts/devenv/versions.sh @@ -13,7 +13,7 @@ export SOLANA_VERSION="2.2.15" export ANCHOR_VERSION="0.31.1" export JQ_VERSION="1.8.0" export PHOTON_VERSION="0.51.0" -export PHOTON_COMMIT="06862b290f32025bc150f82a4acba4961ee24178" +export PHOTON_COMMIT="94b3688b08477668bb946a689b0267319f5c1ae1" export REDIS_VERSION="8.0.1" export ANCHOR_TAG="anchor-v${ANCHOR_VERSION}" diff --git a/sdk-libs/client/src/indexer/indexer_trait.rs b/sdk-libs/client/src/indexer/indexer_trait.rs index 7e94ebd0d2..ab08965ecf 100644 --- a/sdk-libs/client/src/indexer/indexer_trait.rs +++ b/sdk-libs/client/src/indexer/indexer_trait.rs @@ -1,5 +1,4 @@ use async_trait::async_trait; -use light_merkle_tree_metadata::QueueType; use solana_pubkey::Pubkey; use super::{ @@ -187,16 +186,18 @@ pub trait Indexer: std::marker::Send + std::marker::Sync { // TODO: in different pr: // replace num_elements & start_queue_index with PaginatedOptions // - return type should be ItemsWithCursor - /// Returns queue elements from the queue with the given merkle tree pubkey. For input - /// queues account compression program does not store queue elements in the + /// Returns queue elements from the queue with the given merkle tree pubkey. + /// Can fetch from output queue (append), input queue (nullify), or both atomically. + /// For input queues account compression program does not store queue elements in the /// account data but only emits these in the public transaction event. The /// indexer needs the queue elements to create batch update proofs. async fn get_queue_elements( &mut self, merkle_tree_pubkey: [u8; 32], - queue_type: QueueType, - num_elements: u16, - start_queue_index: Option, + output_queue_start_index: Option, + output_queue_limit: Option, + input_queue_start_index: Option, + input_queue_limit: Option, config: Option, ) -> Result, IndexerError>; diff --git a/sdk-libs/client/src/indexer/photon_indexer.rs b/sdk-libs/client/src/indexer/photon_indexer.rs index 8d80086990..ec70157acd 100644 --- a/sdk-libs/client/src/indexer/photon_indexer.rs +++ b/sdk-libs/client/src/indexer/photon_indexer.rs @@ -2,7 +2,6 @@ use std::{fmt::Debug, time::Duration}; use async_trait::async_trait; use bs58; -use light_merkle_tree_metadata::QueueType; use photon_api::{ apis::configuration::{ApiKey, Configuration}, models::GetCompressedAccountsByOwnerPostRequestParams, @@ -1581,9 +1580,10 @@ impl Indexer for PhotonIndexer { async fn get_queue_elements( &mut self, _pubkey: [u8; 32], - _queue_type: QueueType, - _num_elements: u16, - _start_queue_index: Option, + _output_queue_start_index: Option, + _output_queue_limit: Option, + _input_queue_start_index: Option, + _input_queue_limit: Option, _config: Option, ) -> Result, IndexerError> { #[cfg(not(feature = "v2"))] @@ -1592,18 +1592,20 @@ impl Indexer for PhotonIndexer { { use super::MerkleProofWithContext; let pubkey = _pubkey; - let queue_type = _queue_type; - let limit = _num_elements; - let start_queue_index = _start_queue_index; + let output_queue_start_index = _output_queue_start_index; + let output_queue_limit = _output_queue_limit; + let input_queue_start_index = _input_queue_start_index; + let input_queue_limit = _input_queue_limit; let config = _config.unwrap_or_default(); self.retry(config.retry_config, || async { let request: photon_api::models::GetQueueElementsPostRequest = photon_api::models::GetQueueElementsPostRequest { params: Box::from(photon_api::models::GetQueueElementsPostRequestParams { tree: bs58::encode(pubkey).into_string(), - queue_type: queue_type as u16, - limit, - start_queue_index, + output_queue_start_index, + output_queue_limit, + input_queue_start_index, + input_queue_limit, }), ..Default::default() }; @@ -1619,44 +1621,90 @@ impl Indexer for PhotonIndexer { if api_result.context.slot < config.slot { return Err(IndexerError::IndexerNotSyncedToSlot); } - let response = api_result.value; - let proofs: Vec = response - .iter() - .map(|x| { - let proof = x - .proof + + // Parse output queue elements + let output_queue_elements = api_result + .output_queue_elements + .map(|elements| { + elements + .iter() + .map(|x| -> Result<_, IndexerError> { + let proof: Vec = x + .proof + .iter() + .map(|p| Hash::from_base58(p)) + .collect::, _>>()?; + let root = Hash::from_base58(&x.root)?; + let leaf = Hash::from_base58(&x.leaf)?; + let merkle_tree = Hash::from_base58(&x.tree)?; + let tx_hash = x + .tx_hash + .as_ref() + .map(|h| Hash::from_base58(h)) + .transpose()?; + let account_hash = Hash::from_base58(&x.account_hash)?; + + Ok(MerkleProofWithContext { + proof, + root, + leaf_index: x.leaf_index, + leaf, + merkle_tree, + root_seq: x.root_seq, + tx_hash, + account_hash, + }) + }) + .collect::, _>>() + }) + .transpose()?; + + // Parse input queue elements + let input_queue_elements = api_result + .input_queue_elements + .map(|elements| { + elements .iter() - .map(|x| Hash::from_base58(x).unwrap()) - .collect(); - let root = Hash::from_base58(&x.root).unwrap(); - let leaf = Hash::from_base58(&x.leaf).unwrap(); - let merkle_tree = Hash::from_base58(&x.tree).unwrap(); - let tx_hash = - x.tx_hash.as_ref().map(|x| Hash::from_base58(x).unwrap()); - let account_hash = Hash::from_base58(&x.account_hash).unwrap(); - - MerkleProofWithContext { - proof, - root, - leaf_index: x.leaf_index, - leaf, - merkle_tree, - root_seq: x.root_seq, - tx_hash, - account_hash, - } + .map(|x| -> Result<_, IndexerError> { + let proof: Vec = x + .proof + .iter() + .map(|p| Hash::from_base58(p)) + .collect::, _>>()?; + let root = Hash::from_base58(&x.root)?; + let leaf = Hash::from_base58(&x.leaf)?; + let merkle_tree = Hash::from_base58(&x.tree)?; + let tx_hash = x + .tx_hash + .as_ref() + .map(|h| Hash::from_base58(h)) + .transpose()?; + let account_hash = Hash::from_base58(&x.account_hash)?; + + Ok(MerkleProofWithContext { + proof, + root, + leaf_index: x.leaf_index, + leaf, + merkle_tree, + root_seq: x.root_seq, + tx_hash, + account_hash, + }) + }) + .collect::, _>>() }) - .collect(); + .transpose()?; Ok(Response { context: Context { slot: api_result.context.slot, }, value: QueueElementsResult { - elements: proofs, - first_value_queue_index: Some( - api_result.first_value_queue_index, - ), + output_queue_elements, + output_queue_index: api_result.output_queue_index, + input_queue_elements, + input_queue_index: api_result.input_queue_index, }, }) } diff --git a/sdk-libs/client/src/indexer/types.rs b/sdk-libs/client/src/indexer/types.rs index d0c062a5bb..d88d51d1cf 100644 --- a/sdk-libs/client/src/indexer/types.rs +++ b/sdk-libs/client/src/indexer/types.rs @@ -31,8 +31,10 @@ pub type Hash = [u8; 32]; #[derive(Debug, Clone, PartialEq, Default)] pub struct QueueElementsResult { - pub elements: Vec, - pub first_value_queue_index: Option, + pub output_queue_elements: Option>, + pub output_queue_index: Option, + pub input_queue_elements: Option>, + pub input_queue_index: Option, } #[derive(Debug, Clone, PartialEq, Default)] diff --git a/sdk-libs/client/src/rpc/indexer.rs b/sdk-libs/client/src/rpc/indexer.rs index 8548ef7206..acafeac398 100644 --- a/sdk-libs/client/src/rpc/indexer.rs +++ b/sdk-libs/client/src/rpc/indexer.rs @@ -1,5 +1,4 @@ use async_trait::async_trait; -use light_compressed_account::QueueType; use solana_pubkey::Pubkey; use super::LightClient; @@ -205,9 +204,10 @@ impl Indexer for LightClient { async fn get_queue_elements( &mut self, merkle_tree_pubkey: [u8; 32], - queue_type: QueueType, - num_elements: u16, - start_queue_index: Option, + output_queue_start_index: Option, + output_queue_limit: Option, + input_queue_start_index: Option, + input_queue_limit: Option, config: Option, ) -> Result, IndexerError> { Ok(self @@ -216,9 +216,10 @@ impl Indexer for LightClient { .ok_or(IndexerError::NotInitialized)? .get_queue_elements( merkle_tree_pubkey, - queue_type, - num_elements, - start_queue_index, + output_queue_start_index, + output_queue_limit, + input_queue_start_index, + input_queue_limit, config, ) .await?) diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs index d09de2c0cb..b37d9cda17 100644 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs +++ b/sdk-libs/photon-api/src/models/_get_queue_elements_post_200_response_result.rs @@ -14,22 +14,44 @@ use crate::models; pub struct GetQueueElementsPost200ResponseResult { #[serde(rename = "context")] pub context: Box, - #[serde(rename = "firstValueQueueIndex")] - pub first_value_queue_index: u64, - #[serde(rename = "value")] - pub value: Vec, + + #[serde( + rename = "outputQueueElements", + default, + skip_serializing_if = "Option::is_none" + )] + pub output_queue_elements: Option>, + + #[serde( + rename = "outputQueueIndex", + default, + skip_serializing_if = "Option::is_none" + )] + pub output_queue_index: Option, + + #[serde( + rename = "inputQueueElements", + default, + skip_serializing_if = "Option::is_none" + )] + pub input_queue_elements: Option>, + + #[serde( + rename = "inputQueueIndex", + default, + skip_serializing_if = "Option::is_none" + )] + pub input_queue_index: Option, } impl GetQueueElementsPost200ResponseResult { - pub fn new( - context: models::Context, - first_value_queue_index: u64, - value: Vec, - ) -> GetQueueElementsPost200ResponseResult { + pub fn new(context: models::Context) -> GetQueueElementsPost200ResponseResult { GetQueueElementsPost200ResponseResult { context: Box::new(context), - first_value_queue_index, - value, + output_queue_elements: None, + output_queue_index: None, + input_queue_elements: None, + input_queue_index: None, } } } diff --git a/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs b/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs index 778a50578a..64e34e74ef 100644 --- a/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs +++ b/sdk-libs/photon-api/src/models/_get_queue_elements_post_request_params.rs @@ -12,28 +12,47 @@ use crate::models; #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] pub struct GetQueueElementsPostRequestParams { - #[serde(rename = "limit")] - pub limit: u16, - #[serde(rename = "queueType")] - pub queue_type: u16, + /// A 32-byte hash represented as a base58 string. + #[serde(rename = "tree")] + pub tree: String, + #[serde( - rename = "startQueueIndex", + rename = "outputQueueStartIndex", default, skip_serializing_if = "Option::is_none" )] - pub start_queue_index: Option, - /// A 32-byte hash represented as a base58 string. - #[serde(rename = "tree")] - pub tree: String, + pub output_queue_start_index: Option, + + #[serde( + rename = "outputQueueLimit", + default, + skip_serializing_if = "Option::is_none" + )] + pub output_queue_limit: Option, + + #[serde( + rename = "inputQueueStartIndex", + default, + skip_serializing_if = "Option::is_none" + )] + pub input_queue_start_index: Option, + + #[serde( + rename = "inputQueueLimit", + default, + skip_serializing_if = "Option::is_none" + )] + pub input_queue_limit: Option, } impl GetQueueElementsPostRequestParams { - pub fn new(limit: u16, queue_type: u16, tree: String) -> GetQueueElementsPostRequestParams { + pub fn new(tree: String) -> GetQueueElementsPostRequestParams { GetQueueElementsPostRequestParams { - limit, - queue_type, - start_queue_index: None, tree, + output_queue_start_index: None, + output_queue_limit: None, + input_queue_start_index: None, + input_queue_limit: None, } } } diff --git a/sdk-libs/program-test/src/indexer/test_indexer.rs b/sdk-libs/program-test/src/indexer/test_indexer.rs index 0a91049ba3..b5ae74e3d0 100644 --- a/sdk-libs/program-test/src/indexer/test_indexer.rs +++ b/sdk-libs/program-test/src/indexer/test_indexer.rs @@ -42,7 +42,6 @@ use light_compressed_account::{ }; use light_event::event::PublicTransactionEvent; use light_hasher::{bigint::bigint_to_be_bytes_array, Poseidon}; -use light_merkle_tree_metadata::QueueType; use light_merkle_tree_reference::MerkleTree; use light_prover_client::{ constants::{PROVE_PATH, SERVER_ADDRESS}, @@ -619,9 +618,10 @@ impl Indexer for TestIndexer { async fn get_queue_elements( &mut self, _merkle_tree_pubkey: [u8; 32], - _queue_type: QueueType, - _num_elements: u16, - _start_offset: Option, + _output_queue_start_index: Option, + _output_queue_limit: Option, + _input_queue_start_index: Option, + _input_queue_limit: Option, _config: Option, ) -> Result, IndexerError> { #[cfg(not(feature = "v2"))] @@ -629,56 +629,82 @@ impl Indexer for TestIndexer { #[cfg(feature = "v2")] { let merkle_tree_pubkey = _merkle_tree_pubkey; - let queue_type = _queue_type; - let num_elements = _num_elements; + let output_queue_start_index = _output_queue_start_index.unwrap_or(0); + let output_queue_limit = _output_queue_limit; + let input_queue_start_index = _input_queue_start_index.unwrap_or(0); + let input_queue_limit = _input_queue_limit; let pubkey = Pubkey::new_from_array(merkle_tree_pubkey); + + // Check if this is an address tree let address_tree_bundle = self .address_merkle_trees .iter() .find(|x| x.accounts.merkle_tree == pubkey); if let Some(address_tree_bundle) = address_tree_bundle { - let end_offset = std::cmp::min( - num_elements as usize, - address_tree_bundle.queue_elements.len(), - ); - let queue_elements = address_tree_bundle.queue_elements[0..end_offset].to_vec(); + // For address trees, return output queue only + let output_queue_elements = if let Some(limit) = output_queue_limit { + let start = output_queue_start_index as usize; + let end = std::cmp::min( + start + limit as usize, + address_tree_bundle.queue_elements.len(), + ); + let queue_elements = address_tree_bundle.queue_elements[start..end].to_vec(); + + let merkle_proofs_with_context = queue_elements + .iter() + .map(|element| MerkleProofWithContext { + proof: Vec::new(), + leaf: [0u8; 32], + leaf_index: 0, + merkle_tree: address_tree_bundle.accounts.merkle_tree.to_bytes(), + root: address_tree_bundle.root(), + tx_hash: None, + root_seq: output_queue_start_index, + account_hash: *element, + }) + .collect(); + Some(merkle_proofs_with_context) + } else { + None + }; + + let output_queue_index = if output_queue_elements.is_some() { + Some(output_queue_start_index) + } else { + None + }; - let merkle_proofs_with_context = queue_elements - .iter() - .map(|element| MerkleProofWithContext { - proof: Vec::new(), - leaf: [0u8; 32], - leaf_index: 0, - merkle_tree: address_tree_bundle.accounts.merkle_tree.to_bytes(), - root: address_tree_bundle.root(), - tx_hash: None, - root_seq: 0, - account_hash: *element, - }) - .collect(); return Ok(Response { context: Context { slot: self.get_current_slot(), }, value: QueueElementsResult { - elements: merkle_proofs_with_context, - first_value_queue_index: None, + output_queue_elements, + output_queue_index, + input_queue_elements: None, + input_queue_index: None, }, }); } + // Check if this is a state tree let state_tree_bundle = self .state_merkle_trees .iter_mut() .find(|x| x.accounts.merkle_tree == pubkey); - if queue_type == QueueType::InputStateV2 { - if let Some(state_tree_bundle) = state_tree_bundle { - let end_offset = std::cmp::min( - num_elements as usize, + + if let Some(state_tree_bundle) = state_tree_bundle { + // For state trees, return both input and output queues + + // Build input queue elements if requested + let input_queue_elements = if let Some(limit) = input_queue_limit { + let start = input_queue_start_index as usize; + let end = std::cmp::min( + start + limit as usize, state_tree_bundle.input_leaf_indices.len(), ); - let queue_elements = - state_tree_bundle.input_leaf_indices[0..end_offset].to_vec(); + let queue_elements = state_tree_bundle.input_leaf_indices[start..end].to_vec(); + let merkle_proofs = queue_elements .iter() .map(|leaf_info| { @@ -705,6 +731,7 @@ impl Indexer for TestIndexer { } }) .collect::>(); + let leaves = queue_elements .iter() .map(|leaf_info| { @@ -714,6 +741,7 @@ impl Indexer for TestIndexer { .unwrap_or_default() }) .collect::>(); + let merkle_proofs_with_context = merkle_proofs .iter() .zip(queue_elements.iter()) @@ -730,30 +758,26 @@ impl Indexer for TestIndexer { }) .collect(); - return Ok(Response { - context: Context { - slot: self.get_current_slot(), - }, - value: QueueElementsResult { - elements: merkle_proofs_with_context, - first_value_queue_index: None, - }, - }); - } - } + Some(merkle_proofs_with_context) + } else { + None + }; - if queue_type == QueueType::OutputStateV2 { - if let Some(state_tree_bundle) = state_tree_bundle { - let end_offset = std::cmp::min( - num_elements as usize, + // Build output queue elements if requested + let output_queue_elements = if let Some(limit) = output_queue_limit { + let start = output_queue_start_index as usize; + let end = std::cmp::min( + start + limit as usize, state_tree_bundle.output_queue_elements.len(), ); let queue_elements = - state_tree_bundle.output_queue_elements[0..end_offset].to_vec(); + state_tree_bundle.output_queue_elements[start..end].to_vec(); + let indices = queue_elements .iter() .map(|(_, index)| index) .collect::>(); + let merkle_proofs = indices .iter() .map(|index| { @@ -780,6 +804,7 @@ impl Indexer for TestIndexer { } }) .collect::>(); + let leaves = indices .iter() .map(|index| { @@ -789,6 +814,7 @@ impl Indexer for TestIndexer { .unwrap_or_default() }) .collect::>(); + let merkle_proofs_with_context = merkle_proofs .iter() .zip(queue_elements.iter()) @@ -804,20 +830,35 @@ impl Indexer for TestIndexer { account_hash: *element, }) .collect(); - return Ok(Response { - context: Context { - slot: self.get_current_slot(), - }, - value: QueueElementsResult { - elements: merkle_proofs_with_context, - first_value_queue_index: if queue_elements.is_empty() { - None - } else { - Some(queue_elements[0].1) - }, - }, - }); - } + + Some(merkle_proofs_with_context) + } else { + None + }; + + let output_queue_index = if output_queue_elements.is_some() { + Some(output_queue_start_index) + } else { + None + }; + + let input_queue_index = if input_queue_elements.is_some() { + Some(input_queue_start_index) + } else { + None + }; + + let slot = self.get_current_slot(); + + return Ok(Response { + context: Context { slot }, + value: QueueElementsResult { + output_queue_elements, + output_queue_index, + input_queue_elements, + input_queue_index, + }, + }); } Err(IndexerError::InvalidParameters( @@ -902,8 +943,9 @@ impl Indexer for TestIndexer { let address_proof_items = self .get_queue_elements( merkle_tree_pubkey.to_bytes(), - QueueType::AddressV2, - zkp_batch_size, + Some(0), + Some(zkp_batch_size), + None, None, None, ) @@ -911,8 +953,11 @@ impl Indexer for TestIndexer { .map_err(|_| IndexerError::Unknown("Failed to get queue elements".into()))? .value; - let addresses: Vec = address_proof_items - .elements + let output_elements = address_proof_items + .output_queue_elements + .ok_or(IndexerError::Unknown("No output queue elements".into()))?; + + let addresses: Vec = output_elements .iter() .enumerate() .map(|(i, proof)| AddressQueueIndex { @@ -923,11 +968,7 @@ impl Indexer for TestIndexer { let non_inclusion_proofs = self .get_multiple_new_address_proofs( merkle_tree_pubkey.to_bytes(), - address_proof_items - .elements - .iter() - .map(|x| x.account_hash) - .collect(), + output_elements.iter().map(|x| x.account_hash).collect(), None, ) .await diff --git a/sdk-libs/program-test/src/program_test/indexer.rs b/sdk-libs/program-test/src/program_test/indexer.rs index 9020961b19..86191b1972 100644 --- a/sdk-libs/program-test/src/program_test/indexer.rs +++ b/sdk-libs/program-test/src/program_test/indexer.rs @@ -7,7 +7,6 @@ use light_client::indexer::{ OwnerBalance, PaginatedOptions, QueueElementsResult, Response, RetryConfig, SignatureWithMetadata, TokenBalance, ValidityProofWithContext, }; -use light_compressed_account::QueueType; use solana_sdk::pubkey::Pubkey; use crate::program_test::LightProgramTest; @@ -201,9 +200,10 @@ impl Indexer for LightProgramTest { async fn get_queue_elements( &mut self, merkle_tree_pubkey: [u8; 32], - queue_type: QueueType, - num_elements: u16, - start_queue_index: Option, + output_queue_start_index: Option, + output_queue_limit: Option, + input_queue_start_index: Option, + input_queue_limit: Option, config: Option, ) -> Result, IndexerError> { Ok(self @@ -212,9 +212,10 @@ impl Indexer for LightProgramTest { .ok_or(IndexerError::NotInitialized)? .get_queue_elements( merkle_tree_pubkey, - queue_type, - num_elements, - start_queue_index, + output_queue_start_index, + output_queue_limit, + input_queue_start_index, + input_queue_limit, config, ) .await?)