diff --git a/docs.json b/docs.json index 5b56f660..62533083 100644 --- a/docs.json +++ b/docs.json @@ -47,7 +47,15 @@ "group": "DeFi", "pages": [ "light-token/defi/routers", - "light-token/defi/programs" + "light-token/defi/programs", + "light-token/defi/programs-pinocchio" + ] + }, + { + "group": "Data Streaming", + "pages": [ + "light-token/toolkits/for-streaming-tokens", + "light-token/toolkits/for-streaming-mints" ] }, { @@ -55,9 +63,7 @@ "pages": [ "light-token/toolkits/overview", "light-token/toolkits/for-payments", - "light-token/toolkits/for-wallets", - "light-token/toolkits/for-streaming-mints", - "light-token/toolkits/for-streaming-tokens" + "light-token/toolkits/for-wallets" ] }, { diff --git a/light-token/defi/programs-pinocchio.mdx b/light-token/defi/programs-pinocchio.mdx new file mode 100644 index 00000000..71b66f2d --- /dev/null +++ b/light-token/defi/programs-pinocchio.mdx @@ -0,0 +1,512 @@ +--- +title: "Pinocchio Programs" +description: "Build high-performance DeFi programs with rent-free accounts using Pinocchio" +--- + +The Light-SDK sponsors rent-exemption for your PDAs, token accounts, and mints. Your program logic stays the same. + +| | Before | After | +|------------------------|--------|-------| +| Rent (avg. DeFi pool) | ~$2 | ~$0.02 | + +## What Changes + +| Area | Change | +|------|--------| +| State struct | Add `compression_info: CompressionInfo` field, derive `LightPinocchioAccount` | +| Program enum | Derive `LightProgramPinocchio` to generate compress/decompress handlers | +| Entrypoint | Route generated discriminators alongside your custom ones | +| Init handler | Replace `spl_token` CPIs with `light_token_pinocchio` CPIs to create rent-free accounts | +| Other instructions | No changes | + +If you use Anchor instead of Pinocchio, see [Program Integration](./programs). + +--- + +Complete pinocchio Swap reference implementation: [pinocchio-swap](https://github.com/Lightprotocol/examples-light-token/tree/main/pinocchio/swap) + + +## Step 1: Dependencies + +```toml +[dependencies] +light-account-pinocchio = { version = "0.20", features = ["token", "std"] } +light-token-pinocchio = "0.20" + +pinocchio = "0.9" +pinocchio-pubkey = { version = "0.3", features = ["const"] } +pinocchio-system = "0.3" +borsh = { version = "0.10.4", default-features = false } +bytemuck = { version = "1.21", features = ["derive"] } +``` + +## Step 2: State Struct + +Add `compression_info` field and derive `LightPinocchioAccount`: + +```rust +use borsh::{BorshDeserialize, BorshSerialize}; +use light_account_pinocchio::{CompressionInfo, LightPinocchioAccount}; + +#[derive( + Default, Debug, Copy, Clone, PartialEq, + BorshSerialize, BorshDeserialize, + LightPinocchioAccount, + bytemuck::Pod, bytemuck::Zeroable, +)] +#[repr(C)] +pub struct PoolState { + pub compression_info: CompressionInfo, + + // Your regular state... + pub fee_bps: u16, +} +``` + +## Step 3: Program Enum + +Declare your account types with their seed schemas: + +```rust +use light_account_pinocchio::{ + derive_light_cpi_signer, pubkey_array, CpiSigner, LightProgramPinocchio, +}; +use pinocchio::pubkey::Pubkey; + +pub const ID: Pubkey = pubkey_array!("YourProgram11111111111111111111111111111111"); +pub const LIGHT_CPI_SIGNER: CpiSigner = + derive_light_cpi_signer!("YourProgram11111111111111111111111111111111"); + +#[derive(LightProgramPinocchio)] +pub enum ProgramAccounts { + #[light_account(pda::seeds = [POOL_SEED, ctx.mint_a, ctx.mint_b], pda::zero_copy)] + PoolState(PoolState), + + #[light_account(token::seeds = [POOL_VAULT_SEED, ctx.pool, ctx.mint], token::owner_seeds = [POOL_AUTHORITY_SEED])] + Vault, + + #[light_account(associated_token)] + UserToken, +} +``` + +This auto-generates 4 instructions, discriminators, and the `LightAccountVariant` enum used by the client SDK. + +## Step 4: Entrypoint + +Dispatch the generated handlers in your entrypoint + +```rust +pinocchio::entrypoint!(process_instruction); + +pub fn process_instruction( + _program_id: &Pubkey, + accounts: &[AccountInfo], + instruction_data: &[u8], +) -> Result<(), ProgramError> { + if instruction_data.len() < 8 { + return Err(ProgramError::InvalidInstructionData); + } + + let (disc, data) = instruction_data.split_at(8); + let disc: [u8; 8] = disc.try_into().unwrap(); + + match disc { + // your custom program logic... + discriminators::INITIALIZE => process_initialize(accounts, data), + discriminators::SWAP => process_swap(accounts, data), + + // add this: + ProgramAccounts::INITIALIZE_COMPRESSION_CONFIG => { + ProgramAccounts::process_initialize_config(accounts, data) // generated + } + ProgramAccounts::UPDATE_COMPRESSION_CONFIG => { + ProgramAccounts::process_update_config(accounts, data) + } + ProgramAccounts::COMPRESS_ACCOUNTS_IDEMPOTENT => { + ProgramAccounts::process_compress(accounts, data) + } + ProgramAccounts::DECOMPRESS_ACCOUNTS_IDEMPOTENT => { + ProgramAccounts::process_decompress(accounts, data) + } + _ => Err(ProgramError::InvalidInstructionData), + } +} +``` + +## Step 5: Init Handler + +Update your init instruction. Use `light_token_pinocchio` CPI builders to create rent-free token accounts. + + + +```rust +use light_account_pinocchio::CreateTokenAccountCpi; + +CreateTokenAccountCpi { + payer: ctx.payer, + account: vault, + mint, + owner: *pool_authority.key(), +} +.rent_free( + ctx.light_token_config, + ctx.light_token_rent_sponsor, + ctx.system_program, + &crate::ID, +) +.invoke_signed(&[ + POOL_VAULT_SEED, + pool_key.as_ref(), + mint_key.as_ref(), + &[bump], +])?; +``` + + +```rust +use light_account_pinocchio::{CreateMints, CreateMintsStaticAccounts, SingleMintParams}; + +let sdk_mints: [SingleMintParams<'_>; 2] = [ + SingleMintParams { + decimals: 9, + mint_authority: authority_key, + mint_bump: None, + freeze_authority: None, + mint_seed_pubkey: mint_signer_a_key, + authority_seeds: None, + mint_signer_seeds: Some(mint_signer_a_seeds), + token_metadata: None, + }, + // ... +]; + +CreateMints { + mints: &sdk_mints, + proof_data: ¶ms.create_accounts_proof, + mint_seed_accounts: ctx.mint_signers, + mint_accounts: ctx.mints, + static_accounts: CreateMintsStaticAccounts { + fee_payer: ctx.payer, + compressible_config: ctx.light_token_config, + rent_sponsor: ctx.light_token_rent_sponsor, + cpi_authority: ctx.cpi_authority, + }, + cpi_context_offset: 1, +} +.invoke(&cpi_accounts)?; +``` + + + + + +```rust +use light_account_pinocchio::{ + prepare_compressed_account_on_init, CompressedCpiContext, CpiAccounts, CpiAccountsConfig, + CpiContextWriteAccounts, CreateMints, CreateMintsStaticAccounts, CreateTokenAccountCpi, + InstructionDataInvokeCpiWithAccountInfo, InvokeLightSystemProgram, LightAccount, LightConfig, + SingleMintParams, +}; +use pinocchio::sysvars::{clock::Clock, Sysvar}; + +pub fn process( + ctx: &InitializeAccounts<'_>, + params: &InitializeParams, + remaining_accounts: &[AccountInfo], +) -> Result<(), LightSdkTypesError> { + // 1. Build CPI accounts + let config = CpiAccountsConfig::new_with_cpi_context(crate::LIGHT_CPI_SIGNER); + let cpi_accounts = CpiAccounts::new_with_config( + ctx.payer, + &remaining_accounts[params.create_accounts_proof.system_accounts_offset as usize..], + config, + ); + + // 2. Get address tree info + config + let address_tree_info = ¶ms.create_accounts_proof.address_tree_info; + let address_tree_pubkey = address_tree_info.get_tree_pubkey(&cpi_accounts)?; + let light_config = LightConfig::load_checked(ctx.compressible_config, &crate::ID)?; + let current_slot = Clock::get()?.slot; + + // 3. Create pool PDA (write to CPI context) + { + let cpi_context = CompressedCpiContext::first(); + let mut new_address_params = Vec::with_capacity(1); + let mut account_infos = Vec::with_capacity(1); + let pool_key = *ctx.pool.key(); + + prepare_compressed_account_on_init( + &pool_key, &address_tree_pubkey, address_tree_info, + params.create_accounts_proof.output_state_tree_index, + 0, &crate::ID, + &mut new_address_params, &mut account_infos, + )?; + + // Initialize pool state (zero-copy) + { + let mut data = ctx.pool.try_borrow_mut_data()?; + let pool_state: &mut PoolState = bytemuck::from_bytes_mut( + &mut data[8..8 + core::mem::size_of::()] + ); + pool_state.set_decompressed(&light_config, current_slot); + pool_state.token_a_mint = *ctx.mint_a().key(); + pool_state.token_b_mint = *ctx.mint_b().key(); + // ... remaining fields + } + + // Write to CPI context + let instruction_data = InstructionDataInvokeCpiWithAccountInfo { + mode: 1, + bump: crate::LIGHT_CPI_SIGNER.bump, + invoking_program_id: crate::LIGHT_CPI_SIGNER.program_id.into(), + proof: params.create_accounts_proof.proof.0, + new_address_params, + account_infos, + // ... + }; + instruction_data.invoke_write_to_cpi_context_first( + CpiContextWriteAccounts { + fee_payer: cpi_accounts.fee_payer(), + authority: cpi_accounts.authority()?, + cpi_context: cpi_accounts.cpi_context()?, + cpi_signer: crate::LIGHT_CPI_SIGNER, + } + )?; + } + + // 4. Create mints + CreateMints { /* ... */ }.invoke(&cpi_accounts)?; + + // 5. Create vaults (rent-free) + CreateTokenAccountCpi { /* ... */ }.rent_free(/* ... */).invoke_signed(/* ... */)?; + + Ok(()) +} +``` + + +--- + +## Client SDK + +Implement `LightProgramInterface` so clients can detect cold accounts and build load instructions. + + + +```rust +use light_client::interface::{ + AccountInterface, AccountSpec, ColdContext, LightProgramInterface, PdaSpec, +}; +use light_account::token::Token; +use pinocchio_swap::{LightAccountVariant, PoolState, PoolStateSeeds, VaultSeeds}; + +/// Flat SDK struct. All fields populated at construction from pool state data. +pub struct SwapSdk { + pub pool_state_pubkey: Pubkey, + pub token_a_mint: Pubkey, + pub token_b_mint: Pubkey, + pub token_a_vault: Pubkey, + pub token_b_vault: Pubkey, + pub pool_authority: Pubkey, +} + +impl SwapSdk { + pub fn new(pool_state_pubkey: Pubkey, pool_data: &[u8]) -> Result { + let pool = PoolState::deserialize(&mut &pool_data[8..])?; + // ... derive addresses from pool state + Ok(Self { pool_state_pubkey, /* ... */ }) + } +} + +impl LightProgramInterface for SwapSdk { + type Variant = LightAccountVariant; + type Instruction = SwapInstruction; + + fn program_id() -> Pubkey { PROGRAM_ID } + + fn instruction_accounts(&self, ix: &Self::Instruction) -> Vec { + match ix { + SwapInstruction::Swap => vec![ + self.pool_state_pubkey, + self.token_a_vault, + self.token_b_vault, + self.token_a_mint, + self.token_b_mint, + ], + // ... + } + } + + fn load_specs( + &self, + cold_accounts: &[AccountInterface], + ) -> Result>, Box> { + let mut specs = Vec::new(); + for account in cold_accounts { + if account.key == self.pool_state_pubkey { + let pool = PoolState::deserialize(&mut &account.data()[8..])?; + let variant = LightAccountVariant::PoolState { + seeds: PoolStateSeeds { /* ... */ }, + data: pool, + }; + specs.push(AccountSpec::Pda(PdaSpec::new(account.clone(), variant, PROGRAM_ID))); + } else if account.key == self.token_a_vault { + let token: Token = Token::deserialize(&mut &account.data()[..])?; + let variant = LightAccountVariant::Vault(TokenDataWithSeeds { + seeds: VaultSeeds { pool: /* ... */, mint: /* ... */ }, + token_data: token, + }); + specs.push(AccountSpec::Pda(PdaSpec::new(account.clone(), variant, PROGRAM_ID))); + } + // ... token_b_vault, mints + } + Ok(specs) + } +} +``` + + +| Resource | Link | +|----------|------| +| Full SDK implementation | [sdk.rs](https://github.com/Lightprotocol/examples-light-token/blob/main/pinocchio/swap/tests/sdk.rs) | + +--- + +## Testing + + + +```rust +use light_program_test::{LightProgramTest, Rpc}; +use light_client::interface::{ + create_load_instructions, get_create_accounts_proof, + AccountSpec, CreateAccountsProofInput, LightProgramInterface, +}; + + +#[tokio::test] +async fn test_pool_lifecycle() { + let mut rpc = LightProgramTest::new(config).await.unwrap(); + + // 1. Initialize pool (rent-free: pool PDA, 2 mints, 2 vaults) + let proof = get_create_accounts_proof(&rpc, &program_id, vec![ + CreateAccountsProofInput::pda(pool_state), + CreateAccountsProofInput::mint(mint_a_signer), + CreateAccountsProofInput::mint(mint_b_signer), + ]).await.unwrap(); + + rpc.create_and_send_transaction(&[init_ix], &payer.pubkey(), &[&payer, &authority]) + .await.unwrap(); + + // 2. Swap (hot path) + rpc.create_and_send_transaction(&[swap_ix], &user.pubkey(), &[&user]) + .await.unwrap(); + + // 3. Trigger compression for the purpose of the test. + const SLOTS_PER_EPOCH: u64 = 13500; + rpc.warp_slot_forward(SLOTS_PER_EPOCH * 30).await.unwrap(); + + // 4. Build SDK from pool state, fetch cold accounts + let pool_iface = rpc.get_account_interface(&pool_state, None).await.unwrap().value.unwrap(); + assert!(pool_iface.is_cold()); + + let sdk = SwapSdk::new(pool_state, pool_iface.data()).unwrap(); + let pubkeys = sdk.instruction_accounts(&SwapInstruction::Swap); + let accounts = rpc.get_multiple_account_interfaces(pubkeys.iter().collect(), None) + .await.unwrap().value; + let cold: Vec<_> = accounts.into_iter().flatten().filter(|a| a.is_cold()).collect(); + + // 5. Load cold accounts + let mut specs = sdk.load_specs(&cold).unwrap(); + // Add user ATAs + let ata_a = rpc.get_associated_token_account_interface(&user.pubkey(), &mint_a, None) + .await.unwrap().value.unwrap(); + let ata_b = rpc.get_associated_token_account_interface(&user.pubkey(), &mint_b, None) + .await.unwrap().value.unwrap(); + specs.push(AccountSpec::Ata(ata_a)); + specs.push(AccountSpec::Ata(ata_b)); + + let load_ixs = create_load_instructions(&specs, payer.pubkey(), config_pda, &rpc) + .await.unwrap(); + + + // 6. Load and Swap + let mut all_ixs = load_ixs; + all_ixs.push(swap_ix); + rpc.create_and_send_transaction(&all_ixs, &user.pubkey(), &[&user]) + .await.unwrap(); +} +``` + + +| Resource | Link | +|----------|------| +| Full test | [test_lifecycle.rs](https://github.com/Lightprotocol/examples-light-token/blob/main/pinocchio/swap/tests/test_lifecycle.rs) | + + +--- +## How it works + +The SDK pays the rent-exemption cost. After extended inactivity, cold accounts auto-compress. Your program only ever +interacts with hot accounts. Clients can safely load cold accounts back into the +onchain Solana account space when needed via `create_load_instructions`. + +Under the hood, clients use `AccountInterface` - a superset of Solana's +`Account` that unifies hot and cold state. See [Router Integration](./routers) +for details. + +| | Hot (active) | Cold (inactive) | +|---|---|---| +| Storage | On-chain | Compressed | +| Latency/CU | No change | +load instruction | +| Your program code | No change | No change | + +## Existing programs + +If you want to migrate your program to rent-free accounts and would like hands-on support, [join our tech Discord](https://discord.com/invite/7cJ8BhAXhu), +or [email us](mailto:support@lightprotocol.com). + + +## FAQ + + +No. `LightProgramPinocchio` generates the handlers. Simply add the generated handlers to your entrypoint, and update your init instruction. + + + When creating an +account for the first time, the SDK provides a proof that the account doesn't +exist in the cold address space. The SVM already verifies this for the onchain +space. Both address spaces are checked before creation, preventing re-init +attacks, even if the account is currently cold. + + +Miners (Forester nodes) compress accounts that have been inactive for an extended period of time (when their virtual rent balance drops below threshold). +In practice, having to load cold accounts should be rare. The common path (hot) has no extra overhead and does not increase CU or txn size. + + + +When accounts compress after extended inactivity, the on-chain rent-exemption is released back +to the rent sponsor. This creates a revolving lifecycle: active "hot" accounts hold a +rent-exempt lamports balance, inactive "cold" accounts release it back. The +rent sponsor must be derived from the program owner. For all mint, ATA, and +token accounts, the Light Token Program is the rent sponsor. For your own program-owned PDAs, the SDK derives a rent sponsor address automatically. + + + +**Hot path (e.g. swap, deposit, withdraw):** No. Active accounts do not add CU overhead to your instructions. + +**First time init + loading cold accounts:** Yes, adds up to 15k-400k CU, +depending on number and type of accounts being initialized or loaded. + + + + + +--- + + +API is in Beta and subject to change. + +Questions or need hands-on support? [Telegram](https://t.me/swen_light) | [email](mailto:support@lightprotocol.com) | [Discord](https://discord.com/invite/7cJ8BhAXhu) + diff --git a/light-token/defi/programs.mdx b/light-token/defi/programs.mdx index 1f7458ae..719c5099 100644 --- a/light-token/defi/programs.mdx +++ b/light-token/defi/programs.mdx @@ -22,8 +22,7 @@ The Light-SDK sponsors rent-exemption for your PDAs, token accounts, and mints. Audit overhead is minimal as your program logic is mostly untouched. The rest is macro-generated. -If you don't use Anchor, [let us know](https://discord.com/invite/7cJ8BhAXhu). -References for native solana-program integration coming soon. +If you don't use Anchor, see the [Pinocchio Programs](./programs-pinocchio) guide. --- @@ -359,11 +358,14 @@ For a detailed example of how clients use this trait, check out the [Router Inte ```rust pub struct AmmSdk { - pool_state_pubkey: Option, - token_0_vault: Option, - token_1_vault: Option, - // ... other fields - program_owned_specs: HashMap>, + pub pool_state_pubkey: Pubkey, + pub observation_key: Pubkey, + pub token_0_vault: Pubkey, + pub token_1_vault: Pubkey, + pub token_0_mint: Pubkey, + pub token_1_mint: Pubkey, + pub lp_mint: Pubkey, + pub amm_config: Pubkey, } pub enum AmmInstruction { @@ -375,46 +377,51 @@ pub enum AmmInstruction { impl LightProgramInterface for AmmSdk { type Variant = LightAccountVariant; type Instruction = AmmInstruction; - type Error = AmmSdkError; - fn program_id(&self) -> Pubkey { + fn program_id() -> Pubkey { PROGRAM_ID } - fn from_keyed_accounts(accounts: &[AccountInterface]) -> Result { - let mut sdk = Self::new(); - for account in accounts { - sdk.parse_account(account)?; - } - Ok(sdk) - } - - fn get_accounts_to_update(&self, ix: &Self::Instruction) -> Vec { + fn instruction_accounts(&self, ix: &Self::Instruction) -> Vec { match ix { AmmInstruction::Swap => vec![ - AccountToFetch::pda(self.pool_state_pubkey.unwrap(), PROGRAM_ID), - AccountToFetch::token(self.token_0_vault.unwrap()), - AccountToFetch::token(self.token_1_vault.unwrap()), + self.pool_state_pubkey, + self.observation_key, + self.token_0_vault, + self.token_1_vault, + self.token_0_mint, + self.token_1_mint, + ], + AmmInstruction::Deposit | AmmInstruction::Withdraw => vec![ + self.pool_state_pubkey, + self.observation_key, + self.token_0_vault, + self.token_1_vault, + self.token_0_mint, + self.token_1_mint, + self.lp_mint, ], - // ... } } - fn update(&mut self, accounts: &[AccountInterface]) -> Result<(), Self::Error> { - for account in accounts { - self.parse_account(account)?; + fn load_specs( + &self, + cold_accounts: &[AccountInterface], + ) -> Result>, Box> { + // Build AccountSpec for each cold account by matching pubkey + // and deserializing its data into the macro-generated variant. + let mut specs = Vec::new(); + for account in cold_accounts { + let pubkey = account.key(); + if pubkey == self.pool_state_pubkey || pubkey == self.observation_key { + let parsed: PoolState = AnchorDeserialize::deserialize(&mut &account.data()[8..])?; + specs.push(AccountSpec::Pda(PdaSpec { interface: account.clone(), variant: parsed.into() })); + } else if pubkey == self.token_0_vault || pubkey == self.token_1_vault { + specs.push(AccountSpec::Token(account.clone())); + } + // ... } - Ok(()) - } - - fn get_specs_for_instruction(&self, ix: &Self::Instruction) -> Vec> { - // Return specs for accounts needed by this instruction - // Specs include the variant (seeds) needed for loading cold accounts back onchain. - self.program_owned_specs - .values() - .cloned() - .map(AccountSpec::Pda) - .collect() + Ok(specs) } } ``` @@ -434,7 +441,7 @@ impl LightProgramInterface for AmmSdk { ```rust use light_program_test::{LightProgramTest, ProgramTestConfig, Rpc}; use light_sdk::interface::rent::SLOTS_PER_EPOCH; -use light_client::interface::{create_load_instructions, LightProgramInterface, AccountSpec}; +use light_client::interface::{create_load_instructions, LightProgramInterface}; #[tokio::test] async fn test_pool_lifecycle() { @@ -443,7 +450,6 @@ async fn test_pool_lifecycle() { // 1. Init pool (rent-free) // ... build and send init instruction ... - assert!(rpc.get_account_interface(&pool_address, &program_id).await.unwrap().is_some()); // 2. Swap (hot path - works normally) // ... build and send swap instruction ... @@ -452,29 +458,24 @@ async fn test_pool_lifecycle() { rpc.warp_slot_forward(SLOTS_PER_EPOCH * 30).await.unwrap(); let pool_interface = rpc - .get_account_interface(&pool_address, &program_id) + .get_account_interface(&pool_address, None) .await + .unwrap() + .value .unwrap(); - assert!(pool_interface.is_cold()); // get_account would return None + assert!(pool_interface.is_cold()); - // 4. get load instructions - let mut sdk = AmmSdk::from_keyed_accounts(&[pool_interface]).unwrap(); - let accounts_to_fetch = sdk.get_accounts_to_update(&AmmInstruction::Deposit); - let keyed_accounts = rpc.get_multiple_account_interfaces(&accounts_to_fetch).await.unwrap(); - sdk.update(&keyed_accounts).unwrap(); - - let specs = sdk.get_specs_for_instruction(&AmmInstruction::Deposit); - let load_ixs = create_load_instructions( - &specs, - payer.pubkey(), - config_pda, - payer.pubkey(), - &rpc, - ).await.unwrap(); - - // 5. send transaction + // 4. Build SDK and get load instructions + let sdk = AmmSdk::new(pool_address, pool_interface.data()).unwrap(); + let pubkeys = sdk.instruction_accounts(&AmmInstruction::Deposit); + let accounts = rpc.get_multiple_account_interfaces(pubkeys.iter().collect(), None).await.unwrap().value; + let cold: Vec<_> = accounts.into_iter().flatten().filter(|a| a.is_cold()).collect(); + + let specs = sdk.load_specs(&cold).unwrap(); + let load_ixs = create_load_instructions(&specs, payer.pubkey(), config_pda, &rpc).await.unwrap(); + + // 5. Send transaction rpc.create_and_send_transaction(&load_ixs, &payer.pubkey(), &[&payer]).await.unwrap(); - assert!(rpc.get_account_interface(&pool_address, &program_id).await.unwrap().is_hot()); } ``` @@ -516,7 +517,7 @@ space. Both address spaces are checked before creation, preventing re-init attacks, even if the account is currently cold. -Miners compress accounts that have been inactive for an extended period of time (when their virtual rent balance drops below threshold). +Miners (Forester nodes) compress accounts that have been inactive for an extended period of time (when their virtual rent balance drops below threshold). In practice, having to load cold accounts should be rare. The common path (hot) has no extra overhead and does not increase CU or txn size. diff --git a/light-token/defi/routers.mdx b/light-token/defi/routers.mdx index 190caf31..2be65c93 100644 --- a/light-token/defi/routers.mdx +++ b/light-token/defi/routers.mdx @@ -3,85 +3,135 @@ title: "Router Integration" description: "Add support for rent-free AMMs on Solana." --- -1. Use `get_account_interface` instead of `get_account` to store `AccountInterface`. -2. Use the AMM's `LightProgramInterface` trait to load inactive markets when needed. +Your existing quoting, routing, and swap-building logic stays the same. +The only addition: when a market has cold accounts, detect them, and prepend load instructions before the swap. +## What changes -## Step 1: Use `get_account_interface` +| | Hot market (99%+) | Cold market | +|---|---|---| +| Quoting | No change | No change | +| Swap instruction | No change | No change | +| Transaction | No change | Prepend `create_load_instructions` | -`get_account_interface` is a new RPC endpoint that returns a superset of -`get_account`. `AccountInterface` stores additional info in `Option` -that you will need later. +--- + +## Detecting cold accounts + +Add a cache for cold accounts. This can be separate from your regular account cache. + +```rust +cold_cache: HashMap<[u8; 32], AccountInterface>, // Accounts with `ColdContext` (used for loading) +``` + +If you stream, subscribe to accounts and transactions with the Light Token Program. + +| Subscription | Detects | +|:-------------|:--------| +| Account sub (`owner: cToken...`) | Hot state + cold-to-hot | +| Transaction sub (`account_include: cToken...`) | Hot-to-cold | + +**Hot-to-cold** -- in your transaction handler, listen to accounts whose balance dropped to zero. Async-fetch the `AccountInterface` +(which includes the `ColdContext` needed for load instructions): -
-
-

Account

```rust -pub struct Account { - pub lamports: u64, - pub data: Vec, - pub owner: Pubkey, - pub executable: bool, - pub rent_epoch: Epoch, +Some(UpdateOneof::Transaction(tx_update)) => { + if let Some(ref tx_info) = tx_update.transaction { + for pubkey in find_closed_accounts(tx_info) { + if cache.remove(&pubkey).is_some() { + let rpc = rpc.clone(); + let cold_cache = cold_cache.clone(); + tokio::spawn(async move { + if let Ok(Some(iface)) = rpc.get_account_interface(&pubkey, None).await { + cold_cache.insert(pubkey, iface); + } + }); + } + } + } } ``` -
-
-

AccountInterface

+ +`find_closed_accounts` checks `pre_balances[i] > 0 && post_balances[i] == 0` across all transaction +keys (including ALT-loaded addresses). See [full implementation](/light-token/toolkits/for-streaming-tokens#detecting-transitions). + +**Cold-to-hot** -- your existing account subscription picks up the hot account again. No cache changes needed because account state does not change while cold. + ```rust -pub struct AccountInterface { - key: Pubkey, - account: Account, - cold: Option +Some(UpdateOneof::Account(account_update)) => { + if let Some(account) = account_update.account { + let pubkey: [u8; 32] = account.pubkey.as_slice().try_into().unwrap(); + // remove from cold cache. + cold_cache.remove(&pubkey); + } } ``` -
-
+For the full streaming guide, see [Streaming Token Accounts](/light-token/toolkits/for-streaming-tokens) +and [Streaming Mints](/light-token/toolkits/for-streaming-mints). + +If you don't stream, call `get_multiple_account_interfaces` at swap time and check `is_cold()` to detect cold accounts. -## Step 2: Use the LightProgramInterface trait -All rent-free programs expose a `LightProgramInterface` trait in their SDK. +--- -Using this trait allows you to: +## Building swap transactions with cold accounts -1. Maintain a cache of `&[AccountInterface]` -2. Load inactive "cold" accounts back into the onchain account space when - building Swap transactions in a standardized way. +When you detect cold accounts in a market (via your `cold_set` or via `is_cold()` on fetched accounts), +fetch their `ColdContext` via `get_account_interface` and build load instructions. ```rust -// AMM SDK implements this. -pub trait LightProgramInterface { - fn from_keyed_accounts(accounts: &[AccountInterface]) -> Result; - fn get_accounts_to_update(&self, ix: &Self::Instruction) -> Vec; - fn update(&mut self, accounts: &[AccountInterface]) -> Result<(), Self::Error>; - fn get_specs_for_instruction(&self, ix: &Self::Instruction) -> Vec>; +use light_client::interface::{create_load_instructions, LightProgramInterface}; + +// 1. Identify which accounts the swap touches +let pubkeys = sdk.instruction_accounts(&AmmInstruction::Swap); + +// 2. Check which are cold (from your streaming cache, or fetch) +let cold_pubkeys: Vec<_> = pubkeys.iter().filter(|p| cold_set.contains(p)).collect(); + +// 3. If any are cold, fetch their ColdContext and build load instructions +let mut ixs = vec![]; +if !cold_pubkeys.is_empty() { + let interfaces = rpc + .get_multiple_account_interfaces(cold_pubkeys, None) + .await? + .value; + let cold: Vec<_> = interfaces.into_iter().flatten().collect(); + let specs = sdk.load_specs(&cold)?; + ixs.extend(create_load_instructions(&specs, payer, config_pda, &rpc).await?); } + +// 4. Swap instruction is unchanged +ixs.push(sdk.swap_ix(&swap_params)?); ``` -## Step 3: Load cold accounts when building Swap instructions -When building Swap instructions, prepend a `create_load_instructions` call. +--- + +## The LightProgramInterface trait -This only adds latency if markets are cold. +Each rent-free AMM SDK exposes this trait. It tells you which accounts an instruction +touches and how to build load specs for cold ones. ```rust -// add dynamic load instructions if cold -let specs = sdk.get_specs_for_instruction(&ExampleAmmSdk::LightInstruction::Swap); -if specs.iter().any(|s| s.is_cold()) { - let load_ixs = create_load_instructions( - &specs, - payer.pubkey(), - sdk.light_config_pda(), - sdk.light_rent_sponsor_pda(), - &rpc, - ).await?; - instructions.extend(load_ixs); +pub trait LightProgramInterface { + type Variant: Pack + Clone + Debug; + type Instruction; + + fn program_id() -> Pubkey; + fn instruction_accounts(&self, ix: &Self::Instruction) -> Vec; + fn load_specs( + &self, + cold_accounts: &[AccountInterface], + ) -> Result>, Box>; } - -// add swap instruction... ``` +- `instruction_accounts` -- returns the pubkeys the instruction reads/writes. +- `load_specs` -- given cold `AccountInterface`s (with `ColdContext`), returns the `AccountSpec`s + that `create_load_instructions` needs to bring them back on-chain. + +--- -## Full Example +## Full example ### Dependencies @@ -89,73 +139,61 @@ if specs.iter().any(|s| s.is_cold()) { [dependencies] light-client = {version = "0.18.0", features = ["v2"]} -# Example Program SDK that implements LightProgramInterface (provided by AMM team) +# AMM SDK that implements LightProgramInterface (provided by the AMM team) example-amm-sdk = "0.1" ``` ### Code ```rust expandable -use light_client::interface::{ - create_load_instructions, LightProgramInterface, AccountSpec, -}; -use example_amm_sdk::{ExampleAmmSdk}; +use light_client::interface::{create_load_instructions, LightProgramInterface}; +use example_amm_sdk::{ExampleAmmSdk, AmmInstruction}; -// 1. Fetch account interfaces (works for both hot and cold) -let pool_interface = rpc - .get_account_interface(&pool_address, &ExampleAmmSdk::program_id()) - .await?; +// Construct SDK from pool data (same as before -- pool data is always available, +// hot or cold, via get_account_interface or your cache). +let sdk = ExampleAmmSdk::new(pool_address, pool_data)?; -// 2. Initialize SDK from interfaces -let mut sdk = ExampleAmmSdk::from_keyed_accounts(&[pool_interface])?; - -// 3. Fetch related accounts and update SDK state -let accounts_to_fetch = sdk.get_accounts_to_update(&ExampleAmmSdk::LightInstruction::Swap); -let keyed_accounts = rpc.get_multiple_account_interfaces(&accounts_to_fetch).await?; -sdk.update(&keyed_accounts)?; - -// 4. Quote (works same for hot or cold) +// Quote works the same regardless of hot/cold. let quote = sdk.quote(amount_in, min_out)?; -// 5. Build transaction +// Build transaction. let mut ixs = vec![]; -// Prepend load instructions if any accounts are cold -let specs = sdk.get_specs_for_instruction(&ExampleAmmSdk::LightInstruction::Swap); -if specs.iter().any(|s| s.is_cold()) { - let load_ixs = create_load_instructions( - &specs, - payer.pubkey(), - sdk.light_config_pda(), - sdk.light_rent_sponsor_pda(), - &rpc, - ).await?; - ixs.extend(load_ixs); +// Check if any swap accounts are cold. +let pubkeys = sdk.instruction_accounts(&AmmInstruction::Swap); +let cold_pubkeys: Vec<_> = pubkeys.iter().filter(|p| cold_set.contains(p)).collect(); + +if !cold_pubkeys.is_empty() { + // Fetch ColdContext for cold accounts. + let interfaces = rpc + .get_multiple_account_interfaces(cold_pubkeys, None) + .await? + .value; + let cold: Vec<_> = interfaces.into_iter().flatten().collect(); + let specs = sdk.load_specs(&cold)?; + ixs.extend(create_load_instructions(&specs, payer.pubkey(), config_pda, &rpc).await?); } -// Add actual swap instruction +// Swap instruction is the same as without rent-free accounts. ixs.push(sdk.swap_ix(&swap_params)?); -// 6. Send rpc.send_transaction(&ixs, &payer).await?; ``` -### Key Types +### Key types | Type | Source | Purpose | |------|--------|---------| -| `Rpc` trait | `light-client` | RPC client with `get_account_interface` methods | -| `AccountInterface` | `light-client` | Unified hot/cold account type | -| `LightProgramInterface` | `light-client` | Trait that program SDKs implement | -| `AccountSpec` | `light-client` | Specifies account load requirements | +| `AccountInterface` | `light-client` | Account data with optional `ColdContext` | +| `LightProgramInterface` | `light-client` | Trait that AMM SDKs implement | +| `AccountSpec` | `light-client` | Input to `create_load_instructions` | -### Reference Implementation +### Reference implementation | Resource | Link | |----------|------| | AMM Program | [cp-swap-reference](https://github.com/Lightprotocol/cp-swap-reference) | | LightProgramInterface Trait Impl | [CpSwapSdk](https://github.com/Lightprotocol/cp-swap-reference/blob/main/programs/cp-swap/tests/program.rs#L409) | -| Client Test | [program.rs](https://github.com/Lightprotocol/cp-swap-reference/blob/main/programs/cp-swap/tests/program.rs) | --- @@ -170,68 +208,46 @@ rpc.send_transaction(&ixs, &payer).await?; | Tx size | Normal | +100-2400 bytes*| | CU | Normal | +15k-400k CU*| -Latency, tx size, and CU depend on the number and type of cold accounts. +*Depends on the number and type of cold accounts.* ### When does a market go cold? -Accounts become "cold" after extended inactivity, causing their virtual rent balance to fall below a threshold. Once cold, they auto-compress onto the Solana ledger. -They remain cold until the first client loads them back into the hot state in-flight via `create_load_instructions`. +Accounts go cold after extended inactivity. Their virtual rent balance drops +below a threshold and miners compress them onto the Solana ledger. -**In practice, touching cold markets is rare.** The common path ("hot") has no extra latency, tx size, or CU overhead. +They stay cold until any client loads them back in-flight via `create_load_instructions`. ---- -## Error Handling - -```rust -use light_client::error::LightClientError; +**Touching cold markets is rare.** The hot path has zero overhead. -match rpc.get_account_interface(&pool_address, &program_id).await { - Ok(account) => { - // Account is hot or cold - // Proceed with quote and swap - } - Err(LightClientError::AccountNotFound) => { - // Account does not exist - } -} -``` +--- ## FAQ -No. In all cases, swap instructions stay the same. -If the market is active (hot), the transaction is identical to today (UX, CU, latency, txn size,...). -If the market is inactive (cold), you additionally prepend `create_load_instructions`. +No. Swap instructions are identical. If the market is hot, the transaction +is the same as today. If cold, you prepend `create_load_instructions`. - Yes. `get_account_interface` is a -superset of `get_account` and returns the full account state via the same `Account` -type, regardless of whether the account is hot or cold. Quoting works all the same. + +Yes. `get_account_interface` returns full account data regardless of hot/cold. +Quoting works the same. + - - **Active markets (hot path)**: No additional latency. + +**Hot (common path)**: No. - **Inactive markets (cold)**: Yes, loading accounts back into Solana's active state adds - 1-200ms depending on whether a validity proof is needed. If loading multiple - cold accounts exceeds Solana's 1232 byte limit, use Jito bundles to maintain - atomicity and reduce latency. Future updates will continue to reduce - transaction size and CU usage for loading cold accounts. +**Cold**: Loading accounts adds 1-200ms depending on whether a validity proof +is needed. If load + swap exceed Solana's 1232 byte limit, use Jito bundles. -Accounts stay hot until they become inactive again. After extended inactivity (configurable by the program owner, e.g., 24h of no writes), their virtual rent balance falls below a threshold and miners compress them back to cold state. Each write extends the "hot" period. +Until they go inactive again. Each write resets the timer. The inactivity +threshold is configurable by the program owner (e.g. 24h of no writes). - - -In some cases, yes. You can detect this at runtime by inspecting the Instructions returned by `create_load_instructions`. - -Note that the SDK deduplicates many of the account keys over the wire, so -instructions that may appear large in isolation will be incremental when -combined with other instructions, such as Swap and Deposit. - -If load instructions + swap instructions exceed Solana's 1232 byte limit, send as a Jito bundle: + +Send as a Jito bundle: ```rust expandable use solana_sdk::{instruction::Instruction, pubkey::Pubkey, system_instruction}; @@ -253,8 +269,7 @@ fn jito_tip_ix(payer: &Pubkey, tip_lamports: u64) -> Instruction { system_instruction::transfer(payer, &tip_account, tip_lamports) } -// Add tip to last transaction, serialize, send to Jito -let tip_ix = jito_tip_ix(&payer.pubkey(), 10_000); // 10k lamports +let tip_ix = jito_tip_ix(&payer.pubkey(), 10_000); swap_ixs.push(tip_ix); let bundle = vec![load_tx_base64, swap_tx_base64]; @@ -270,13 +285,21 @@ let resp = client ``` - -Yes. The relevant RPC methods are supported by providers such as Helius and Triton and can also be self-hosted via the open-source Photon indexer, which is maintained by Helius Labs. + +Yes. Supported by Helius and Triton. Can also be self-hosted via the +open-source Photon indexer. + + + +Hot markets work as long as Solana is up. Cold accounts can't be loaded until +the indexer recovers. Compression is cryptographically verifiable -- integrity +doesn't depend on the indexer. - -Hot markets work all the same as long as Solana is up. Cold accounts cannot be loaded into hot state until your indexer or RPC provider recovers. -Note that compression is cryptographically verifiable, so integrity and safety are not dependent on the indexer or any other external service beyond the onchain protocol. + +Yes. At swap time, call `get_multiple_account_interfaces` for the instruction's +accounts and check `is_cold()`. This adds a round-trip per swap but requires +no streaming setup. --- diff --git a/light-token/toolkits/for-streaming-mints.mdx b/light-token/toolkits/for-streaming-mints.mdx index 7046300c..9ac85a1d 100644 --- a/light-token/toolkits/for-streaming-mints.mdx +++ b/light-token/toolkits/for-streaming-mints.mdx @@ -1,39 +1,49 @@ --- -title: "Toolkit for Streaming Light Mint Accounts" -sidebarTitle: "For Streaming Mints" -description: "Guide to stream light-mints and metadata using Laserstream." -keywords: ["streaming mints for token issuers", "bulk token minting for protocols", "rent free token mints on solana"] +title: "Streaming Mint Accounts" +sidebarTitle: "Mint Accounts" +description: "Stream light-mint accounts and metadata via Laserstream." +keywords: ["streaming mint accounts solana", "laserstream light mint", "grpc mint streaming"] --- +import FindClosedAccounts from '/snippets/code-snippets/streaming/find-closed-accounts.mdx'; + | Event | Description | |:------|:------------| | **New mints** | Raw mint data | | **Mint updates** | Supply changes, authority changes | | **TokenMetadata** | Name, symbol, URI, additional_metadata | +| **Cold/hot transitions** | Mint compressed or decompressed | -Find devnet examples [here](https://github.com/Lightprotocol/examples-light-token/tree/main/toolkits/streaming-tokens). +This guide is for teams building custom data pipelines (aggregators, market makers). +If you just need account lookups, use [`get_account_interface`](/light-token/defi/routers) instead. -## Setup +## Architecture + +Light mints are Solana accounts owned by the Light Token Program. The streaming +setup requires two gRPC subscriptions: + +| Subscription | Detects | How | +|:-------------|:--------|:----| +| Account sub (`owner: cToken...`, `account_type == 1`) | Hot state + cold-to-hot | Pubkey cache lookup | +| Transaction sub (`account_include: cToken...`) | Hot-to-cold | Balance heuristic (`pre > 0, post == 0`) | -Light mints are Solana accounts owned by the Light Token Program. Subscribe to account updates to detect new mints and changes. +The account subscription delivers all state changes while mints are hot. +The transaction subscription is needed to detect mints going cold +(`CompressAndCloseMint` changes the owner to System Program, which the account +subscription no longer matches). + +## Setup ```toml Cargo.toml [dependencies] helius-laserstream = "0.1" tokio = { version = "1", features = ["full"] } futures = "0.3" -anyhow = "1" -dotenvy = "0.15" bs58 = "0.5" borsh = "0.10" - light-token-interface = "0.3.0" -light-compressed-account = { version = "0.9.0", features = ["std"] } - -# Pin blake3 to avoid constant_time_eq edition2024 issue -blake3 = "=1.5.5" ``` ```rust @@ -43,22 +53,19 @@ use helius_laserstream::grpc::subscribe_request_filter_accounts_filter::Filter; use helius_laserstream::grpc::subscribe_request_filter_accounts_filter_memcmp::Data; use helius_laserstream::grpc::{ SubscribeRequestFilterAccounts, SubscribeRequestFilterAccountsFilter, - SubscribeRequestFilterAccountsFilterMemcmp, + SubscribeRequestFilterAccountsFilterMemcmp, SubscribeRequestFilterTransactions, }; use helius_laserstream::{subscribe, LaserstreamConfig}; use light_token_interface::state::{ExtensionStruct, Mint}; const LIGHT_TOKEN_PROGRAM_ID: &str = "cTokenmWW8bLPjZEBAUgYy3zKxQZW6VKi7bqNFEVv3m"; - -/// Byte offset of account_type in the Mint account data. -/// BaseMint (82) + MintMetadata (67) + reserved (16) = 165 const ACCOUNT_TYPE_OFFSET: u64 = 165; ``` -### Connect to Laserstream +### Connect @@ -83,34 +90,44 @@ let config = LaserstreamConfig::new( + + + +### Subscribe + ```rust -// Subscribe to Solana accounts owned by the Light Token Program -// with account_type = 1 (Mint) at byte offset 165 -let request = helius_laserstream::grpc::SubscribeRequest { - accounts: [( - "light_mints".to_string(), - SubscribeRequestFilterAccounts { - owner: vec![LIGHT_TOKEN_PROGRAM_ID.to_string()], - filters: vec![SubscribeRequestFilterAccountsFilter { - filter: Some(Filter::Memcmp(SubscribeRequestFilterAccountsFilterMemcmp { - offset: ACCOUNT_TYPE_OFFSET, - data: Some(Data::Bytes(vec![1])), - })), - }], - nonempty_txn_signature: Some(true), - ..Default::default() - }, - )] - .into(), - ..Default::default() -}; +let mut request = helius_laserstream::grpc::SubscribeRequest::default(); + +// 1. Account sub: mint state tracking + cold-to-hot detection. +// account_type == 1 (Mint) at byte offset 165. +request.accounts.insert( + "light_mints".to_string(), + SubscribeRequestFilterAccounts { + owner: vec![LIGHT_TOKEN_PROGRAM_ID.to_string()], + filters: vec![SubscribeRequestFilterAccountsFilter { + filter: Some(Filter::Memcmp(SubscribeRequestFilterAccountsFilterMemcmp { + offset: ACCOUNT_TYPE_OFFSET, + data: Some(Data::Bytes(vec![1])), + })), + }], + nonempty_txn_signature: Some(true), + ..Default::default() + }, +); + +// 2. Transaction sub: hot-to-cold detection. +request.transactions.insert( + "light_token_txns".to_string(), + SubscribeRequestFilterTransactions { + vote: Some(false), + failed: Some(false), + account_include: vec![LIGHT_TOKEN_PROGRAM_ID.to_string()], + ..Default::default() + }, +); let (stream, _handle) = subscribe(config, request); tokio::pin!(stream); - -while let Some(update) = stream.next().await { - // Process account updates... -} ``` @@ -119,41 +136,23 @@ while let Some(update) = stream.next().await { ### Deserialize mint accounts ```rust -if let Some(helius_laserstream::grpc::subscribe_update::UpdateOneof::Account( - account_update, -)) = msg.update_oneof -{ +use helius_laserstream::grpc::subscribe_update::UpdateOneof; + +Some(UpdateOneof::Account(account_update)) => { if let Some(account_info) = account_update.account { - let pubkey = bs58::encode(&account_info.pubkey).into_string(); - let tx_sig = account_info - .txn_signature - .as_ref() - .map(|s| bs58::encode(s).into_string()) - .unwrap_or_default(); + let pubkey: [u8; 32] = account_info.pubkey.as_slice().try_into().unwrap(); match Mint::deserialize(&mut account_info.data.as_slice()) { Ok(mint) => { - println!("Mint: {}", pubkey); - println!(" tx: {}", tx_sig); - println!(" decimals: {}", mint.base.decimals); - println!(" supply: {}", mint.base.supply); - - if let Some(authority) = &mint.base.mint_authority { - println!( - " mint_authority: {}", - bs58::encode(authority.to_bytes()).into_string() - ); - } - - if let Some(authority) = &mint.base.freeze_authority { - println!( - " freeze_authority: {}", - bs58::encode(authority.to_bytes()).into_string() - ); - } + cold_cache.remove(&pubkey); // no longer cold + cache.insert(pubkey, mint); } Err(e) => { - eprintln!("Failed to deserialize mint {}: {}", pubkey, e); + eprintln!( + "Failed to deserialize mint {}: {}", + bs58::encode(&pubkey).into_string(), + e + ); } } } @@ -163,7 +162,14 @@ if let Some(helius_laserstream::grpc::subscribe_update::UpdateOneof::Account( -### Extract Token Metadata from mint extensions +### Detect mints going cold + + + + + + +### Extract TokenMetadata ```rust fn extract_metadata(mint: &Mint) -> Option<(String, String, String)> { @@ -181,14 +187,6 @@ fn extract_metadata(mint: &Mint) -> Option<(String, String, String)> { } ``` -```rust -if let Some((name, symbol, uri)) = extract_metadata(&mint) { - println!(" Name: {}", name); - println!(" Symbol: {}", symbol); - println!(" URI: {}", uri); -} -``` - @@ -208,7 +206,6 @@ pub struct Mint { pub extensions: Option>, } -/// SPL-compatible base mint structure #[repr(C)] pub struct BaseMint { pub mint_authority: Option, @@ -218,7 +215,6 @@ pub struct BaseMint { pub freeze_authority: Option, } -/// Light Protocol metadata for mints (67 bytes) #[repr(C)] pub struct MintMetadata { pub version: u8, @@ -248,10 +244,8 @@ pub struct AdditionalMetadata { } ``` -# Index Tokens - +Programs using the Light SDK expose standardized instruction discriminators for +`compress_accounts_idempotent` and `decompress_accounts_idempotent`. +Programs that don't implement these discriminators are not indexable with this approach. + + +## Architecture + +Two per-program gRPC subscriptions. Nothing else. + +| Subscription | Detects | How | +|:-------------|:--------|:----| +| Account sub (`owner: ProgramX`) | Hot state + cold-to-hot | Pubkey cache lookup | +| Transaction sub (`account_include: ProgramX`) | Hot-to-cold | 8-byte discriminator check + balance heuristic | + +**Why two?** Owner-filtered account subscriptions miss close events. When +`compress_accounts_idempotent` closes a PDA, the owner changes to System Program. +The account sub stops matching. The transaction sub catches it because the program +is still in the transaction's account list. + +## Setup + +```toml Cargo.toml +[dependencies] +helius-laserstream = "0.1" +tokio = { version = "1", features = ["full"] } +futures = "0.3" +bs58 = "0.5" +``` + +```rust +use futures::StreamExt; +use helius_laserstream::grpc::{ + SubscribeRequestFilterAccounts, SubscribeRequestFilterTransactions, +}; +use helius_laserstream::{subscribe, LaserstreamConfig}; +``` + + + + +### Connect + + + + +```rust +let config = LaserstreamConfig::new( + "https://laserstream-mainnet-ewr.helius-rpc.com".to_string(), + std::env::var("HELIUS_API_KEY")?, +); +``` + + + + +```rust +let config = LaserstreamConfig::new( + "https://laserstream-devnet-ewr.helius-rpc.com".to_string(), + std::env::var("HELIUS_API_KEY")?, +); +``` + + + + + + + +### Subscribe + +```rust +const PROGRAM_ID: &str = "YourProgramId1111111111111111111111111111111"; + +let mut request = helius_laserstream::grpc::SubscribeRequest::default(); + +// 1. Account sub: hot state tracking + cold-to-hot detection. +request.accounts.insert( + "program_pdas".to_string(), + SubscribeRequestFilterAccounts { + owner: vec![PROGRAM_ID.to_string()], + nonempty_txn_signature: Some(true), + ..Default::default() + }, +); + +// 2. Transaction sub: hot-to-cold detection. +// Catches all transactions involving the program. +// 99.999% are regular operations -- filtered out by an 8-byte discriminator check. +request.transactions.insert( + "program_txns".to_string(), + SubscribeRequestFilterTransactions { + vote: Some(false), + failed: Some(false), + account_include: vec![PROGRAM_ID.to_string()], + ..Default::default() + }, +); + +let (stream, _handle) = subscribe(config, request); +tokio::pin!(stream); +``` + + + + + +## Detecting transitions + +### Hot-to-cold + +The Light SDK generates a standardized `compress_accounts_idempotent` instruction on +every conformant program. Its discriminator is stable (`sha256("global:compress_accounts_idempotent")[..8]`): + +```rust +const COMPRESS_DISCRIMINATOR: [u8; 8] = [70, 236, 171, 120, 164, 93, 113, 181]; +``` + +For each transaction update, check outer and inner instructions for this discriminator. +If matched, use the balance heuristic to identify which PDAs were closed: + +```rust +use helius_laserstream::grpc::subscribe_update::UpdateOneof; + +Some(UpdateOneof::Transaction(tx_update)) => { + if let Some(ref tx_info) = tx_update.transaction { + if !has_compress_instruction(tx_info) { + return; // 99.999% of transactions exit here. + } + for pubkey in find_closed_accounts(tx_info) { + // Only process accounts we're tracking. + // This filters out unrelated accounts that went to 0 in the same tx. + if let Some(last_hot_state) = cache.remove(&pubkey) { + cold_cache.insert(pubkey, last_hot_state); + } + } + } +} +``` + +```rust +fn has_compress_instruction( + tx_info: &helius_laserstream::grpc::SubscribeUpdateTransactionInfo, +) -> bool { + let tx = match tx_info.transaction.as_ref() { + Some(t) => t, + None => return false, + }; + let meta = match tx_info.meta.as_ref() { + Some(m) => m, + None => return false, + }; + let msg = match tx.message.as_ref() { + Some(m) => m, + None => return false, + }; + + // Check outer instructions. + for ix in &msg.instructions { + if ix.data.len() >= 8 && ix.data[..8] == COMPRESS_DISCRIMINATOR { + return true; + } + } + // Check inner instructions (covers CPI-wrapped calls). + for inner in &meta.inner_instructions { + for ix in &inner.instructions { + if ix.data.len() >= 8 && ix.data[..8] == COMPRESS_DISCRIMINATOR { + return true; + } + } + } + false +} +``` + +```rust +fn find_closed_accounts( + tx_info: &helius_laserstream::grpc::SubscribeUpdateTransactionInfo, +) -> Vec<[u8; 32]> { + let meta = match &tx_info.meta { + Some(m) => m, + None => return vec![], + }; + let msg = match tx_info.transaction.as_ref().and_then(|t| t.message.as_ref()) { + Some(m) => m, + None => return vec![], + }; + + // Full account list: static keys + loaded ALT addresses. + let mut all_keys: Vec<&[u8]> = msg.account_keys.iter().map(|k| k.as_slice()).collect(); + all_keys.extend(meta.loaded_writable_addresses.iter().map(|k| k.as_slice())); + all_keys.extend(meta.loaded_readonly_addresses.iter().map(|k| k.as_slice())); + + let mut closed = Vec::new(); + for (i, key) in all_keys.iter().enumerate() { + if key.len() == 32 + && meta.pre_balances.get(i).copied().unwrap_or(0) > 0 + && meta.post_balances.get(i).copied().unwrap_or(1) == 0 + { + closed.push(<[u8; 32]>::try_from(*key).unwrap()); + } + } + closed +} +``` + +**Why this is tight:** +- Discriminator check → only inspect compress transactions (no false positives from unrelated txns). +- Balance heuristic → only flag accounts that actually closed (idempotent no-ops produce no balance changes). +- `cache.remove` filter → only process accounts you're tracking (ignores unrelated closures in the same tx). + +**Edge case:** If compress and decompress execute in the same slot for the same PDA, +the account sub may deliver the hot state before the tx sub delivers the cold event. +The cold handler would then override the correct hot state. Resolve with slot tracking: +record the slot when marking cold, ignore cold events older than the latest hot event. +In practice this doesn't happen -- the forester compresses idle accounts, decompression +is user-initiated, and both in the same slot is not a realistic scenario. + +### Cold-to-hot + +The account subscription delivers the re-created PDA. Match the pubkey against +your cold cache: + +```rust +Some(UpdateOneof::Account(account_update)) => { + if let Some(account) = account_update.account { + let pubkey: [u8; 32] = account.pubkey.as_slice().try_into().unwrap(); + + if cold_cache.remove(&pubkey).is_some() { + // Was cold, now hot. Parse with your program's deserializer. + cache.upsert_hot(pubkey, &account.data); + } else { + // New account or hot state update. + cache.upsert(pubkey, &account.data); + } + } +} +``` + +This works because PDA pubkeys are deterministic. Same seeds, same program, same +pubkey across hot/cold cycles. If the pubkey is in your cold cache, it was cold. Period. + +## Point queries + +`getAccountInfo` returns null for cold PDAs. Use `get_account_interface()` -- +it races hot and cold lookups: + +```rust +use light_client::rpc::{LightClient, LightClientConfig, Rpc}; + +let client = LightClient::new(config).await?; +let result = client.get_account_interface(&pda_pubkey, None).await?; + +if let Some(account) = result.value { + let data = account.data(); + if account.is_cold() { + // Compressed. data() returns full account bytes. + } +} +``` + +## Lifecycle + +``` + PDA created (hot) + | + | compress_accounts_idempotent (rent expired, forester) + v + PDA closed (cold) -- state in Merkle tree + | + | decompress_accounts_idempotent (user) + v + PDA re-created (hot) -- same pubkey +``` + +The compressed address `derive_address(pda_pubkey, tree, program_id)` is stable +across cycles. + + + +If you don't know which programs to index, subscribe to **Light System Program** +transactions instead. All compression events CPI through it. + +Use `light-event` to parse compression events from inner instructions: + +```rust +use light_event::parse::event_from_light_transaction; +use light_compressible::DECOMPRESSED_PDA_DISCRIMINATOR; +``` + +**Going cold**: output accounts with `discriminator != DECOMPRESSED_PDA_DISCRIMINATOR` +are real data being written to the Merkle tree. + +**Going hot**: output accounts with `discriminator == DECOMPRESSED_PDA_DISCRIMINATOR` +are shell placeholders. `data[..32]` contains the PDA pubkey. + +This approach requires borsh deserialization and CPI pattern matching. It's the +same pipeline used by the [Photon indexer](https://github.com/helius-labs/photon). + + + + +**TODO**: Consider exposing `compress_accounts_idempotent` / `decompress_accounts_idempotent` +discriminators via the `LightProgramInterface` trait so indexers can discover compression +instructions programmatically without hardcoding discriminator bytes. + + + + + + diff --git a/light-token/toolkits/for-streaming-tokens.mdx b/light-token/toolkits/for-streaming-tokens.mdx index d1e78f7b..fc0bd7c3 100644 --- a/light-token/toolkits/for-streaming-tokens.mdx +++ b/light-token/toolkits/for-streaming-tokens.mdx @@ -1,50 +1,260 @@ --- -title: "Toolkit to Index Light Token Accounts" -sidebarTitle: "For Indexing Tokens" -description: "Light token accounts follow the same layout as SPL-token accounts, so you can reuse your existing parsers." -keywords: ["streaming tokens for solana apps", "scalable token distribution on solana", "token streaming for developers"] +title: "Streaming Token Accounts" +sidebarTitle: "Token Accounts" +description: "Stream light-token accounts via Laserstream. Same base layout as SPL Token." +keywords: ["streaming token accounts solana", "laserstream light token", "grpc token streaming"] --- -import ActionCode from "/snippets/code-snippets/light-token/load-ata/action.mdx"; -import InstructionCode from "/snippets/code-snippets/light-token/load-ata/instruction.mdx"; +import FindClosedAccounts from '/snippets/code-snippets/streaming/find-closed-accounts.mdx'; -When a market becomes inactive, its token accounts and related PDAs will -be compressed automatically (cold storage). -The state is cryptographically preserved on the Solana ledger. -While compressed, pure on-chain lookups will return uninitialized. + +This guide is for teams building custom data pipelines (aggregators, market makers). +If you just need account lookups, use [`get_account_interface`](/light-token/defi/routers) instead. + -Your indexer should keep tracking, quoting, and routing markets even if the -on-chain account shows `is_initialized: false`, `is_compressed: true`. -To trade a cold market, the first client must prepend an -idempotent decompress "warm up" instruction. +## Architecture - - Find the source code - [here](https://github.com/Lightprotocol/light-protocol/blob/main/js/compressed-token/tests/e2e/load-ata-standard.test.ts). - +Light token accounts share the same base layout as SPL Token (165 bytes), so you can +use your existing parser. The streaming setup requires two gRPC subscriptions, both +targeting the Light Token Program: + +| Subscription | Detects | How | +|:-------------|:--------|:----| +| Account sub (`owner: cToken...`) | Hot state + cold-to-hot | Pubkey cache lookup | +| Transaction sub (`account_include: cToken...`) | Hot-to-cold | Balance heuristic (`pre > 0, post == 0`) | + +The account subscription delivers all state changes while accounts are hot. +The transaction subscription is needed to detect accounts going cold +(`compress_and_close` changes the owner to System Program, which the account +subscription no longer matches). + +## Parsing + +```rust +use spl_pod::bytemuck::pod_from_bytes; +use spl_token_2022_interface::pod::PodAccount; // works for SPL-token, SPL-token-2022, and Light-token + +let parsed: &PodAccount = pod_from_bytes(&data[..165])?; +``` + +For accounts with extensions, truncate to 165 bytes before parsing. + +## Streaming + +```toml Cargo.toml +[dependencies] +helius-laserstream = "0.1" +tokio = { version = "1", features = ["full"] } +futures = "0.3" +bs58 = "0.5" +borsh = "0.10" +light-token-interface = "0.3.0" +``` + +```rust +use futures::StreamExt; +use helius_laserstream::grpc::subscribe_request_filter_accounts_filter::Filter; +use helius_laserstream::grpc::subscribe_request_filter_accounts_filter_memcmp::Data; +use helius_laserstream::grpc::{ + SubscribeRequestFilterAccounts, SubscribeRequestFilterAccountsFilter, + SubscribeRequestFilterAccountsFilterMemcmp, SubscribeRequestFilterTransactions, +}; +use helius_laserstream::{subscribe, LaserstreamConfig}; + +const LIGHT_TOKEN_PROGRAM_ID: &str = "cTokenmWW8bLPjZEBAUgYy3zKxQZW6VKi7bqNFEVv3m"; +const TOKEN_ACCOUNT_SIZE: u64 = 165; +const ACCOUNT_TYPE_OFFSET: u64 = 165; +const ACCOUNT_TYPE_TOKEN: u8 = 2; +``` + -### Load compressed tokens (cold storage) to Associated Token Account (hot balance) +### Connect - - + + +```rust +let config = LaserstreamConfig::new( + "https://laserstream-mainnet-ewr.helius-rpc.com".to_string(), + std::env::var("HELIUS_API_KEY")?, +); +``` + - - + + +```rust +let config = LaserstreamConfig::new( + "https://laserstream-devnet-ewr.helius-rpc.com".to_string(), + std::env::var("HELIUS_API_KEY")?, +); +``` + + + +### Subscribe + +```rust +let mut request = helius_laserstream::grpc::SubscribeRequest::default(); + +// 1. Account sub: hot state tracking + cold-to-hot detection. +request.accounts.insert( + "light_tokens".to_string(), + SubscribeRequestFilterAccounts { + owner: vec![LIGHT_TOKEN_PROGRAM_ID.to_string()], + filters: vec![SubscribeRequestFilterAccountsFilter { + filter: Some(Filter::Datasize(TOKEN_ACCOUNT_SIZE)), + }], + nonempty_txn_signature: Some(true), + ..Default::default() + }, +); +request.accounts.insert( + "light_tokens_extended".to_string(), + SubscribeRequestFilterAccounts { + owner: vec![LIGHT_TOKEN_PROGRAM_ID.to_string()], + filters: vec![SubscribeRequestFilterAccountsFilter { + filter: Some(Filter::Memcmp(SubscribeRequestFilterAccountsFilterMemcmp { + offset: ACCOUNT_TYPE_OFFSET, + data: Some(Data::Bytes(vec![ACCOUNT_TYPE_TOKEN])), + })), + }], + nonempty_txn_signature: Some(true), + ..Default::default() + }, +); + +// 2. Transaction sub: hot-to-cold detection. +request.transactions.insert( + "light_token_txns".to_string(), + SubscribeRequestFilterTransactions { + vote: Some(false), + failed: Some(false), + account_include: vec![LIGHT_TOKEN_PROGRAM_ID.to_string()], + ..Default::default() + }, +); + +let (stream, _handle) = subscribe(config, request); +tokio::pin!(stream); +``` + + + -# Stream light-mint accounts +## Detecting transitions + +### Hot-to-cold + +For each transaction update, find accounts whose lamport balance dropped to zero. +The `cache.remove` call ensures only accounts you're already tracking are processed: + + + +### Cold-to-hot + +When a token account is decompressed, the account subscription delivers the +re-created account. Match its pubkey against `cold_cache`: + +```rust +Some(UpdateOneof::Account(account_update)) => { + if let Some(account) = account_update.account { + let pubkey: [u8; 32] = account.pubkey.as_slice().try_into().unwrap(); + let parsed: &PodAccount = pod_from_bytes(&account.data[..165])?; + + cold_cache.remove(&pubkey); // no longer cold + cache.insert(pubkey, *parsed); + } +} +``` + +## Point queries + +`getAccountInfo` returns null for cold accounts. `get_account_interface()` races +hot and cold lookups and returns raw account bytes that work with your standard SPL parser: + +```rust +use light_client::rpc::{LightClient, LightClientConfig, Rpc}; +use spl_pod::bytemuck::pod_from_bytes; +use spl_token_2022_interface::pod::PodAccount; + +let config = LightClientConfig::new( + "https://api.devnet.solana.com".to_string(), + Some("https://photon.helius.com?api-key=YOUR_KEY".to_string()), +); +let client = LightClient::new(config).await?; +let result = client.get_account_interface(&pubkey, None).await?; + +if let Some(account) = result.value { + let parsed: &PodAccount = pod_from_bytes(&account.data()[..165])?; + if account.is_cold() { + // Compressed -- still valid for routing. + } +} +``` + +## Data layout + +165 bytes base, identical to SPL Token Account. + +| Field | Offset | Size | +|:------|:-------|:-----| +| `mint` | 0 | 32 | +| `owner` | 32 | 32 | +| `amount` | 64 | 8 | +| `delegate` | 72 | 36 | +| `state` | 108 | 1 | +| `is_native` | 109 | 12 | +| `delegated_amount` | 121 | 8 | +| `close_authority` | 129 | 36 | +| `account_type` | 165 | 1 | + +`account_type = 2` at byte 165 indicates extensions follow (borsh-encoded `Option>`). + + + +These are not needed for indexing or trading. + +```rust +use borsh::BorshDeserialize; +use light_token_interface::state::{Token, ExtensionStruct}; + +let token = Token::deserialize(&mut data.as_slice())?; + +if let Some(exts) = &token.extensions { + for ext in exts { + if let ExtensionStruct::Compressible(info) = ext { + // info.compression_authority, info.rent_sponsor, info.last_claimed_slot + } + } +} +``` + +| Variant | Description | +|:--------|:------------| +| `TokenMetadata(TokenMetadata)` | Name, symbol, URI, additional metadata | +| `PausableAccount(PausableAccountExtension)` | Marker: mint is pausable (no data; pause state lives on mint) | +| `PermanentDelegateAccount(PermanentDelegateAccountExtension)` | Marker: mint has permanent delegate | +| `TransferFeeAccount(TransferFeeAccountExtension)` | Withheld fees from transfers | +| `TransferHookAccount(TransferHookAccountExtension)` | Marker: mint has transfer hook | +| `CompressedOnly(CompressedOnlyExtension)` | Compressed-only token (stores delegated amount) | +| `Compressible(CompressibleExtension)` | Compression config: authority, rent sponsor, timing | + +Source: [`light-token-interface`](https://github.com/Lightprotocol/light-protocol/tree/main/program-libs/token-interface/src/state/extensions) + + \ No newline at end of file +/> diff --git a/light-token/toolkits/overview.mdx b/light-token/toolkits/overview.mdx index 30e36eb4..d2841d47 100644 --- a/light-token/toolkits/overview.mdx +++ b/light-token/toolkits/overview.mdx @@ -25,22 +25,3 @@ keywords: ["token sdk for solana developers", "token infrastructure for solana a Allow users to display and swap light-tokens. - - Stream mint events from the network in real-time. - - - - Stream token events from the network in real-time. - diff --git a/snippets/code-samples/code-compare-snippets.jsx b/snippets/code-samples/code-compare-snippets.jsx index 6b1af600..6087a889 100644 --- a/snippets/code-samples/code-compare-snippets.jsx +++ b/snippets/code-samples/code-compare-snippets.jsx @@ -180,8 +180,8 @@ export const lightCreateMintRustCode = [ " params,", " mint_seed.pubkey(),", " payer.pubkey(),", - " address_tree.tree,", - " output_queue,", + " address_tree.tree,", + " output_queue,", ")", ".instruction()?;", ].join("\n"); diff --git a/snippets/code-snippets/streaming/find-closed-accounts.mdx b/snippets/code-snippets/streaming/find-closed-accounts.mdx new file mode 100644 index 00000000..50140fd0 --- /dev/null +++ b/snippets/code-snippets/streaming/find-closed-accounts.mdx @@ -0,0 +1,62 @@ +Two data structures: +- `cache: HashMap<[u8; 32], T>` -- hot account state (for quoting/routing) +- `cold_cache: HashMap<[u8; 32], AccountInterface>` -- cold accounts with `ColdContext` (for building load instructions) + +```rust +use helius_laserstream::grpc::subscribe_update::UpdateOneof; + +Some(UpdateOneof::Transaction(tx_update)) => { + if let Some(ref tx_info) = tx_update.transaction { + for pubkey in find_closed_accounts(tx_info) { + if cache.remove(&pubkey).is_some() { + // Async: fetch AccountInterface with ColdContext. + // Cold accounts are inactive, so this completes well + // before anyone tries to swap through them. + let rpc = rpc.clone(); + let cold_cache = cold_cache.clone(); + tokio::spawn(async move { + if let Ok(Some(iface)) = rpc.get_account_interface(&pubkey, None).await { + cold_cache.insert(pubkey, iface); + } + }); + } + } + } +} +``` + +```rust +fn find_closed_accounts( + tx_info: &helius_laserstream::grpc::SubscribeUpdateTransactionInfo, +) -> Vec<[u8; 32]> { + let meta = match &tx_info.meta { + Some(m) => m, + None => return vec![], + }; + let msg = match tx_info.transaction.as_ref().and_then(|t| t.message.as_ref()) { + Some(m) => m, + None => return vec![], + }; + + let mut all_keys: Vec<&[u8]> = msg.account_keys.iter().map(|k| k.as_slice()).collect(); + all_keys.extend(meta.loaded_writable_addresses.iter().map(|k| k.as_slice())); + all_keys.extend(meta.loaded_readonly_addresses.iter().map(|k| k.as_slice())); + + let mut closed = Vec::new(); + for (i, key) in all_keys.iter().enumerate() { + if key.len() == 32 + && meta.pre_balances.get(i).copied().unwrap_or(0) > 0 + && meta.post_balances.get(i).copied().unwrap_or(1) == 0 + { + closed.push(<[u8; 32]>::try_from(*key).unwrap()); + } + } + closed +} +``` + +`cache.remove` filters out unrelated closures in the same transaction. No discriminator +check is needed -- `compress_and_close` always drains lamports to zero. + +To build transactions that decompress cold accounts, see +[Router Integration](/light-token/defi/routers).