diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000000..4f153e416d --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,112 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +language: en-US +tone_instructions: "You are a principal engineer with natural teaching abilities. You detect issues and clearly explain why. Read the docs and CLAUDE.md files." +reviews: + profile: assertive + high_level_summary: true + #paths to ignore, customize for your stack + path_filters: + - "!node_modules/**" + - "!dist/**" + - "!target/**" + - "!.git/**" + - "program-libs/**" + - "programs/**" + - "sdk-libs/**" + - "prover/**" + - "forester/**" + - "docs/**" + - "*.md" + - "!LICENSE" + + path_instructions: + - path: "**/docs/**/*.md" + instructions: | + When reviewing batched Merkle tree documentation changes: + 1. **Critical**: Verify that all function signatures, struct definitions, and behavior described in the documentation accurately match the actual implementation in `**/src/` + 2. Cross-reference any mentioned function names, parameters, return types, and error conditions with the source code + 3. Check that code examples and usage patterns reflect the current API in the crate + 4. Validate that any referenced constants, enums, or type definitions exist and have correct values + 5. Ensure documentation describes the actual behavior, not outdated or planned behavior + 6. Flag any references to deprecated functions, renamed structs, or changed interfaces + 7. Verify that error handling and edge cases mentioned in docs match the implementation + 8. Check that performance characteristics and complexity claims are accurate + 9. Do you see any inconsistencies between the documentation and the code in either way? + 10. Do you see any weird patterns or anything that doesn't make sense in code or docs? + + # add linters and other tools, CodeRabbit will run and check these as part of its review process + # Pre-merge quality gates + pre_merge_checks: + docstrings: + mode: warning + threshold: 70 + + # Finishing touches for code quality + finishing_touches: + docstrings: + enabled: true + unit_tests: + enabled: true + + tools: + eslint: + enabled: true + ruff: + enabled: true + gitleaks: + enabled: true + clippy: + enabled: true + yamllint: + enabled: true + markdownlint: + enabled: true + shellcheck: + enabled: true + auto_review: + enabled: true + drafts: false + ignore_title_keywords: + - "wip" + - "draft" + - "temp" + - "test" + - "experimental" + ignore_usernames: + - "dependabot[bot]" + - "dependabot" + labels: + - "!skip-review" + - "!no-review" + - "!dependabot" + base_branches: + - "main" + - "release/*" + +chat: + auto_reply: true + art: false + +# Enhanced knowledge base configuration +knowledge_base: + opt_out: false + web_search: + enabled: true + learnings: + scope: global + issues: + scope: global + # Coding guidelines for Rust and TypeScript projects + code_guidelines: + enabled: true + filePatterns: + - "*/docs/**" + - "*/**/README.md" + - "program-libs/batched-merkle-tree/docs/**" + - "program-libs/account-checks/docs/**" + - "program-libs/compressible/docs/**" + - "*/**/CLAUDE.md" + - "DOCS.md" + +# Additional configuration for Light Protocol specific patterns +early_access: true diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..b64c170c7a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,199 @@ +# Light Protocol - AI Assistant Reference Guide + +## Repository Overview + +Light Protocol is the ZK Compression Protocol for Solana, enabling developers to create rent-free tokens and PDAs without sacrificing performance, security, or composability. The protocol uses zero-knowledge proofs to compress account state into Merkle trees, reducing storage costs while maintaining full Solana compatibility. + +**Core Technologies:** Rust, Solana, ZK circuits (Gnark), Poseidon hashing, batched Merkle trees +**Architecture:** On-chain programs + off-chain ZK provers + client SDKs + forester service, see light_paper_v0.1.0.pdf for details. +**Detailed docs:** See `CLAUDE.md` files in individual crates and `**/*/docs/` + +## Directory Structure + +``` +light-protocol/ +├── program-libs/ # Core Rust libraries (used in programs and sdk-libs) +│ ├── account-checks/ # Solana account validation (solana-program + pinocchio) +│ ├── aligned-sized/ # Macro to get the aligned size of rust structs +│ ├── array-map/ # Array-based map data structure +│ ├── batched-merkle-tree/ # Batched Merkle tree (Merkle tree updates with zk proofs) +│ ├── bloom-filter/ # Bloom filters +│ ├── compressed-account/ # Compressed account types and utilities +│ ├── compressible/ # Configuration for compressible token accounts +│ ├── concurrent-merkle-tree/ # Concurrent Merkle tree operations +│ ├── ctoken-types/ # Compressed token types and interfaces +│ ├── hash-set/ # Hash set implementation for Solana programs +│ ├── hasher/ # Poseidon hash implementation +│ ├── heap/ # Heap data structure for Solana programs +│ ├── indexed-array/ # Indexed array utilities +│ ├── indexed-merkle-tree/ # Indexed Merkle tree with address management +│ ├── macros/ # Procedural macros for Light Protocol +│ ├── merkle-tree-metadata/ # Metadata types for Merkle trees +│ ├── verifier/ # ZKP verification logic in Solana programs +│ ├── zero-copy/ # Zero-copy serialization for efficient account access +│ └── zero-copy-derive/ # Derive macros for zero-copy serialization +├── programs/ # Light Protocol Solana programs +│ ├── account-compression/ # Core compression program (owns Merkle tree accounts) +│ ├── system/ # Light system program (compressed account validation) +│ ├── compressed-token/ # Compressed token implementation (similar to SPL Token) +│ └── registry/ # Protocol configuration and forester access control +├── sdk-libs/ # Rust libraries used in custom programs and clients +│ ├── client/ # RPC client for querying compressed accounts +│ ├── sdk/ # Core SDK for Rust/Anchor programs +│ ├── sdk-pinocchio/ # Pinocchio-specific SDK implementation +│ ├── compressed-token-sdk/ # Compressed token client utilities +│ └── program-test/ # Fast local test environment (LiteSVM) +├── prover/ # ZK proof generation +│ ├── server/ # Go-based prover server and circuit implementation (Gnark) +│ └── client/ # Rust client for requesting proofs used in sdk/client and forester +├── forester/ # Server implementation to insert values from queue accounts into tree accounts. +├── cli/ # Light CLI tool (@lightprotocol/zk-compression-cli) +├── js/ # JavaScript/TypeScript libraries (@lightprotocol/stateless.js, @lightprotocol/compressed-token) +├── program-tests/ # Integration tests for programs +├── sdk-tests/ # Integration tests for sdk libraries in solana programs that integrate light protocol. +└── scripts/ # Build, test, and deployment scripts +``` + +### Program libs +- depend on other program-libs or external crates only +- unit test must not depend on light-test-utils, any test that requires light-test-utils must be in program-tests + +### Programs +- depend on program-libs and external crates only +- are used in program-tests, in sdk-libs only with devenv feature but should be avoided. +- unit test must not depend on light-test-utils, any test that requires light-test-utils must be in program-tests +- integration tests must be in program-tests +- light-test-utils contains assert functions to assert instruction success in integration tests. + +### SDK libs +- depend on program-libs, light-prover-client and external crates only +- must not depend on programs without devenv feature +- unit test must not depend on light-test-utils, any test that requires light-test-utils must be in sdk-tests +- integration tests must be in sdk-tests + +## Development Workflow + +### Build Commands +```bash +# Build entire monorepo (uses Nx) +./scripts/build.sh +``` + +### Testing Patterns + +**IMPORTANT**: Do not run `cargo test` at the monorepo root. Always target specific packages with `-p`. + +The repository has three main categories of tests: + +#### 1. Unit Tests (program-libs/) +Fast-running tests that don't require Solana runtime. Located in `program-libs/` crates. + +```bash +# Run with: cargo test -p +cargo test -p light-batched-merkle-tree +cargo test -p light-account-checks +cargo test -p light-hasher --all-features +cargo test -p light-compressed-account --all-features +# ... see individual crate docs for specific tests +``` + +**Environment variables used in CI:** +- `RUSTFLAGS="-D warnings"` (fail on warnings) +- `REDIS_URL=redis://localhost:6379` + +#### 2. Integration Tests (program-tests/) +Long-running integration tests that require Solana runtime (SBF). Located in `program-tests/`. + +**Why tests live here:** +- Most depend on `program-tests/utils` (light-test-utils) +- `batched-merkle-tree-test` is here because it depends on program-tests/utils +- `zero-copy-derive-test` is here only to avoid cyclic dependencies (it's NOT a long-running integration test) + +```bash +# Run with: cargo test-sbf -p +cargo test-sbf -p account-compression-test +cargo test-sbf -p system-test +cargo test-sbf -p compressed-token-test +# ... see program-tests/CLAUDE.md for complete list +``` + +**For detailed test commands, see:** `program-tests/CLAUDE.md` + +#### 3. SDK Tests (sdk-tests/) +SDK integration tests for various SDK implementations (native, Anchor, Pinocchio, token). + +```bash +# Run with: cargo test-sbf -p +cargo test-sbf -p sdk-native-test +cargo test-sbf -p sdk-anchor-test +cargo test-sbf -p sdk-token-test +# ... see sdk-tests/CLAUDE.md for complete list +``` + +**For detailed test commands, see:** `sdk-tests/CLAUDE.md` + +#### 4. JavaScript/TypeScript Tests +Version-specific tests (V1 and V2) for JS/TS packages. + +```bash +# Build and test with Nx +npx nx build @lightprotocol/zk-compression-cli +npx nx test @lightprotocol/stateless.js +npx nx test @lightprotocol/compressed-token +npx nx test @lightprotocol/zk-compression-cli +``` + +**Environment variables:** +- `LIGHT_PROTOCOL_VERSION=V1` or `V2` +- `REDIS_URL=redis://localhost:6379` +- `CI=true` + +**For available test scripts, see:** `package.json` files in `js/` directory + +#### 5. Go Prover Tests +Tests for the ZK proof generation server (Gnark circuits). + +```bash +# Run from prover/server directory +cd prover/server + +# Unit tests +go test ./prover/... -timeout 60m + +# Redis integration tests +TEST_REDIS_URL=redis://localhost:6379/15 go test -v -run TestRedis -timeout 10m + +# Integration tests +go test -run TestLightweight -timeout 15m +``` + +**For detailed test commands, see:** `prover/server/` directory + +#### 6. Forester Tests +End-to-end tests for the off-chain tree maintenance service. + +```bash +TEST_MODE=local cargo test --package forester e2e_test -- --nocapture +``` + +**Environment variables:** +- `RUST_BACKTRACE=1` +- `TEST_MODE=local` +- `REDIS_URL=redis://localhost:6379` + +#### 7. Linting +Format and clippy checks across the entire codebase. + +```bash +./scripts/lint.sh +``` + +**Note:** This requires nightly Rust toolchain and clippy components. + +### Test Organization Principles + +- **`program-libs/`**: Pure Rust libraries, unit tests with `cargo test` +- **`sdk-libs/`**: Pure Rust libraries, unit tests with `cargo test` +- **`program-tests/`**: Integration tests requiring Solana runtime, depend on `light-test-utils` +- **`sdk-tests/`**: SDK-specific integration tests +- **Special case**: `zero-copy-derive-test` in `program-tests/` only to break cyclic dependencies diff --git a/DOCS.md b/DOCS.md new file mode 100644 index 0000000000..8e9167006e --- /dev/null +++ b/DOCS.md @@ -0,0 +1,227 @@ +# Documentation Guidelines + +## 1. Crate Documentation Structure + +### 1.1 Root CLAUDE.md File +Every crate must have a `CLAUDE.md` file containing: + +**Required sections:** +- **Summary** - 2-5 bullet points describing crate functionality and purpose +- **Used in** - List of crates/programs that use this crate with one-liner descriptions +- **Navigation** - Instructions for navigating the documentation structure +- **High-level sections** - Major components organized by type: + - For Solana programs: Accounts, Instructions, Source Code Structure + - For libraries: Core Types, Public APIs, Utilities + - For SDKs: Client Functions, Instruction Builders, Types + - For test utilities: Test Helpers, Mock Accounts, Fixtures + +**Optional sections:** +- **Config Requirements** - For programs with configuration state +- **Security Considerations** - Critical security notes + +**Source Code Structure:** +Document the `src/` directory organization based on crate type: + +For programs: +- **Core Instructions** - Main program operations +- **Account State** - Account structures and data layouts +- **Shared Components** - Utilities and helpers + +For libraries: +- **Core Types** - Main data structures and traits + +For SDKs: +- **Client Functions** - Public API methods +- **Instruction Builders** - Transaction construction +- **Types** - Shared data structures + +For each module include: +- File/directory name +- Brief description of functionality +- Related features or dependencies + +Example: See `programs/compressed-token/program/CLAUDE.md` Source Code Structure section + +### 1.2 docs/ Directory +When documentation is extensive, create a `docs/` directory with: +- `CLAUDE.md` - Navigation guide for the docs folder +- Subdirectories for major sections (e.g., `instructions/`, `accounts/`) +- Individual `.md` files for detailed documentation + +## 2. Topic-Specific Documentation + +### 2.1 Account Documentation + +Every account must include: + +**Required fields:** +- **description** - What the account represents and its role in the program. Key concepts should be integrated here, NOT in a separate section +- **state layout** - Path to struct definition and field descriptions +- **associated instructions** - List of instructions that create/read/update/delete this account with discriminators + +**For Solana accounts:** +- **discriminator** - The 8-byte discriminator value if applicable +- **size** - Account data size in bytes +- **ownership** - Expected program owner +- **serialization** - Zero-copy (programs) and Borsh (clients) examples with code snippets + +**For PDAs:** +- **derivation** - Seeds used to derive the account (e.g., `[owner, program_id, mint]`) +- **bump** - Whether bump is stored or derived + +**For compressed accounts:** +- **version** - Versioning scheme for data format changes +- **hashing** - Hash method (Poseidon/SHA256) and discriminator encoding +- **data layout** - Compressed data structure + +**Optional fields:** +- **extensions** - Supported extension types and their layouts +- **security notes** - Critical validation requirements + +**Methods/Implementations:** +For accounts with associated methods, add a Methods section with: +- Group methods by purpose (Validation, Constructors, PDA Derivation, etc.) +- Use concise parameter names in signatures +- One-line action-oriented descriptions +- Include concrete values where helpful (constants, defaults) + +**Examples:** +- `programs/compressed-token/program/docs/ACCOUNTS.md` +- `program-libs/compressible/docs/CONFIG_ACCOUNT.md` + +### 2.2 Instruction Documentation + +Every instruction must include: + +**Required sections:** +- **discriminator** - The instruction discriminator value (e.g., `18`) +- **enum** - The instruction enum variant (e.g., `CTokenInstruction::CreateTokenAccount`) +- **path** - Path to instruction processor code in the program +- **description** - High-level overview including key concepts integrated within (NOT as separate section): + - What the instruction does + - Key state changes + - Usage scenarios + - Config validation requirements (if applicable) + - Any important notes or considerations (do NOT add a separate "Notes" section) + +- **instruction_data** - Path to instruction data struct with field descriptions + +- **Accounts** - Ordered list with for each account: + - Name and type + - Signer/writable requirements + - Validation checks performed + - Purpose in the instruction + +- **instruction logic and checks** - Step-by-step processing: + 1. Input validation + 2. State deserialization + 3. Business logic + 4. State updates + 5. CPIs (if any) + +- **Errors** - Comprehensive error list: + - Use format: `ErrorType::Variant` (error code: N) - Description + - Include actual numeric codes that appear in transaction logs + - Group related errors together for clarity + +**Optional sections:** +- **CPIs** - Cross-program invocations with target programs and data +- **Events** - Emitted events and their data +- **Security considerations** - Attack vectors and mitigations + +**Anti-patterns to avoid:** +- Generic performance optimization comments (e.g., "uses X for performance") +- Implementation details that don't affect usage +- Internal optimizations unless they have security implications + +**Examples:** +- `programs/compressed-token/program/docs/instructions/` + +### 2.3 Error Documentation + +Document all error codes that can be returned: + +**Error format in instruction docs:** +- Use bullet list format: `ErrorType::Variant` (error code: N) - Triggering condition +- For standard Solana ProgramError variants, use their actual codes: + - InvalidInstructionData = 3 + - InvalidAccountData = 4 + - InsufficientFunds = 6 + - MissingRequiredSignature = 8 + - NotEnoughAccountKeys = 11 + - InvalidSeeds = 14 + - (See Solana documentation for complete list) +- For custom error enums, show the u32 value that appears in transaction logs +- For errors from external crates, show them directly (e.g., `CompressibleError::InvalidState` not `ProgramError::Custom`) +- To find error codes for your program, create a test like: `programs/compressed-token/program/tests/print_error_codes.rs` + +**Required for custom error documentation:** +- **Error name** - The error variant name +- **Error code** - Numeric code that appears in logs +- **Description** - What the error indicates +- **Common causes** - Typical scenarios that trigger this error +- **Resolution** - How to fix or avoid the error +- **Location** - Where error enum is defined (e.g., `anchor_compressed_token::ErrorCode`, `light_ctoken_types::CTokenError`) + +**Common error crate locations in Light Protocol:** +- `anchor_compressed_token::ErrorCode` - Compressed token program errors +- `light_ctoken_types::CTokenError` - CToken type errors (18001-18037 range) +- `light_compressible::CompressibleError` - Compressible account errors (19001-19002 range) +- `light_account_checks::AccountError` - Account validation errors (12006-12021 range) +- `light_hasher::HasherError` - Hasher operation errors +- `light_compressed_account::CompressedAccountError` - Compressed account errors + +**Note:** All `light-*` crates implement automatic error conversions to `ProgramError::Custom(u32)` for both pinocchio and solana_program, allowing seamless error propagation across the codebase. + +**DON'Ts:** +- **DON'T document external crate errors in detail** - For errors from other crates (e.g., HasherError from light-hasher), only note the conversion exists and reference the source crate's documentation +- **DON'T include generic "best practices" sections** - Avoid preachy or overly general advice. Focus on specific, actionable information for each error +- **DON'T document ProgramError::Custom conversions** - Show the original error type directly with its code + +### 2.4 Serialization Documentation + +When documenting serialization: + +**Zero-copy (for programs):** +```rust +use light_zero_copy::traits::{ZeroCopyAt, ZeroCopyAtMut}; +let (data, _) = DataType::zero_copy_at(&bytes)?; +``` + +**Borsh (for clients):** +```rust +use borsh::BorshDeserialize; +let data = DataType::deserialize(&mut &bytes[..])?; +``` + +**Note:** Always specify which method to use in which context + +### 2.5 CPI Documentation + +For wrapper programs and CPI patterns: + +**Required elements:** +- **Target program** - Program being called +- **PDA signer** - Seeds and bump for CPI authority +- **Account mapping** - How accounts are passed through +- **Data passthrough** - Instruction data handling +- **Example code** - Complete CPI invocation + +## 3. Documentation Standards +- be concise and precise + +### 3.1 Path References +- Always use absolute paths from repository root +- Example: `program-libs/ctoken-types/src/state/solana_ctoken.rs` + +### 3.2 Code Examples +- Include working code snippets +- Show both correct usage and common mistakes +- Add inline comments explaining key points +- DON'T include print/log statements unless essential to the demonstrated functionality +- Focus on the core logic without debugging output + +### 3.3 Cross-References +- Link to related documentation +- Reference source files with specific line numbers when relevant +- Use relative links within the same crate diff --git a/program-libs/batched-merkle-tree/docs/CLAUDE.md b/program-libs/batched-merkle-tree/docs/CLAUDE.md new file mode 100644 index 0000000000..45205920cc --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/CLAUDE.md @@ -0,0 +1,148 @@ +# Batched Merkle Tree Library + +The `light-batched-merkle-tree` crate provides batched Merkle tree implementations for the Light Protocol account compression program. Instead of updating trees one leaf at a time, this library batches multiple insertions and updates them with zero-knowledge proofs (ZKPs), enabling efficient on-chain verification. Trees maintain a cyclic root history for validity proofs, and use bloom filters for non-inclusion proofs while batches are being filled. + +There are two tree types: **state trees** (two accounts tree account (input queue, tree metadata, roots), output queue account) for compressed accounts, and **address trees** (one account that contains the address queue, tree metadata, roots) for address registration. + +## Accounts + +### Account Types + +- **[TREE_ACCOUNT.md](TREE_ACCOUNT.md)** - BatchedMerkleTreeAccount (state and address trees) +- **[QUEUE_ACCOUNT.md](QUEUE_ACCOUNT.md)** - BatchedQueueAccount (output queue for state trees) + +### Overview + +The batched merkle tree library uses two main Solana account types: + +**BatchedMerkleTreeAccount:** +The main tree account storing tree roots, root history, and integrated input queue (bloom filters + hash chains for nullifiers or addresses). Used for both state trees and address trees. + +**Details:** [TREE_ACCOUNT.md](TREE_ACCOUNT.md) + +**BatchedQueueAccount:** +Output queue account for state trees that temporarily stores compressed account hashes before tree insertion. Enables immediate spending via proof-by-index. + +**Details:** [QUEUE_ACCOUNT.md](QUEUE_ACCOUNT.md) + +### State Trees vs Address Trees + +**State Trees (2 accounts):** +- `BatchedMerkleTreeAccount` with integrated input queue (for nullifiers) +- Separate `BatchedQueueAccount` for output operations (appending new compressed accounts) + +**Address Trees (1 account):** +- `BatchedMerkleTreeAccount` with integrated input queue (for addresses) +- No separate output queue + +## Operations + +### Initialization +- **[INITIALIZE_STATE_TREE.md](INITIALIZE_STATE_TREE.md)** - Create state tree + output queue pair (2 solana accounts) + - Source: [`src/initialize_state_tree.rs`](../src/initialize_state_tree.rs) + +- **[INITIALIZE_ADDRESS_TREE.md](INITIALIZE_ADDRESS_TREE.md)** - Create address tree with integrated queue (1 solana account) + - Source: [`src/initialize_address_tree.rs`](../src/initialize_address_tree.rs) + +### Queue Insertion Operations +- **[INSERT_OUTPUT_QUEUE.md](INSERT_OUTPUT_QUEUE.md)** - Insert compressed account hash into output queue (state tree) + - Source: [`src/queue.rs`](../src/queue.rs) - `BatchedQueueAccount::insert_into_current_batch` + +- **[INSERT_INPUT_QUEUE.md](INSERT_INPUT_QUEUE.md)** - Insert nullifiers into input queue (state tree) + - Source: [`src/merkle_tree.rs`](../src/merkle_tree.rs) - `BatchedMerkleTreeAccount::insert_nullifier_into_queue` + +- **[INSERT_ADDRESS_QUEUE.md](INSERT_ADDRESS_QUEUE.md)** - Insert addresses into address queue + - Source: [`src/merkle_tree.rs`](../src/merkle_tree.rs) - `BatchedMerkleTreeAccount::insert_address_into_queue` + +### Tree Update Operations +- **[UPDATE_FROM_OUTPUT_QUEUE.md](UPDATE_FROM_OUTPUT_QUEUE.md)** - Batch append with ZKP verification + - Source: [`src/merkle_tree.rs`](../src/merkle_tree.rs) - `BatchedMerkleTreeAccount::update_tree_from_output_queue_account` + +- **[UPDATE_FROM_INPUT_QUEUE.md](UPDATE_FROM_INPUT_QUEUE.md)** - Batch nullify/address updates with ZKP + - Source: [`src/merkle_tree.rs`](../src/merkle_tree.rs) - `update_tree_from_input_queue`, `update_tree_from_address_queue` + +## Key Concepts + +**Batching System:** Trees use 2 alternating batches. While one batch is being filled, the previous batch can be updated into the tree with a ZKP. + +**ZKP Batches:** Each batch is divided into smaller ZKP batches (`batch_size / zkp_batch_size`). Trees are updated incrementally by ZKP batch. + +**Bloom Filters:** Input queues (nullifier queue for state trees, address queue for address trees) use bloom filters for non-inclusion proofs. While a batch is filling, values are inserted into the bloom filter. After the batch is fully inserted into the tree and the next batch is 50% full, the bloom filter is zeroed to prevent false positives. Output queues do not use bloom filters. + +**Value Vecs:** Output queues store the actual compressed account hashes in value vectors (one per batch). Values can be accessed by leaf index even before they're inserted into the tree, enabling immediate spending of newly created compressed accounts. + +**Hash Chains:** Each ZKP batch has a hash chain storing the Poseidon hash of all values in that ZKP batch. These hash chains are used as public inputs for ZKP verification. + +**ZKP Verification:** Tree updates require zero-knowledge proofs proving the correctness of batch operations (old root + queue values → new root). Public inputs: old root, new root, hash chain (commitment to queue elements), and for appends: start_index (output queue) or next_index (address queue). + +**Root History:** Trees maintain a cyclic buffer of recent roots (default: 200). This enables validity proofs for recently spent compressed accounts even as the tree continues to update. + +**Rollover:** When a tree reaches capacity (2^height leaves), it must be replaced with a new tree. The rollover process creates a new tree and marks the old tree as rolled over, preserving the old tree's roots for ongoing validity proofs. A rollover can be performed once the rollover threshold is met (default: 95% full). + +**State vs Address Trees:** +- **State trees** have a separate `BatchedQueueAccount` for output operations (appending new leaves). Input operations (nullifying) use the integrated input queue on the tree account. +- **Address trees** have only an integrated input queue on the tree account - no separate output queue. + +## ZKP Verification + +Batch update operations require zero-knowledge proofs generated by the Light Protocol prover: + +- **Prover Server:** `prover/server/` - Generates ZK proofs for batch operations +- **Prover Client:** `prover/client/` - Client libraries for requesting proofs +- **Batch Update Circuits:** `prover/server/prover/v2/` - Circuit definitions for batch append, batch update (nullify), and batch address append operations + +## Dependencies + +This crate relies on several Light Protocol libraries: + +- **`light-bloom-filter`** - Bloom filter implementation for non-inclusion proofs +- **`light-hasher`** - Poseidon hash implementation for hash chains and tree operations +- **`light-verifier`** - ZKP verification for batch updates +- **`light-zero-copy`** - Zero-copy serialization for efficient account data access +- **`light-merkle-tree-metadata`** - Shared metadata structures for merkle trees +- **`light-compressed-account`** - Compressed account types and utilities +- **`light-account-checks`** - Account validation and discriminator checks + +## Testing and Reference Implementations + +**IndexedMerkleTree Reference Implementation:** +- **`light-merkle-tree-reference`** - Reference implementation of indexed Merkle trees (dev dependency) +- Source: `program-tests/merkle-tree/src/indexed.rs` - Canonical IndexedMerkleTree implementation used for generating constants and testing +- Used to generate constants like `ADDRESS_TREE_INIT_ROOT_40` in [`src/constants.rs`](../src/constants.rs) +- Initializes address trees with a single leaf: `H(0, HIGHEST_ADDRESS_PLUS_ONE)` + +## Source Code Structure + +**Core Account Types:** +- [`src/merkle_tree.rs`](../src/merkle_tree.rs) - `BatchedMerkleTreeAccount` (prove inclusion, nullify existing state, create new addresses) +- [`src/queue.rs`](../src/queue.rs) - `BatchedQueueAccount` (add new state (transaction outputs)) +- [`src/batch.rs`](../src/batch.rs) - `Batch` state machine (Fill → Full → Inserted) +- [`src/queue_batch_metadata.rs`](../src/queue_batch_metadata.rs) - `QueueBatches` metadata + +**Metadata and Configuration:** +- [`src/merkle_tree_metadata.rs`](../src/merkle_tree_metadata.rs) - `BatchedMerkleTreeMetadata` and account size calculations +- [`src/constants.rs`](../src/constants.rs) - Default configuration values + +**ZKP Infrastructure:** +- `prover/server/` - Prover server that generates ZK proofs for batch operations +- `prover/client/` - Client libraries for requesting proofs +- `prover/server/prover/v2/` - Batch update circuit definitions (append, nullify, address append) + +**Initialization:** +- [`src/initialize_state_tree.rs`](../src/initialize_state_tree.rs) - State tree initialization +- [`src/initialize_address_tree.rs`](../src/initialize_address_tree.rs) - Address tree initialization +- [`src/rollover_state_tree.rs`](../src/rollover_state_tree.rs) - State tree rollover +- [`src/rollover_address_tree.rs`](../src/rollover_address_tree.rs) - Address tree rollover + +**Errors:** +- [`src/errors.rs`](../src/errors.rs) - `BatchedMerkleTreeError` enum with all error types + +## Error Codes + +All errors are defined in [`src/errors.rs`](../src/errors.rs) and map to u32 error codes (14301-14312 range): +- `BatchNotReady` (14301) - Batch is not ready to be inserted +- `BatchAlreadyInserted` (14302) - Batch is already inserted +- `TreeIsFull` (14310) - Batched Merkle tree reached capacity +- `NonInclusionCheckFailed` (14311) - Value exists in bloom filter +- `BloomFilterNotZeroed` (14312) - Bloom filter must be zeroed before reuse +- Additional errors from underlying libraries (hasher, zero-copy, verifier, etc.) diff --git a/program-libs/batched-merkle-tree/docs/INITIALIZE_ADDRESS_TREE.md b/program-libs/batched-merkle-tree/docs/INITIALIZE_ADDRESS_TREE.md new file mode 100644 index 0000000000..22906c9c91 --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/INITIALIZE_ADDRESS_TREE.md @@ -0,0 +1,101 @@ +# Initialize Address Tree + +**path:** src/initialize_address_tree.rs + +**description:** +Initializes an address tree with integrated address queue. This operation creates **one Solana account**: + +**Address Merkle tree account** (`BatchedMerkleTreeAccount`) - Stores tree roots, root history, and integrated address queue (bloom filters + hash chains for addresses) +- Account layout `BatchedMerkleTreeAccount` defined in: src/merkle_tree.rs +- Metadata `BatchedMerkleTreeMetadata` defined in: src/merkle_tree_metadata.rs +- Tree type: `TreeType::AddressV2` (5) +- Initial root: `ADDRESS_TREE_INIT_ROOT_40` (pre-initialized with one indexed array element) +- Starts at next_index: 1 (index 0 contains sentinel element) +- Discriminator: b`BatchMta` `[66, 97, 116, 99, 104, 77, 116, 97]` (8 bytes) + +Address trees are used for address registration in the Light Protocol. New addresses are inserted into the address queue, then batch-updated into the tree with ZKPs. Unlike state trees, address trees have no separate output queue - the address queue is integrated into the tree account. + +**Instruction data:** +Instruction data is defined in: src/initialize_address_tree.rs + +`InitAddressTreeAccountsInstructionData` struct: + +**Tree configuration:** +- `height`: u32 - Tree height (default: 40, capacity = 2^40 leaves) +- `index`: u64 - Unchecked identifier of the address tree +- `root_history_capacity`: u32 - Size of root history cyclic buffer (default: 200) + +**Batch sizes:** +- `input_queue_batch_size`: u64 - Elements per address queue batch (default: 15,000) +- `input_queue_zkp_batch_size`: u64 - Elements per ZKP batch for address insertions (default: 250) + +**Validation:** Batch size must be divisible by ZKP batch size. Error: `BatchSizeNotDivisibleByZkpBatchSize` if validation fails. + +**Bloom filter configuration:** +- `bloom_filter_capacity`: u64 - Capacity in bits (default: batch_size * 8) +- `bloom_filter_num_iters`: u64 - Number of hash functions (default: 3 for test, 10 for production) + +**Validation:** +- Capacity must be divisible by 8 +- Capacity must be >= batch_size * 8 + +**Access control:** +- `program_owner`: Option - Optional program owning the tree +- `forester`: Option - Optional forester pubkey for non-Light foresters +- `owner`: Pubkey - Account owner (passed separately as function parameter, not in struct) + +**Rollover and fees:** +- `rollover_threshold`: Option - Percentage threshold for rollover (default: 95%) +- `network_fee`: Option - Network fee amount (default: 10,000 lamports) +- `close_threshold`: Option - Placeholder, unimplemented + +**Accounts:** +1. merkle_tree_account + - mutable + - Address Merkle tree account being initialized + - Must be rent-exempt for calculated size + +Note: No signer accounts required - account is expected to be pre-created with correct size + +**Instruction Logic and Checks:** + +1. **Calculate account size:** + - Merkle tree account size: Based on input_queue_batch_size, bloom_filter_capacity, input_queue_zkp_batch_size, root_history_capacity, and height + - Account size formula defined in: src/merkle_tree.rs (`get_merkle_tree_account_size`) + +2. **Verify rent exemption:** + - Check: merkle_tree_account balance >= minimum rent exemption for mt_account_size + - Uses: `check_account_balance_is_rent_exempt` from `light-account-checks` + - Store rent amount for rollover fee calculation + +3. **Initialize address Merkle tree account:** + - Set discriminator: `BatchMta` (8 bytes) + - Create tree metadata: + - tree_type: `TreeType::AddressV2` (5) + - associated_queue: Pubkey::default() (address trees have no separate queue) + - Calculate rollover_fee: Based on rollover_threshold, height, and merkle_tree_rent + - access_metadata: Set owner, program_owner, forester + - rollover_metadata: Set index, rollover_fee, rollover_threshold, network_fee, close_threshold, additional_bytes=None + - Initialize root history: Cyclic buffer with capacity=root_history_capacity, first entry = `ADDRESS_TREE_INIT_ROOT_40` + - Initialize integrated address queue: + - 2 bloom filter stores (one per batch), size = bloom_filter_capacity / 8 bytes each + - 2 hash chain stores (one per batch), capacity = (input_queue_batch_size / input_queue_zkp_batch_size) each + - Batch metadata with input_queue_batch_size and input_queue_zkp_batch_size + - Compute hashed_pubkey: Hash and truncate to 31 bytes for bn254 field compatibility + - next_index: 1 (starts at 1 because index 0 contains pre-initialized sentinel element) + - sequence_number: 0 (increments with each tree update) + - Rollover fee: Charged on address tree operations + +4. **Validate configurations:** + - root_history_capacity >= (input_queue_batch_size / input_queue_zkp_batch_size) + - Rationale: Ensures sufficient space for roots generated by address queue operations + - ZKP batch sizes must be 10 or 250 (only supported circuit sizes for address trees) + - height must be 40 (fixed for address trees) + +**Errors:** +- `AccountError::AccountNotRentExempt` (error code: 12011) - Account balance insufficient for rent exemption at calculated size +- `AccountError::InvalidAccountSize` (error code: 12006) - Account data length doesn't match calculated size requirements +- `BatchedMerkleTreeError::BatchSizeNotDivisibleByZkpBatchSize` (error code: 14305) - Batch size is not evenly divisible by ZKP batch size +- `MerkleTreeMetadataError::InvalidRolloverThreshold` - Rollover threshold value is invalid (must be percentage) +- `ZeroCopyError::Size` - Account size mismatch during zero-copy deserialization +- `BorshError` - Failed to serialize or deserialize metadata structures diff --git a/program-libs/batched-merkle-tree/docs/INITIALIZE_STATE_TREE.md b/program-libs/batched-merkle-tree/docs/INITIALIZE_STATE_TREE.md new file mode 100644 index 0000000000..fd68b63f3c --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/INITIALIZE_STATE_TREE.md @@ -0,0 +1,133 @@ +# Initialize State Tree + +**path:** src/initialize_state_tree.rs + +**description:** +Initializes a state tree with integrated input queue and separate output queue. This operation creates **two Solana accounts**: + +1. **State Merkle tree account** (`BatchedMerkleTreeAccount`) - Stores tree roots, root history, and integrated input queue (bloom filters + hash chains for nullifiers) + - Account layout `BatchedMerkleTreeAccount` defined in: src/merkle_tree.rs + - Metadata `BatchedMerkleTreeMetadata` defined in: src/merkle_tree_metadata.rs + - Tree type: `TreeType::StateV2` (4) + - Initial root: zero bytes for specified height + - Discriminator: b`BatchMta` `[66, 97, 116, 99, 104, 77, 116, 97]` (8 bytes) + +2. **Output queue account** (`BatchedQueueAccount`) - Temporarily stores compressed account hashes before tree insertion + - Account layout `BatchedQueueAccount` defined in: src/queue.rs + - Metadata `BatchedQueueMetadata` defined in: src/queue.rs + - Queue type: `QueueType::OutputStateV2` + - Enables immediate spending via proof-by-index + - Discriminator: b`queueacc` `[113, 117, 101, 117, 101, 97, 99, 99]` (8 bytes) + +State trees are used for compressed account lifecycle management. The output queue stores newly created compressed accounts, while the input queue (integrated into the tree account) tracks nullifiers when compressed accounts are spent. + +**Instruction data:** +Instruction data is defined in: src/initialize_state_tree.rs + +`InitStateTreeAccountsInstructionData` struct: + +**Tree configuration:** +- `height`: u32 - Tree height (default: 32, capacity = 2^32 leaves) +- `index`: u64 - Unchecked identifier of the state tree +- `root_history_capacity`: u32 - Size of root history cyclic buffer (default: 200) + +**Batch sizes:** +- `input_queue_batch_size`: u64 - Elements per input queue batch (default: 15,000) +- `output_queue_batch_size`: u64 - Elements per output queue batch (default: 15,000) +- `input_queue_zkp_batch_size`: u64 - Elements per ZKP batch for nullifications (default: 500) +- `output_queue_zkp_batch_size`: u64 - Elements per ZKP batch for appends (default: 500) + +**Validation:** Batch sizes must be divisible by their respective ZKP batch sizes. Error: `BatchSizeNotDivisibleByZkpBatchSize` if validation fails. + +**Bloom filter configuration (input queue only):** +- `bloom_filter_capacity`: u64 - Capacity in bits (default: batch_size * 8) +- `bloom_filter_num_iters`: u64 - Number of hash functions (default: 3 for test, 10 for production) + +**Validation:** +- Capacity must be divisible by 8 +- Capacity must be >= batch_size * 8 + +**Access control:** +- `program_owner`: Option - Optional program owning the tree +- `forester`: Option - Optional forester pubkey for non-Light foresters +- `owner`: Pubkey - Account owner (passed separately as function parameter, not in struct) + +**Rollover and fees:** +- `rollover_threshold`: Option - Percentage threshold for rollover (default: 95%) +- `network_fee`: Option - Network fee amount (default: 5,000 lamports) +- `additional_bytes`: u64 - CPI context account size for rollover (default: 20KB + 8 bytes) +- `close_threshold`: Option - Placeholder, unimplemented + +**Accounts:** +1. merkle_tree_account + - mutable + - State Merkle tree account being initialized + - Must be rent-exempt for calculated size + +2. queue_account + - mutable + - Output queue account being initialized + - Must be rent-exempt for calculated size + +Note: No signer accounts required - accounts are expected to be pre-created with correct sizes + +**Instruction Logic and Checks:** + +1. **Calculate account sizes:** + - Queue account size: Based on output_queue_batch_size and output_queue_zkp_batch_size + - Merkle tree account size: Based on input_queue_batch_size, bloom_filter_capacity, input_queue_zkp_batch_size, root_history_capacity, and height + - Account size formulas defined in: src/queue.rs (`get_output_queue_account_size`) and src/merkle_tree.rs (`get_merkle_tree_account_size`) + +2. **Verify rent exemption:** + - Check: queue_account balance >= minimum rent exemption for queue_account_size + - Check: merkle_tree_account balance >= minimum rent exemption for mt_account_size + - Uses: `check_account_balance_is_rent_exempt` from `light-account-checks` + - Store rent amounts for rollover fee calculation + +3. **Initialize output queue account:** + - Set discriminator: `queueacc` (8 bytes) + - Create queue metadata: + - queue_type: `QueueType::OutputStateV2` + - associated_merkle_tree: merkle_tree_account pubkey + - Calculate rollover_fee: Based on rollover_threshold, height, and total rent (merkle_tree_rent + additional_bytes_rent + queue_rent) + - access_metadata: Set owner, program_owner, forester + - rollover_metadata: Set index, rollover_fee, rollover_threshold, network_fee, close_threshold, additional_bytes + - Initialize batch metadata: + - 2 batches (alternating) + - batch_size: output_queue_batch_size + - zkp_batch_size: output_queue_zkp_batch_size + - bloom_filter_capacity: 0 (output queues don't use bloom filters) + - Initialize value vecs: 2 vectors (one per batch), capacity = batch_size each + - Initialize hash chain stores: 2 vectors (one per batch), capacity = (batch_size / zkp_batch_size) each + - Compute hashed pubkeys: Hash and truncate to 31 bytes for bn254 field compatibility + - tree_capacity: 2^height + - Rollover fee: Charged when creating output compressed accounts (insertion into output queue) + +4. **Initialize state Merkle tree account:** + - Set discriminator: `BatchMta` (8 bytes) + - Create tree metadata: + - tree_type: `TreeType::StateV2` (4) + - associated_queue: queue_account pubkey + - access_metadata: Set owner, program_owner, forester + - rollover_metadata: Set index, rollover_fee=0 (charged on queue insertion, not tree ops), rollover_threshold, network_fee, close_threshold, additional_bytes=None + - Initialize root history: Cyclic buffer with capacity=root_history_capacity, first entry = zero bytes for tree height + - Initialize integrated input queue: + - 2 bloom filter stores (one per batch), size = bloom_filter_capacity / 8 bytes each + - 2 hash chain stores (one per batch), capacity = (input_queue_batch_size / input_queue_zkp_batch_size) each + - Batch metadata with input_queue_batch_size and input_queue_zkp_batch_size + - Compute hashed_pubkey: Hash and truncate to 31 bytes for bn254 field compatibility + - next_index: 0 (starts empty) + - sequence_number: 0 (increments with each tree update) + +5. **Validate configurations:** + - root_history_capacity >= (output_queue_batch_size / output_queue_zkp_batch_size) + (input_queue_batch_size / input_queue_zkp_batch_size) + - Rationale: Ensures sufficient space for roots generated by both input and output operations + - ZKP batch sizes must be 10 or 500 (only supported circuit sizes) + +**Errors:** +- `AccountError::AccountNotRentExempt` (error code: 12011) - Account balance insufficient for rent exemption at calculated size +- `AccountError::InvalidAccountSize` (error code: 12006) - Account data length doesn't match calculated size requirements +- `BatchedMerkleTreeError::BatchSizeNotDivisibleByZkpBatchSize` (error code: 14305) - Batch size is not evenly divisible by ZKP batch size +- `MerkleTreeMetadataError::InvalidRolloverThreshold` - Rollover threshold value is invalid (must be percentage) +- `ZeroCopyError::Size` - Account size mismatch during zero-copy deserialization +- `BorshError` - Failed to serialize or deserialize metadata structures diff --git a/program-libs/batched-merkle-tree/docs/INSERT_ADDRESS_QUEUE.md b/program-libs/batched-merkle-tree/docs/INSERT_ADDRESS_QUEUE.md new file mode 100644 index 0000000000..5ee8cd5b25 --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/INSERT_ADDRESS_QUEUE.md @@ -0,0 +1,96 @@ +# Insert Into Address Queue + +**path:** src/merkle_tree.rs + +**description:** +Inserts an address into the address tree's integrated address queue when creating a new address for compressed accounts. The bloom filter prevents address reuse by checking that the address doesn't already exist in any batch's bloom filter. The address is stored in the hash chain and will be inserted into the tree by a batch update. The address queue stores addresses in both hash chains and bloom filters until the bloom filter is zeroed (which occurs after the batch is fully inserted into the tree AND the next batch reaches 50% capacity AND at least one batch update has occurred since batch completion). + +Key characteristics: +1. Inserts address into both bloom filter and hash chain (same value in both) +2. Checks non-inclusion: address must not exist in any bloom filter (prevents address reuse) +3. Checks tree capacity before insertion (address trees have fixed capacity) +4. Increments queue_next_index (address queue index; used by indexers as sequence number) + +The address queue uses a two-batch alternating system to enable zeroing out one bloom filter while the other is still being used for non-inclusion checks. + +**Operation:** +Method: `BatchedMerkleTreeAccount::insert_address_into_queue` + +**Parameters:** +- `address`: &[u8; 32] - Address to insert (32-byte hash) +- `current_slot`: &u64 - Current Solana slot number (sets batch start_slot on first insertion; used by indexers to track when batch started filling, not used for batch logic) + +**Accounts:** +This operation modifies a `BatchedMerkleTreeAccount`: +- Must be type `TreeType::AddressV2` +- Account layout defined in: src/merkle_tree.rs +- Account documentation: TREE_ACCOUNT.md +- Is initialized via `initialize_address_tree` +- Has integrated address queue (bloom filters + hash chains) + +**Operation Logic and Checks:** + +1. **Verify tree type:** + - Check: `tree_type == TreeType::AddressV2` + - Error if state tree (state trees don't have address queues) + +2. **Check tree capacity:** + - Call `check_queue_next_index_reached_tree_capacity()` + - Error if `queue_next_index >= tree_capacity` + - Ensures all queued addresses can be inserted into the tree + +3. **Insert into current batch:** + Calls `insert_into_current_queue_batch` helper which: + + a. **Check batch state (readiness):** + - If batch state is `Fill`: Ready for insertion, continue + - If batch state is `Inserted`: Batch was fully processed, needs clearing: + - Check bloom filter is zeroed; error if not + - Clear hash chain stores (reset all hash chains) + - Advance batch state to `Fill` + - Reset batch metadata (start_index, sequence_number, etc.) + - If batch state is `Full`: Error - batch not ready for insertion + + b. **Insert address into batch:** + - Call `current_batch.insert`: + - Insert address into bloom filter + - Check non-inclusion: address must not exist in any other bloom filter + - Update hash chain with address: `Poseidon(prev_hash_chain, address)` + - Store updated hash chain in hash chain store + - Increment batch's internal element counter + + c. **Check if batch is full:** + - If `num_inserted_elements == batch_size`: + - Transition batch state from `Fill` to `Full` + - Increment `currently_processing_batch_index` (switches to other batch) + - Update `pending_batch_index` (marks this batch ready for tree update) + +4. **Increment queue_next_index:** + - `queue_next_index += 1` + - Used as sequence number by indexers to track address order + +**Validations:** +- Tree must be address tree (enforced by tree type check) +- Tree must not be full: `queue_next_index < tree_capacity` (checked before insertion) +- Batch must be in `Fill` or `Inserted` state (enforced by `insert_into_current_queue_batch`) +- Bloom filter must be zeroed before reuse (enforced when clearing batch in `Inserted` state) +- Non-inclusion check: address must not exist in any bloom filter (prevents address reuse) + +**State Changes:** +- Bloom filter: Stores address for non-inclusion checks +- Hash chain store: Updates running Poseidon hash with address for ZKP batch +- Batch metadata: + - `num_inserted_elements`: Incremented + - `state`: May transition `Fill` → `Full` when batch fills + - `currently_processing_batch_index`: May switch to other batch + - `pending_batch_index`: Updated when batch becomes full +- Tree metadata: + - `queue_next_index`: Always incremented (sequence number for indexers) + +**Errors:** +- `MerkleTreeMetadataError::InvalidTreeType` - Tree is not an address tree (state trees don't support address insertion) +- `BatchedMerkleTreeError::TreeIsFull` (error code: 14310) - Address tree has reached capacity (queue_next_index >= tree_capacity) +- `BatchedMerkleTreeError::BatchNotReady` (error code: 14301) - Batch is in `Full` state and cannot accept insertions +- `BatchedMerkleTreeError::BloomFilterNotZeroed` (error code: 14312) - Attempting to reuse batch before bloom filter has been zeroed by forester +- `BatchedMerkleTreeError::NonInclusionCheckFailed` (error code: 14311) - Address already exists in bloom filter (address reuse attempt) +- `ZeroCopyError` - Failed to access bloom filter stores or hash chain stores diff --git a/program-libs/batched-merkle-tree/docs/INSERT_INPUT_QUEUE.md b/program-libs/batched-merkle-tree/docs/INSERT_INPUT_QUEUE.md new file mode 100644 index 0000000000..edf6a320d4 --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/INSERT_INPUT_QUEUE.md @@ -0,0 +1,97 @@ +# Insert Into Input Queue (Nullifier) + +**path:** src/merkle_tree.rs + +**description:** +Inserts a nullifier into the state tree's integrated input queue when spending a compressed account. The bloom filter prevents double-spending by checking that the compressed account hash doesn't already exist in any batch's bloom filter. The nullifier (which will replace the compressed account hash in the tree once inserted by a batch update) is stored in the hash chain. The input queue stores nullifiers in hash chains and compressed account hashes in bloom filters until the bloom filter is zeroed (which occurs after the batch is fully inserted into the tree AND the next batch reaches 50% capacity). + +Key characteristics: +1. Creates nullifier: `Hash(compressed_account_hash, leaf_index, tx_hash)` +2. Inserts nullifier into hash chain (value that will replace the leaf in the tree) +3. Inserts compressed_account_hash into bloom filter (for non-inclusion checks in subsequent transactions) +4. Checks non-inclusion: compressed_account_hash must not exist in any bloom filter (prevents double-spending) +5. Increments nullifier_next_index (nullifier queue index; used by indexers as sequence number) + +The input queue uses a two-batch alternating system to enable zeroing out one bloom filter while the other is still being used for non-inclusion checks. + +**Operation:** +Method: `BatchedMerkleTreeAccount::insert_nullifier_into_queue` + +**Parameters:** +- `compressed_account_hash`: &[u8; 32] - Hash of compressed account being nullified +- `leaf_index`: u64 - Index in the tree where the compressed account exists (note: although leaf_index is already inside the compressed_account_hash, it's added to the nullifier hash to expose it efficiently in the batch update ZKP) +- `tx_hash`: &[u8; 32] - Transaction hash; enables ZK proofs showing how a compressed account was spent and what other accounts exist in that transaction +- `current_slot`: &u64 - Current Solana slot number (sets batch start_slot on first insertion; used by indexers to track when batch started filling, not used for batch logic) + +**Accounts:** +This operation modifies a `BatchedMerkleTreeAccount`: +- Must be type `TreeType::StateV2` (we nullify state not addresses) +- Account layout defined in: src/merkle_tree.rs +- Account documentation: TREE_ACCOUNT.md +- Is initialized via `initialize_state_tree` +- Has integrated input queue (bloom filters + hash chains) + +**Operation Logic and Checks:** + +1. **Verify tree type:** + - Check: `tree_type == TreeType::StateV2` + - Error if address tree + +2. **Create nullifier:** + - Compute: `nullifier = Hash(compressed_account_hash, leaf_index, tx_hash)` + - Nullifier is transaction-specific (depends on tx_hash) + - Note, a nullifier could be any value other than the original compressed_account_hash. The only requirement is that post nullifier insertion we cannot prove inclusion of the original compressed_account_hash in the tree. + +3. **Insert into current batch:** + Calls `insert_into_current_queue_batch` helper which: + + a. **Check batch state (readiness):** + - If batch state is `Fill`: Ready for insertion, continue + - If batch state is `Inserted`: Batch was fully processed, needs clearing: + - Check bloom filter is zeroed; error if not + - Clear hash chain stores (reset all hash chains) + - Advance batch state to `Fill` + - Reset batch metadata (start_index, sequence_number, etc.) + - If batch state is `Full`: Error - batch not ready for insertion + + b. **Insert values into batch:** + - Call `current_batch.insert`: + - Insert compressed_account_hash into bloom filter (NOT the nullifier, since nullifier is tx-specific) + - Check non-inclusion: compressed_account_hash must not exist in any other bloom filter + - Update hash chain with nullifier: `Poseidon(prev_hash_chain, nullifier)` + - Store updated hash chain in hash chain store + - Increment batch's internal element counter + + c. **Check if batch is full:** + - If `num_inserted_elements == batch_size`: + - Transition batch state from `Fill` to `Full` + - Increment `currently_processing_batch_index` (switches to other batch) + - Update `pending_batch_index` (marks this batch ready for tree update) + +4. **Increment nullifier_next_index:** + - `nullifier_next_index += 1` + - Used as sequence number by indexers to track nullifier order + +**Validations:** +- Tree must be state tree (enforced by tree type check) +- Batch must be in `Fill` or `Inserted` state (enforced by `insert_into_current_queue_batch`) +- Bloom filter must be zeroed before reuse (enforced when clearing batch in `Inserted` state) +- Non-inclusion check: compressed_account_hash must not exist in any bloom filter (prevents double-spending) + +**State Changes:** +- Bloom filter: Stores compressed_account_hash for non-inclusion checks +- Hash chain store: Updates running Poseidon hash with nullifier for ZKP batch +- Batch metadata: + - `num_inserted_elements`: Incremented + - `state`: May transition `Fill` → `Full` when batch fills + - `currently_processing_batch_index`: May switch to other batch + - `pending_batch_index`: Updated when batch becomes full +- Tree metadata: + - `nullifier_next_index`: Always incremented (sequence number for indexers) + +**Errors:** +- `MerkleTreeMetadataError::InvalidTreeType` - Tree is not a state tree (address trees don't support nullifiers) +- `BatchedMerkleTreeError::BatchNotReady` (error code: 14301) - Batch is in `Full` state and cannot accept insertions +- `BatchedMerkleTreeError::BloomFilterNotZeroed` (error code: 14312) - Attempting to reuse batch before bloom filter has been zeroed by forester +- `BatchedMerkleTreeError::NonInclusionCheckFailed` (error code: 14311) - compressed_account_hash already exists in bloom filter (double-spend attempt) +- `ZeroCopyError` - Failed to access bloom filter stores or hash chain stores diff --git a/program-libs/batched-merkle-tree/docs/INSERT_OUTPUT_QUEUE.md b/program-libs/batched-merkle-tree/docs/INSERT_OUTPUT_QUEUE.md new file mode 100644 index 0000000000..1e10a3d582 --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/INSERT_OUTPUT_QUEUE.md @@ -0,0 +1,90 @@ +# Insert Into Output Queue + +**path:** src/queue.rs + +**description:** +Inserts a compressed account hash into the output queue's currently processing batch. Output queues store compressed account hashes until the batch is zeroed (which occurs after the batch is fully inserted into the tree AND the next batch reaches 50% capacity). + +Key characteristics: +1. Inserts values into value vec (for immediate spending via proof-by-index) +2. Updates hash chain (for ZKP verification) +3. Automatically transitions batches when full (Fill → Full state when num_inserted_elements reaches batch_size) +4. Assigns leaf index at insertion (increments next_index; tree insertion order is determined at queue insertion) +5. No bloom filters (only input queues use bloom filters) + +The output queue uses a two-batch alternating system. The alternating batch system is not strictly necessary for output queues (no bloom filters to zero out), but is used to unify input and output queue code. + +Output queues enable **immediate spending**: Values can be spent via proof-by-index before tree insertion. Unlike input queues that only store bloom filters, output queues store actual values in value vecs for proof-by-index. Hash chains are used as public inputs when verifying the ZKP that appends this batch to the tree. + +**Operation:** +Method: `BatchedQueueAccount::insert_into_current_batch` + +**Parameters:** +- `hash_chain_value`: &[u8; 32] - Compressed account hash to insert +- `current_slot`: &u64 - Current Solana slot number (sets batch start_slot on first insertion; used by indexers to track when batch started filling, not used for batch logic) + +**Accounts:** +This operation modifies a `BatchedQueueAccount`: +- Must be type `QueueType::OutputStateV2` +- Account layout defined in: src/queue.rs +- Must have been initialized via `initialize_state_tree` +- Associated with a state Merkle tree + +**Operation Logic and Checks:** + +1. **Get current insertion index:** + - Read `batch_metadata.next_index` to determine leaf index for this value + - This index is used for proof-by-index when spending compressed accounts + - Assigned at insertion time and determines the leaf position in the tree + +2. **Insert into current batch:** + Calls `insert_into_current_queue_batch` helper which: + + a. **Check batch state (readiness):** + - If batch state is `Fill`: Ready for insertion, continue + - If batch state is `Inserted`: Batch was fully processed, needs clearing: + - Clear value vec (reset all values to zero) + - Clear hash chain stores (reset all hash chains) + - Advance batch state to `Fill` + - Reset batch metadata (start_index, sequence_number, etc.) + - If batch state is `Full`: Error - batch not ready for insertion + + b. **Insert value into batch:** + - Call `current_batch.store_and_hash_value`: + - Store hash_chain_value in value vec at next position + - Update hash chain: + - Get current ZKP batch index + - Hash: `Poseidon(prev_hash_chain, hash_chain_value)` + - Store updated hash chain in hash chain store + - Increment batch's internal element counter + + c. **Check if batch is full:** + - If `num_inserted_elements == batch_size`: + - Transition batch state from `Fill` to `Full` + - Increment `currently_processing_batch_index` (switches to other batch) + - Update `pending_batch_index` (marks this batch ready for tree update) + +3. **Increment queue next_index:** + - `batch_metadata.next_index += 1` + - The assigned leaf index in the tree (tree insertion order is determined at queue insertion) + +**Validations:** +- Batch must be in `Fill` or `Inserted` state (enforced by `insert_into_current_queue_batch`) +- Tree must not be full: `next_index < tree_capacity` (checked by caller before insertion) + +**State Changes:** +- Value vec: Stores compressed account hash at index position +- Hash chain store: Updates running Poseidon hash for ZKP batch +- Batch metadata: + - `num_inserted_elements`: Incremented + - `state`: May transition `Fill` → `Full` when batch fills + - `currently_processing_batch_index`: May switch to other batch + - `pending_batch_index`: Updated when batch becomes full +- Queue metadata: + - `next_index`: Always incremented (leaf index for this value) + +**Errors:** +- `BatchedMerkleTreeError::TreeIsFull` (error code: 14310) - Output queue has reached tree capacity (next_index >= tree_capacity) +- `BatchedMerkleTreeError::BatchNotReady` (error code: 14301) - Batch is in `Full` state and cannot accept insertions +- `BatchedMerkleTreeError::BloomFilterNotZeroed` (error code: 14312) - N/A for output queues (no bloom filters) +- `ZeroCopyError` - Failed to access value vec or hash chain stores diff --git a/program-libs/batched-merkle-tree/docs/QUEUE_ACCOUNT.md b/program-libs/batched-merkle-tree/docs/QUEUE_ACCOUNT.md new file mode 100644 index 0000000000..dfd9285fdf --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/QUEUE_ACCOUNT.md @@ -0,0 +1,100 @@ +# BatchedQueueAccount + +**Description:** +Output queue account for state trees that temporarily stores compressed account hashes. Enables immediate spending of newly created compressed accounts via proof-by-index. + +**Note:** In the current implementation, `BatchedQueueAccount` is always an output queue (type `OutputStateV2`). Input queues are integrated into the `BatchedMerkleTreeAccount`. + +**Discriminator:** b`queueacc` `[113, 117, 101, 117, 101, 97, 99, 99]` (8 bytes) + +**Path:** +- Struct: `src/queue.rs` - `BatchedQueueAccount` +- Metadata: `src/queue.rs` - `BatchedQueueMetadata` + +## Components + +### 1. Metadata (`BatchedQueueMetadata`) +- Queue metadata (queue type, associated merkle tree) +- Batch metadata (`QueueBatches`): + - Batch sizes (`batch_size`, `zkp_batch_size`) + - `currently_processing_batch_index`: Index of batch accepting new insertions (Fill state) + - `pending_batch_index`: Index of batch ready for ZKP processing and tree insertion (Full or being incrementally inserted) + - Two `Batch` structures tracking state and progress + - **Note:** These indices can differ, enabling parallel insertion while tree updates from the previous batch are being verified +- Tree capacity +- Hashed merkle tree pubkey +- Hashed queue pubkey + +### 2. Value Vecs (`[ZeroCopyVecU64<[u8; 32]>; 2]`) +- Two value vectors, one per batch +- Stores the actual compressed account hashes +- Values accessible by leaf index even before tree insertion +- Enables proof-by-index for immediate spending + +### 3. Hash Chain Stores (`[ZeroCopyVecU64<[u8; 32]>; 2]`) +- Two hash chain vectors, one per batch +- Each batch has `batch_size / zkp_batch_size` hash chains +- Each hash chain stores Poseidon hash of all values in that ZKP batch +- Used as public inputs for batch append ZKP verification + +**Note:** Output queues do NOT have bloom filters (only input queues use bloom filters). + +## Serialization + +All deserialization is zero-copy. + +**In Solana programs:** +```rust +use light_batched_merkle_tree::queue::BatchedQueueAccount; +use light_account_checks::AccountInfoTrait; + +// Deserialize output queue +let queue = BatchedQueueAccount::output_from_account_info(account_info)?; +``` + +**In client code:** +```rust +use light_batched_merkle_tree::queue::BatchedQueueAccount; + +// Deserialize output queue +let queue = BatchedQueueAccount::output_from_bytes(&mut account_data)?; +``` + +## Account Validation + +**`output_from_account_info` checks:** +1. Account owned by Light account compression program (`check_owner` using `light-account-checks`) +2. Account discriminator is `queueacc` (`check_discriminator` using `light-account-checks`) +3. Queue type is `OUTPUT_STATE_QUEUE_TYPE_V2` + +**`output_from_bytes` checks (client only):** +1. Account discriminator is `queueacc` +2. Queue type is `OUTPUT_STATE_QUEUE_TYPE_V2` + +**Error codes:** +- `AccountError::AccountOwnedByWrongProgram` (12012) - Account not owned by compression program +- `AccountError::InvalidAccountSize` (12006) - Account size less than 8 bytes +- `AccountError::InvalidDiscriminator` (12007) - Discriminator mismatch +- `MerkleTreeMetadataError::InvalidQueueType` - Queue type mismatch + +## Associated Operations + +- [INITIALIZE_STATE_TREE.md](INITIALIZE_STATE_TREE.md) - Create output queue with state tree +- [INSERT_OUTPUT_QUEUE.md](INSERT_OUTPUT_QUEUE.md) - Insert compressed account hashes +- [UPDATE_FROM_OUTPUT_QUEUE.md](UPDATE_FROM_OUTPUT_QUEUE.md) - Update tree from output queue with ZKP + +## Supporting Structures + +### BatchedQueueMetadata + +**Description:** +Metadata for a batched queue account (output queues only). + +**Path:** `src/queue.rs` + +**Key Fields:** +- `metadata`: Base `QueueMetadata` (queue type, associated merkle tree) +- `batch_metadata`: `QueueBatches` structure +- `tree_capacity`: Associated tree's capacity (2^height). Checked on insertion to prevent overflow +- `hashed_merkle_tree_pubkey`: Pre-hashed tree pubkey (31 bytes + 1 padding). Pubkeys are hashed and truncated to 31 bytes (248 bits) to fit within bn254 field size requirements for Poseidon hashing in ZK circuits +- `hashed_queue_pubkey`: Pre-hashed queue pubkey (31 bytes + 1 padding). Same truncation for bn254 field compatibility diff --git a/program-libs/batched-merkle-tree/docs/TREE_ACCOUNT.md b/program-libs/batched-merkle-tree/docs/TREE_ACCOUNT.md new file mode 100644 index 0000000000..74d8fcb3f6 --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/TREE_ACCOUNT.md @@ -0,0 +1,193 @@ +# BatchedMerkleTreeAccount + +**Description:** +The main Merkle tree account that stores tree roots, root history, and integrated input queue (bloom filters + hash chains for nullifiers or addresses). Used for both state trees and address trees. + +**Discriminator:** b`BatchMta` `[66, 97, 116, 99, 104, 77, 116, 97]` (8 bytes) + +**Path:** +- Struct: `src/merkle_tree.rs` - `BatchedMerkleTreeAccount` +- Metadata: `src/merkle_tree_metadata.rs` - `BatchedMerkleTreeMetadata` + +## Components + +### 1. Metadata (`BatchedMerkleTreeMetadata`) +- Tree type: `TreeType::StateV2` or `TreeType::AddressV2` +- Tree height and capacity (2^height leaves) +- Sequence number (increments with each batched tree update (not input or output queue insertions)) +- Next index (next available leaf index) +- Nullifier next index (for state trees, address/nullifier queue index) +- Root history capacity +- Queue batch metadata +- Hashed pubkey (31 bytes for bn254 field compatibility) + +### 2. Root History (`ZeroCopyCyclicVecU64<[u8; 32]>`) +- Cyclic buffer storing recent tree roots +- Default capacity: 200 roots +- Latest root accessed via `root_history.last()` +- Validity proofs pick root by index from root history + since proofs need a static root value to verify against. + +### 3. Bloom Filter Stores (`[&mut [u8]; 2]`) +- Two bloom filters, one per batch +- Used only for input queues (nullifiers for state trees, addresses for address trees) +- Ensures no duplicate insertions in the queue. +- Zeroed after batch is fully inserted and next batch is 50% full and at least one batch update occured since batch completion. + +### 4. Hash Chain Stores (`[ZeroCopyVecU64<[u8; 32]>; 2]`) +- Two hash chain vectors, one per batch (length = `batch_size / zkp_batch_size`) +- Each hash chain stores Poseidon hash of all values in that ZKP batch +- Used as public inputs for ZKP verification + +## Tree Type Variants + +### State Tree +- Tree type: `STATE_MERKLE_TREE_TYPE_V2` +- Has separate `BatchedQueueAccount` for output operations (appending compressed accounts) +- Uses integrated input queue for nullifier operations +- Initial root: zero bytes root for specified height + +### Address Tree +- Tree type: `ADDRESS_MERKLE_TREE_TYPE_V2` +- No separate output queue (only integrated input queue for address insertions) +- Initial root: `ADDRESS_TREE_INIT_ROOT_40` (hardcoded for height 40) +- Starts with next_index = 1 (pre-initialized with one element at index 0) + +## Serialization + +All deserialization is zero-copy. + +**In Solana programs:** +```rust +use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeAccount; +use light_account_checks::AccountInfoTrait; + +// Deserialize state tree +let tree = BatchedMerkleTreeAccount::state_from_account_info(account_info)?; + +// Deserialize address tree +let tree = BatchedMerkleTreeAccount::address_from_account_info(account_info)?; + +// Access root by index +let root = tree.get_root_by_index(index)?; +``` + +**In client code:** +```rust +use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeAccount; + +// Deserialize state tree +let tree = BatchedMerkleTreeAccount::state_from_bytes(&mut account_data, &pubkey)?; + +// Deserialize address tree +let tree = BatchedMerkleTreeAccount::address_from_bytes(&mut account_data, &pubkey)?; +``` + +## Account Validation + +**`state_from_account_info` checks:** +1. Account owned by Light account compression program (`check_owner` using `light-account-checks`) +2. Account discriminator is `BatchMta` (`check_discriminator` using `light-account-checks`) +3. Tree type is `STATE_MERKLE_TREE_TYPE_V2` (4) + +**`address_from_account_info` checks:** +1. Account owned by Light account compression program (`check_owner` using `light-account-checks`) +2. Account discriminator is `BatchMta` (`check_discriminator` using `light-account-checks`) +3. Tree type is `ADDRESS_MERKLE_TREE_TYPE_V2` (5) + +**`state_from_bytes` checks (client only):** +1. Account discriminator is `BatchMta` +2. Tree type is `STATE_MERKLE_TREE_TYPE_V2` (4) + +**`address_from_bytes` checks (client only):** +1. Account discriminator is `BatchMta` +2. Tree type is `ADDRESS_MERKLE_TREE_TYPE_V2` (5) + +**Error codes:** +- `AccountError::AccountOwnedByWrongProgram` (12012) - Account not owned by compression program +- `AccountError::InvalidAccountSize` (12006) - Account size less than 8 bytes +- `AccountError::InvalidDiscriminator` (12007) - Discriminator mismatch +- `MerkleTreeMetadataError::InvalidTreeType` - Tree type mismatch (state vs address) + +## Associated Operations + +- [INITIALIZE_STATE_TREE.md](INITIALIZE_STATE_TREE.md) - Create state tree +- [INITIALIZE_ADDRESS_TREE.md](INITIALIZE_ADDRESS_TREE.md) - Create address tree +- [INSERT_INPUT_QUEUE.md](INSERT_INPUT_QUEUE.md) - Insert nullifiers (state trees) +- [INSERT_ADDRESS_QUEUE.md](INSERT_ADDRESS_QUEUE.md) - Insert addresses (address trees) +- [UPDATE_FROM_INPUT_QUEUE.md](UPDATE_FROM_INPUT_QUEUE.md) - Update tree from input queue with ZKP + +## Supporting Structures + +### Batch + +**Description:** +State machine tracking the lifecycle of a single batch from filling to insertion. + +**Path:** `src/batch.rs` + +**States:** +- **Fill** (0) - Batch is accepting new insertions. ZKP processing can begin as soon as individual ZKP batches are complete (when `num_full_zkp_batches > 0`) +- **Full** (2) - All ZKP batches are complete (`num_full_zkp_batches == batch_size / zkp_batch_size`). No more insertions accepted +- **Inserted** (1) - All ZKP batches have been inserted into the tree + +**State Transitions:** +- Fill → Full: When all ZKP batches are complete (`num_full_zkp_batches == batch_size / zkp_batch_size`) +- Full → Inserted: When all ZKP batches are inserted into tree (`num_inserted_zkp_batches == num_full_zkp_batches`) +- Inserted → Fill: When batch is reset for reuse (after bloom filter zeroing) + +**Key Insight:** ZKP processing happens incrementally. A batch doesn't need to be in Full state for ZKP processing to begin - individual ZKP batches can be processed as soon as they're complete, even while the overall batch is still in Fill state. + +**Key Fields:** +- `num_inserted`: Number of elements inserted in the current batch +- `num_full_zkp_batches`: Number of ZKP batches ready for insertion +- `num_inserted_zkp_batches`: Number of ZKP batches already inserted into tree +- `sequence_number`: Threshold value set at batch insertion (`tree_seq + root_history_capacity`). Used to detect if sufficient tree updates have occurred since batch insertion to overwrite the last root that was inserted with this batch. When clearing bloom filter, overlapping roots in history must also be zeroed to prevent inclusion proofs of nullified values +- `root_index`: Root index at batch insertion. Identifies which roots in history could prove inclusion of values from this batch's bloom filter. These roots are zeroed when clearing the bloom filter +- `start_index`: Starting leaf index for this batch +- `start_slot`: Slot of first insertion (for indexer reindexing) +- `bloom_filter_is_zeroed`: Whether bloom filter has been zeroed + +### QueueBatches + +**Description:** +Metadata structure managing the 2-batch system for queues. + +**Path:** `src/queue_batch_metadata.rs` + +**Key Fields:** +- `num_batches`: Always 2 (alternating batches) +- `batch_size`: Number of elements in a full batch +- `zkp_batch_size`: Number of elements per ZKP batch (batch_size must be divisible by zkp_batch_size) +- `bloom_filter_capacity`: Bloom filter size in bits (0 for output queues) +- `currently_processing_batch_index`: Index of batch accepting new insertions (Fill state) +- `pending_batch_index`: Index of batch ready for ZKP processing and tree insertion (Full or being incrementally inserted) +- `next_index`: Next available leaf index in queue +- `batches`: Array of 2 `Batch` structures + +**Variants:** +- **Output Queue** (`new_output_queue`): No bloom filters, has value vecs +- **Input Queue** (`new_input_queue`): Has bloom filters, no value vecs + +**Key Validation:** +- `batch_size` must be divisible by `zkp_batch_size` +- Error: `BatchSizeNotDivisibleByZkpBatchSize` if not + +### BatchedMerkleTreeMetadata + +**Description:** +Complete metadata for a batched Merkle tree account. + +**Path:** `src/merkle_tree_metadata.rs` + +**Key Fields:** +- `tree_type`: `TreeType::StateV2` (4) or `TreeType::AddressV2` (5) +- `metadata`: Base `MerkleTreeMetadata` (access control, rollover, etc.) +- `sequence_number`: Increments with each tree update +- `next_index`: Next available leaf index in tree +- `nullifier_next_index`: Nullifier sequence tracker (state trees only) +- `height`: Tree height (default: 32 for state, 40 for address) +- `capacity`: Maximum leaves (2^height) +- `root_history_capacity`: Size of root history buffer (default: 200) +- `queue_batches`: Queue batch metadata +- `hashed_pubkey`: Pre-hashed tree pubkey (31 bytes + 1 padding). Pubkeys are hashed and truncated to 31 bytes (248 bits) to fit within bn254 field size requirements for Poseidon hashing in ZK circuits diff --git a/program-libs/batched-merkle-tree/docs/UPDATE_FROM_INPUT_QUEUE.md b/program-libs/batched-merkle-tree/docs/UPDATE_FROM_INPUT_QUEUE.md new file mode 100644 index 0000000000..832f58ce6a --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/UPDATE_FROM_INPUT_QUEUE.md @@ -0,0 +1,264 @@ +# Update Tree From Input Queue + +**path:** src/merkle_tree.rs + +**description:** +Batch updates Merkle tree from input queue with zero-knowledge proof verification. This operation covers two distinct update types: + +1. **Batch Nullify** (State Trees): Nullifies existing leaves by overwriting compressed account hashes with nullifiers +2. **Batch Address Append** (Address Trees): Appends new addresses to the tree using indexed Merkle tree insertion + +Both operations process one ZKP batch at a time, verifying correctness of: old root + queue values → new root. + +**Circuit implementations:** +- Batch nullify: `prover/server/prover/v2/batch_update_circuit.go` +- Batch address append: `prover/server/prover/v2/batch_address_append_circuit.go` + +Key characteristics: +1. Verifies ZKP proving correctness of: old root + queue values → new root +2. Updates tree root +3. Increments tree sequence_number (tracks number of tree updates) +4. For address trees: increments tree next_index by zkp_batch_size +5. For state trees: increments nullifier_next_index (offchain indexer tracking only) +6. Marks ZKP batch as inserted in the queue +7. Transitions batch state to Inserted when all ZKP batches complete +8. Zeros out bloom filter when current batch is 50% inserted + +**Operations:** + +## Batch Nullify (State Trees) + +Method: `BatchedMerkleTreeAccount::update_tree_from_input_queue` + +**Parameters:** +- `instruction_data`: InstructionDataBatchNullifyInputs - Contains new_root and compressed ZK proof + +**Accounts:** +- `BatchedMerkleTreeAccount` (state tree): + - Must be type `TreeType::StateV2` + - Contains integrated input queue with nullifiers + - Account layout defined in: src/merkle_tree.rs + - Account documentation: TREE_ACCOUNT.md + +**Public inputs for ZKP verification:** +- old_root: Current tree root before update +- new_root: New tree root after batch nullify +- leaves_hash_chain: Hash chain from input queue (nullifiers) +- Public input hash: Hash([old_root, new_root, leaves_hash_chain]) + +**What the ZKP (circuit) proves:** + +The batch update circuit proves that nullifiers have been correctly inserted into the Merkle tree: + +1. **Verify public input hash:** + - Computes Hash([old_root, new_root, leaves_hash_chain]) + - Asserts equals circuit.PublicInputHash + +2. **Create and verify nullifiers:** + - For each position i in batch (zkp_batch_size): + - Computes nullifier[i] = Hash(Leaves[i], PathIndices[i], TxHashes[i]) + - Where Leaves[i] is the compressed_account_hash being nullified + - PathIndices[i] is the leaf index in the tree + - TxHashes[i] is the transaction hash + - Computes hash chain of all nullifiers + - Asserts equals circuit.LeavesHashchainHash + +3. **Perform Merkle updates:** + - Initialize running root = circuit.OldRoot + - For each position i (zkp_batch_size positions): + - Convert PathIndices[i] to binary (tree height bits) + - Call MerkleRootUpdateGadget: + - OldRoot: running root + - OldLeaf: circuit.OldLeaves[i] (can be 0 if not yet appended, or compressed_account_hash) + - NewLeaf: nullifier[i] + - PathIndex: PathIndices[i] as bits + - MerkleProof: circuit.MerkleProofs[i] + - Height: tree height + - Update running root with result + - Assert final running root equals circuit.NewRoot + +4. **Public inputs:** Hash([old_root, new_root, leaves_hash_chain]) + +**Key circuit characteristics:** +- Path index is included in nullifier hash to ensure correct leaf is nullified even when old_leaf is 0 +- Since input and output queues are independent, nullifiers can be inserted before values are appended to the tree +- Merkle proof verifies old_leaf value against onchain root, ensuring correct position +- If old_leaf is 0: value not yet appended, but path index in nullifier ensures correct future position +- If old_leaf is non-zero: should equal compressed_account_hash (verified by Merkle proof) + +## Batch Address Append (Address Trees) + +Method: `BatchedMerkleTreeAccount::update_tree_from_address_queue` + +**Parameters:** +- `instruction_data`: InstructionDataAddressAppendInputs - Contains new_root and compressed ZK proof + +**Accounts:** +- `BatchedMerkleTreeAccount` (address tree): + - Must be type `TreeType::AddressV2` + - Contains integrated input queue with addresses + - Account layout defined in: src/merkle_tree.rs + - Account documentation: TREE_ACCOUNT.md + +**Public inputs for ZKP verification:** +- old_root: Current tree root before update +- new_root: New tree root after batch address append +- leaves_hash_chain: Hash chain from address queue (addresses) +- start_index: Tree next_index (where batch append begins) +- Public input hash: Hash([old_root, new_root, leaves_hash_chain, start_index]) + +**What the ZKP (circuit) proves:** + +The batch address append circuit proves that addresses have been correctly appended using indexed Merkle tree insertion: + +1. **Initialize running root:** + - Set current root = circuit.OldRoot + +2. **For each address i in batch (zkp_batch_size positions):** + + a. **Update low leaf (insert into sorted linked list):** + - Compute old low leaf hash: + - Uses LeafHashGadget to verify old low leaf structure + - Inputs: LowElementValues[i], LowElementNextValues[i], NewElementValues[i] + - Verifies low_value < new_address < low_next_value (sorted order) + - Compute new low leaf hash: + - Hash(LowElementValues[i], NewElementValues[i]) + - Updates low leaf to point to new address instead of old next value + - Convert LowElementIndices[i] to binary (tree height bits) + - Call MerkleRootUpdateGadget: + - OldRoot: current root + - OldLeaf: old low leaf hash + - NewLeaf: new low leaf hash (Hash(low_value, new_address)) + - PathIndex: LowElementIndices[i] as bits + - MerkleProof: circuit.LowElementProofs[i] + - Height: tree height + - Update current root with result + + b. **Insert new leaf:** + - Compute new leaf hash: + - Hash(NewElementValues[i], LowElementNextValues[i]) + - New address points to what low leaf previously pointed to + - Compute insertion index: start_index + i + - Convert insertion index to binary (tree height bits) + - Call MerkleRootUpdateGadget: + - OldRoot: current root (after low leaf update) + - OldLeaf: 0 (position must be empty) + - NewLeaf: new leaf hash (Hash(new_address, low_next_value)) + - PathIndex: (start_index + i) as bits + - MerkleProof: circuit.NewElementProofs[i] + - Height: tree height + - Update current root with result + +3. **Verify final root:** + - Assert current root equals circuit.NewRoot + +4. **Verify leaves hash chain:** + - Compute hash chain of all NewElementValues + - Assert equals circuit.HashchainHash + +5. **Verify public input hash:** + - Compute Hash([old_root, new_root, hash_chain, start_index]) + - Assert equals circuit.PublicInputHash + +6. **Public inputs:** Hash([old_root, new_root, leaves_hash_chain, start_index]) + +**Key circuit characteristics:** +- Performs TWO Merkle updates per address (low leaf update + new leaf insertion) +- Maintains sorted order via indexed Merkle tree linked list structure +- Verifies new address fits between low_value and low_next_value (sorted insertion) +- New leaf position must be empty (old_leaf = 0) +- Enables efficient non-inclusion proofs (prove address not in sorted tree) + +## Operation Logic and Checks (Both Operations) + +1. **Check tree type:** + - Nullify: Verify tree type is `TreeType::StateV2` + - Address: Verify tree type is `TreeType::AddressV2` + +2. **Check tree capacity (address trees only):** + - Verify: `tree.next_index + zkp_batch_size <= tree_capacity` + - Error if tree would exceed capacity after this batch + +3. **Get batch information:** + - Get `pending_batch_index` from queue (batch ready for tree insertion) + - Get `first_ready_zkp_batch_index` from batch (next ZKP batch to insert) + - Verify batch has ready ZKP batches: `num_full_zkp_batches > num_inserted_zkp_batches` + +4. **Create public inputs hash:** + - Get `leaves_hash_chain` from hash chain store for this ZKP batch + - Get `old_root` from tree root history (most recent root) + - Nullify: Compute `public_input_hash = Hash([old_root, new_root, leaves_hash_chain])` + - Address: Get `start_index` from tree, compute `public_input_hash = Hash([old_root, new_root, leaves_hash_chain, start_index])` + +5. **Verify ZKP and update tree:** + Calls `verify_update` which: + - Nullify: Verifies proof with `verify_batch_update(zkp_batch_size, public_input_hash, proof)` + - Address: Verifies proof with `verify_batch_address_update(zkp_batch_size, public_input_hash, proof)` + - Increments sequence_number (tree update counter) + - Appends new_root to root_history (cyclic buffer) + - Nullify: Increments nullifier_next_index by zkp_batch_size (offchain indexer tracking) + - Address: Increments tree next_index by zkp_batch_size (new leaves appended) + - Returns (old_next_index, new_next_index) for event + +6. **Mark ZKP batch as inserted:** + - Call `mark_as_inserted_in_merkle_tree` on batch: + - Increment `num_inserted_zkp_batches` + - If all ZKP batches inserted: + - Set batch `sequence_number = tree_sequence_number + root_history_capacity` (threshold at which root at root_index has been overwritten in cyclic root history) + - Set batch `root_index` (identifies root that must not exist when bloom filter is zeroed) + - Transition batch state to `Inserted` + - Return batch state for next step + +7. **Increment pending_batch_index if batch complete:** + - If batch state is now `Inserted`: + - Increment `pending_batch_index` (switches to other batch) + +8. **Zero out bloom filter if ready:** + - Same mechanism as described in UPDATE_FROM_OUTPUT_QUEUE.md + - See that document for detailed explanation of bloom filter and root zeroing + +9. **Return batch event:** + - Contains merkle_tree_pubkey, batch indices, root info, next_index range + - Nullify: No output_queue_pubkey + - Address: No output_queue_pubkey + +**Validations:** +- Tree type must match operation (StateV2 for nullify, AddressV2 for address) +- Address trees: Tree must not be full after this batch insertion +- Batch must have ready ZKP batches: `num_full_zkp_batches > num_inserted_zkp_batches` +- Batch must not be in `Inserted` state +- ZKP must verify correctly against public inputs + +**State Changes:** + +**Tree account (Nullify - State Trees):** +- `nullifier_next_index`: Incremented by zkp_batch_size (offchain indexer tracking) +- `sequence_number`: Incremented by 1 (tracks tree updates) +- `root_history`: New root appended (cyclic buffer, may overwrite oldest) +- Input queue bloom filter: May be zeroed if current batch is 50% inserted AND previous batch is fully inserted AND bloom filter not yet zeroed + +**Tree account (Address Append - Address Trees):** +- `next_index`: Incremented by zkp_batch_size (new leaves appended) +- `sequence_number`: Incremented by 1 (tracks tree updates) +- `root_history`: New root appended (cyclic buffer, may overwrite oldest) +- Input queue bloom filter: May be zeroed if current batch is 50% inserted AND previous batch is fully inserted AND bloom filter not yet zeroed + +**Input queue (Both):** +- Batch `num_inserted_zkp_batches`: Incremented +- Batch `state`: May transition to `Inserted` when all ZKP batches complete +- Batch `sequence_number`: Set to `tree_sequence_number + root_history_capacity` when batch fully inserted (threshold at which root at root_index has been overwritten in cyclic root history) +- Batch `root_index`: Set when batch fully inserted (identifies root that must not exist when bloom filter is zeroed) +- Queue `pending_batch_index`: May increment when batch complete + +**Errors:** +- `MerkleTreeMetadataError::InvalidTreeType` (error code: 14007) - Tree type doesn't match operation +- `MerkleTreeMetadataError::InvalidQueueType` (error code: 14004) - Queue type invalid +- `BatchedMerkleTreeError::TreeIsFull` (error code: 14310) - Address tree would exceed capacity after this batch +- `BatchedMerkleTreeError::BatchNotReady` (error code: 14301) - Batch is not in correct state for insertion +- `BatchedMerkleTreeError::InvalidIndex` (error code: 14309) - Root history is empty or index out of bounds +- `BatchedMerkleTreeError::InvalidBatchIndex` (error code: 14308) - Batch index out of range +- `BatchedMerkleTreeError::CannotZeroCompleteRootHistory` (error code: 14313) - Cannot zero out complete or more than complete root history +- `VerifierError::ProofVerificationFailed` (error code: 13006) - ZKP verification failed (proof is invalid) +- `VerifierError::InvalidPublicInputsLength` (error code: 13004) - Public inputs length doesn't match expected +- `ZeroCopyError` (error codes: 15001-15017) - Failed to access root history or hash chain stores +- `HasherError` (error codes: 7001-7012) - Hashing operation failed diff --git a/program-libs/batched-merkle-tree/docs/UPDATE_FROM_OUTPUT_QUEUE.md b/program-libs/batched-merkle-tree/docs/UPDATE_FROM_OUTPUT_QUEUE.md new file mode 100644 index 0000000000..914433a163 --- /dev/null +++ b/program-libs/batched-merkle-tree/docs/UPDATE_FROM_OUTPUT_QUEUE.md @@ -0,0 +1,191 @@ +# Update Tree From Output Queue + +**path:** src/merkle_tree.rs + +**description:** +Batch appends values from the output queue to the state Merkle tree with zero-knowledge proof verification. This operation processes one ZKP batch at a time, verifying that the tree update from old root + queue values → new root is correct. The ZKP proves that the batch of values from the output queue has been correctly appended to the tree. + +**Circuit implementation:** `prover/server/prover/v2/batch_append_circuit.go` + +Key characteristics: +1. Verifies ZKP proving correctness of: old root + queue values → new root +2. Updates tree root and increments tree next_index by zkp_batch_size +3. Increments tree sequence_number (tracks number of tree updates) +4. Marks ZKP batch as inserted in the queue +5. Transitions batch state to Inserted when all ZKP batches of a batch are complete +6. Zeros out input queue bloom filter when current batch is 50% inserted + +Public inputs for ZKP verification: +- old_root: Current tree root before update +- new_root: New tree root after batch append +- leaves_hash_chain: Hash chain from output queue (commitment to queue values) +- start_index: Tree index where batch append begins + +**Operation:** +Method: `BatchedMerkleTreeAccount::update_tree_from_output_queue_account` + +**Parameters:** +- `queue_account`: &mut BatchedQueueAccount - Output queue account containing values to append +- `instruction_data`: InstructionDataBatchAppendInputs - Contains new_root and compressed ZK proof + +**Accounts:** +This operation modifies: +1. `BatchedMerkleTreeAccount` (state tree): + - Must be type `TreeType::StateV2` + - Account layout defined in: src/merkle_tree.rs + - Account documentation: TREE_ACCOUNT.md + +2. `BatchedQueueAccount` (output queue): + - Must be associated with the state tree (pubkeys match) + - Account layout defined in: src/queue.rs + - Account documentation: QUEUE_ACCOUNT.md + +**Operation Logic and Checks:** + +1. **Check tree is not full:** + - Verify: `tree.next_index + zkp_batch_size <= tree_capacity` + - Error if tree would exceed capacity after this batch + +2. **Get batch information:** + - Get `pending_batch_index` from queue (batch ready for tree insertion) + - Get `first_ready_zkp_batch_index` from batch (next ZKP batch to insert) + - Verify batch has ready ZKP batches: `num_full_zkp_batches > num_inserted_zkp_batches` + - Batch can be in `Fill` (being filled) or `Full` state + +3. **Create public inputs hash:** + - Get `leaves_hash_chain` from output queue for this ZKP batch + - Get `old_root` from tree root history (most recent root) + - Get `start_index` from tree (where this batch will be appended) + - Compute: `public_input_hash = Hash([old_root, new_root, leaves_hash_chain, start_index])` + +4. **Verify ZKP and update tree:** + Calls `verify_update` which: + - Verifies proof: `verify_batch_append_with_proofs(zkp_batch_size, public_input_hash, proof)` + - Increments tree next_index by zkp_batch_size + - Increments sequence_number (tree update counter) + - Appends new_root to root_history (cyclic buffer) + - Returns (old_next_index, new_next_index) for event + + **What the ZKP (circuit) proves:** + The batch append circuit proves that a batch of values has been correctly appended to the Merkle tree: + 1. Verifies the public input hash matches Hash([old_root, new_root, leaves_hash_chain, start_index]) + 2. Verifies the leaves_hash_chain matches the hash chain of all new leaves + 3. For each position in the batch (zkp_batch_size positions): + - Checks if old_leaf is zero (empty slot) or non-zero (contains nullifier): + - If zero: insert the new leaf + - If non-zero: keep the old leaf (don't overwrite nullified values) + - Provides Merkle proof for the old leaf value + - Computes Merkle root update using MerkleRootUpdateGadget + - Updates running root for next iteration + 4. Verifies the final computed root equals the claimed new_root + 5. Public inputs: Hash([old_root, new_root, leaves_hash_chain, start_index]) + Note: Since input and output queues are independent, a nullifier can be inserted into the tree before the value is appended to the tree. The circuit handles this by checking if the position already contains a nullifier (old_leaf is non-zero) and keeping it instead of overwriting. + +5. **Mark ZKP batch as inserted:** + - Call `mark_as_inserted_in_merkle_tree` on queue batch: + - Increment `num_inserted_zkp_batches` + - If all ZKP batches inserted: + - Set batch `sequence_number = tree_sequence_number + root_history_capacity` (threshold at which root at root_index has been overwritten in cyclic root history) + - Set batch `root_index` (identifies root that must not exist when bloom filter is zeroed) + - Transition batch state to `Inserted` + - Return batch state for next step + +6. **Increment pending_batch_index if batch complete:** + - If batch state is now `Inserted`: + - Increment `pending_batch_index` (switches to other batch) + +7. **Zero out input queue bloom filter if ready:** + + Clears input queue bloom filter after batch insertion to enable batch reuse. This operation runs during both output queue updates AND input queue updates (nullify and address operations). + + **Why zeroing is necessary:** + - Input queue bloom filters store compressed account hashes to prevent double-spending + - After batch insertion, old bloom filter values prevent batch reuse (non-inclusion checks fail for legitimate new insertions) + - Roots from batch insertion period can prove inclusion of bloom filter values + - Bloom filter must be zeroed to reuse batch; unsafe roots must be zeroed if they still exist in root history + + **When zeroing occurs (all conditions must be true):** + 1. Current batch is at least 50% full: `num_inserted_elements >= batch_size / 2` + 2. Current batch is NOT in `Inserted` state (still being filled) + 3. Previous batch is in `Inserted` state (fully processed) + 4. Previous batch bloom filter NOT already zeroed: `!bloom_filter_is_zeroed()` + 5. At least one tree update occurred since batch completion: `batch.sequence_number != current_tree.sequence_number` + + **Why wait until 50% full:** + - Zeroing is computationally expensive (foresters perform this, not users) + - Don't zero when inserting last zkp of batch (would cause failing user transactions) + - Grace period for clients to switch from proof-by-index to proof-by-zkp for previous batch values + + **Zeroing procedure:** + + a. **Mark bloom filter as zeroed** - Sets flag to prevent re-zeroing + + b. **Zero out bloom filter bytes** - All bytes set to 0 + + c. **Zero out overlapping roots** (if any exist): + + **Check for overlapping roots:** + - Overlapping roots exist if: `batch.sequence_number > current_tree.sequence_number` + - Cyclic root history has NOT yet overwritten all roots from batch insertion period + - `batch.sequence_number` was set to `tree_sequence_number + root_history_capacity` at batch completion + - Represents threshold at which root at `batch.root_index` would be naturally overwritten + + **Calculate unsafe roots:** + - `num_remaining_roots = batch.sequence_number - current_tree.sequence_number` + - Roots NOT overwritten since batch insertion + - These roots can still prove inclusion of bloom filter values + - `first_safe_root_index = batch.root_index + 1` + + **Safety check:** + - Verify: `num_remaining_roots < root_history.len()` (never zero complete or more than complete root history) + + **Zero unsafe roots:** + - Start at `oldest_root_index = root_history.first_index()` + - Zero `num_remaining_roots` consecutive roots in cyclic buffer + - Loop wraps: `oldest_root_index = (oldest_root_index + 1) % root_history.len()` + - Sets each root to `[0u8; 32]` + + **Defensive assertion:** + - Verify ended at `first_safe_root_index` (ensures correct range zeroed) + + **Why safe:** + - `sequence_number` mechanism determines when roots are safe to keep + - Roots at or after `first_safe_root_index` are from updates after batch insertion + - These roots cannot prove inclusion of zeroed bloom filter values + - Manual zeroing of overlapping roots prevents cyclic buffer race conditions + +8. **Return batch append event:** + - Contains merkle_tree_pubkey, output_queue_pubkey, batch indices, root info, next_index range + +**Validations:** +- Tree must be state tree (enforced by tree type check) +- Tree must not be full after this batch insertion +- Queue and tree must be associated (pubkeys match) +- Batch must have ready ZKP batches: `num_full_zkp_batches > num_inserted_zkp_batches` +- Batch must not be in `Inserted` state +- ZKP must verify correctly against public inputs + +**State Changes:** + +**Tree account:** +- `next_index`: Incremented by zkp_batch_size (is the leaf index for next insertion) +- `sequence_number`: Incremented by 1 (tracks the number of tree updates) +- `root_history`: New root appended (cyclic buffer, overwrites oldest) +- Input queue bloom filter: May be zeroed if current batch is 50% inserted AND previous batch is fully inserted AND bloom filter not yet zeroed + +**Queue account:** +- Batch `num_inserted_zkp_batches`: Incremented +- Batch `state`: May transition to `Inserted` when all ZKP batches complete +- Batch `sequence_number`: Set to `tree_sequence_number + root_history_capacity` when batch fully inserted (threshold at which root at root_index has been overwritten in cyclic root history) +- Batch `root_index`: Set when batch fully inserted (identifies root that must not exist when bloom filter is zeroed) +- Queue `pending_batch_index`: Increments when batch complete + +**Errors:** +- `MerkleTreeMetadataError::InvalidTreeType` (error code: 14007) - Tree is not a state tree +- `MerkleTreeMetadataError::MerkleTreeAndQueueNotAssociated` (error code: 14001) - Queue and tree pubkeys don't match +- `BatchedMerkleTreeError::TreeIsFull` (error code: 14310) - Tree would exceed capacity after this batch +- `BatchedMerkleTreeError::BatchNotReady` (error code: 14301) - Batch is not in correct state for insertion +- `BatchedMerkleTreeError::InvalidIndex` (error code: 14309) - Root history is empty or index out of bounds +- `BatchedMerkleTreeError::CannotZeroCompleteRootHistory` (error code: 14313) - Cannot zero out complete or more than complete root history +- `VerifierError::ProofVerificationFailed` (error code: 13006) - ZKP verification failed (proof is invalid) +- `ZeroCopyError` (error codes: 15001-15017) - Failed to access root history or hash chain stores diff --git a/program-tests/CLAUDE.md b/program-tests/CLAUDE.md new file mode 100644 index 0000000000..cbacfb4b41 --- /dev/null +++ b/program-tests/CLAUDE.md @@ -0,0 +1,147 @@ +# Program Tests - Integration Test Suite + +This directory contains long-running integration tests that require Solana runtime (SBF). All tests (except `zero-copy-derive-test`) depend on `program-tests/utils` (light-test-utils). + +## Test Organization + +### Why Tests Live Here + +- **Most tests**: Depend on `program-tests/utils` (light-test-utils) for shared test infrastructure +- **`batched-merkle-tree-test`**: Specifically located here because it depends on program-tests/utils +- **`zero-copy-derive-test`**: Exception - placed here only to avoid cyclic dependencies (NOT a long-running integration test) + +### Test Execution + +All tests use `cargo test-sbf` which compiles programs to SBF (Solana Bytecode Format) and runs them in a Solana runtime environment. + +## Environment Variables + +```bash +export RUSTFLAGS="-D warnings" +export REDIS_URL=redis://localhost:6379 +``` + +## Test Packages + +### Account Compression Tests +```bash +cargo test-sbf -p account-compression-test +``` +Tests for the core account compression program (Merkle tree management). + +### Registry Tests +```bash +cargo test-sbf -p registry-test +``` +Tests for protocol configuration and forester registration. + +### Light System Program Tests + +#### Address Tests +```bash +cargo test-sbf -p system-test -- test_with_address +``` +Tests for address-based operations in the Light system program. + +#### Compression Tests +```bash +cargo test-sbf -p system-test -- test_with_compression +cargo test-sbf -p system-test --test test_re_init_cpi_account +``` +Tests for compressed account operations. + +### System CPI Tests + +#### V1 CPI Tests +```bash +cargo test-sbf -p system-cpi-test +``` +Tests for Cross-Program Invocation (CPI) with Light system program V1. + +#### V2 CPI Tests +```bash +# Main tests (excluding functional and event parsing) +cargo test-sbf -p system-cpi-v2-test -- --skip functional_ --skip event::parse + +# Event parsing tests +cargo test-sbf -p system-cpi-v2-test -- event::parse + +# Functional tests - read-only +cargo test-sbf -p system-cpi-v2-test -- functional_read_only + +# Functional tests - account infos +cargo test-sbf -p system-cpi-v2-test -- functional_account_infos +``` +Tests for Cross-Program Invocation (CPI) with Light system program V2. + +### Compressed Token Tests + +#### Core Token Tests +```bash +cargo test-sbf -p compressed-token-test --test ctoken +cargo test-sbf -p compressed-token-test --test v1 +cargo test-sbf -p compressed-token-test --test mint +cargo test-sbf -p compressed-token-test --test transfer2 +``` + +#### Batched Tree Tests (with retry logic in CI) +```bash +cargo test-sbf -p compressed-token-test -- test_transfer_with_photon_and_batched_tree +``` +Note: CI runs this with retry logic (max 3 attempts, 5s delay) due to known flakiness. + +### E2E Tests +```bash +cargo test-sbf -p e2e-test +``` +End-to-end integration tests across multiple programs. + +#### E2E Extended Tests +After building the small compressed token program: +```bash +pnpm --filter @lightprotocol/programs run build-compressed-token-small +cargo test-sbf -p e2e-test -- --test test_10_all +``` + +### Batched Merkle Tree Tests +```bash +# Skip long-running simulation and e2e tests +cargo test -p batched-merkle-tree-test -- --skip test_simulate_transactions --skip test_e2e + +# Run simulation test with logging +RUST_LOG=light_prover_client=debug cargo test -p batched-merkle-tree-test -- --test test_simulate_transactions + +# Run e2e test +cargo test -p batched-merkle-tree-test -- --test test_e2e +``` +Note: Located in program-tests because it depends on program-tests/utils. + +### Pinocchio No-std Tests +```bash +cargo test-sbf -p pinocchio-nostd-test +``` +Tests for Pinocchio library in no-std environment. + +### Zero-Copy Derive Tests (Unit Test Exception) +```bash +cargo test -p zero-copy-derive-test +``` +**Special case**: This is NOT an integration test. It's a unit test located in program-tests only to avoid cyclic dependencies in the package dependency graph. + +## Supporting Infrastructure + +### Test Utilities (light-test-utils) +Located at `program-tests/utils`, this crate provides shared test infrastructure used by most integration tests. + +### Test Programs +Some directories contain test programs (not tests themselves) used by the integration tests: +- `create-address-test-program` +- `merkle-tree` + +## CI Workflow Reference + +These tests are run in the following GitHub Actions workflows: +- `.github/workflows/programs.yml` - Main program integration tests +- `.github/workflows/rust.yml` - Batched Merkle tree tests (partial) + +For the exact test matrix and execution order, see the workflow files. diff --git a/sdk-tests/CLAUDE.md b/sdk-tests/CLAUDE.md new file mode 100644 index 0000000000..c587319290 --- /dev/null +++ b/sdk-tests/CLAUDE.md @@ -0,0 +1,129 @@ +# SDK Tests - SDK Integration Test Suite + +This directory contains integration tests for various SDK implementations. Tests verify that Light Protocol SDKs work correctly with Solana programs using different frameworks (native, Anchor, Pinocchio) and use cases (token operations). + +## Test Organization + +All tests in this directory are integration tests that run with `cargo test-sbf`, compiling programs to SBF (Solana Bytecode Format) and executing them in a Solana runtime environment. + +## Environment Variables + +```bash +export RUSTFLAGS="-D warnings" +export REDIS_URL=redis://localhost:6379 +``` + +## Test Packages + +V1 means v1 Merkle tree accounts (concurrent Merkle trees, state & address Merkle trees are height 26, address queue and tree are separate solana accounts). +V2 means v2 Merkle tree accounts (batched Merkle trees, state Merkle tree height 32, address Merkle tree height 40 address queue and tree are the same solana account). + +### Native SDK Tests + +#### V1 Native SDK +```bash +cargo test-sbf -p sdk-v1-native-test +``` +Tests for Light SDK V1 with native Solana programs (without Anchor framework). + +#### V2 Native SDK +```bash +cargo test-sbf -p sdk-native-test +``` +Tests for Light SDK V2 with native Solana programs (without Anchor framework). + +### Anchor SDK Tests + +#### Rust Tests +```bash +cargo test-sbf -p sdk-anchor-test +``` +Tests for Light SDK with Anchor framework (Rust tests). + +#### TypeScript Tests +```bash +npx nx build @lightprotocol/sdk-anchor-test +cd sdk-tests/sdk-anchor-test +npm run test-ts +``` +TypeScript integration tests for Anchor SDK. + +**Test scripts** (see `sdk-tests/sdk-anchor-test/package.json`): +- `npm run build` - Build the Anchor program +- `npm run test-ts` - Run TypeScript integration tests with Light test validator + +### Pinocchio SDK Tests + +#### V1 Pinocchio SDK +```bash +cargo test-sbf -p sdk-pinocchio-v1-test +``` +Tests for Light SDK V1 with Pinocchio framework (high-performance Solana program framework). + +#### V2 Pinocchio SDK +```bash +cargo test-sbf -p sdk-pinocchio-v2-test +``` +Tests for Light SDK V2 with Pinocchio framework. + +### Token SDK Tests +```bash +cargo test-sbf -p sdk-token-test +``` +Tests for compressed token SDK operations (mint, transfer, burn, etc.). + +### Client Tests +```bash +cargo test-sbf -p client-test +``` +Tests for the Light RPC client library for querying compressed accounts. + +## SDK Libraries (Unit Tests) + +While the above are integration tests, the actual SDK libraries are located in `sdk-libs/` and have their own unit tests: + +```bash +# SDK core libraries +cargo test -p light-sdk-macros +cargo test -p light-sdk-macros --all-features +cargo test -p light-sdk +cargo test -p light-sdk --all-features + +# Supporting libraries +cargo test -p light-program-test +cargo test -p light-client +cargo test -p light-sparse-merkle-tree +cargo test -p light-compressed-token-types +cargo test -p light-compressed-token-sdk +``` + +## Test Categories + +### By Framework +- **Native**: Direct Solana program development without frameworks +- **Anchor**: Anchor framework for Solana program development +- **Pinocchio**: High-performance Solana program framework with minimal overhead + +### By SDK Version +- **V1**: Original Light SDK API +- **V2**: Updated Light SDK API with improvements + +### By Use Case +- **General SDK**: Core compressed account operations +- **Token SDK**: Compressed SPL token operations +- **Client SDK**: RPC client for querying compressed accounts + +## CI Workflow Reference + +These tests are run in the following GitHub Actions workflows: +- `.github/workflows/sdk-tests.yml` - Main SDK integration tests (also called "examples-tests") +- `.github/workflows/js-v2.yml` - TypeScript SDK tests (V2) + +For the exact test matrix and execution order, see the workflow files. + +## TypeScript Tests + +For TypeScript/JavaScript tests in this directory, see the `package.json` files in individual test directories: +- `sdk-anchor-test/package.json` - Anchor TypeScript tests + +Additional TypeScript tests for SDK libraries are located in the `js/` directory.