feat: compressed mint photon api

.github/workflows/programs.yml

@@ -7,7 +7,7 @@ on:
       - "program-tests/**"
       - "program-libs/**"
       - "prover/client/**"
-      - ".github/workflows/light-system-programs-tests.yml"
+      - ".github/workflows/programs.yml"
   pull_request:
     branches:
       - "*"
@@ -16,22 +16,24 @@ on:
       - "program-tests/**"
       - "program-libs/**"
       - "prover/client/**"
-      - ".github/workflows/light-system-programs-tests.yml"
+      - ".github/workflows/programs.yml"
     types:
       - opened
       - synchronize
       - reopened
       - ready_for_review
 
 name: programs
+permissions:
+  contents: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   system-programs:
-    name: programs
+    name: ${{ matrix.test-group }}
     if: github.event.pull_request.draft == false
     runs-on: warp-ubuntu-latest-x64-4x
     timeout-minutes: 90
@@ -52,27 +54,16 @@ jobs:
 
     strategy:
       matrix:
-        include:
-          - program: account-compression-and-registry
-            sub-tests: '["cargo-test-sbf -p account-compression-test", "cargo-test-sbf -p registry-test"]'
-          - program: light-system-program-address
-            sub-tests: '["cargo-test-sbf -p system-test -- test_with_address", "cargo-test-sbf -p e2e-test", "cargo-test-sbf -p compressed-token-test --test light_token"]'
-          - program: light-system-program-compression
-            sub-tests: '["cargo-test-sbf -p system-test -- test_with_compression", "cargo-test-sbf -p system-test --test test_re_init_cpi_account"]'
-          - program: compressed-token-and-e2e
-            sub-tests: '["cargo test -p light-compressed-token", "cargo-test-sbf -p compressed-token-test --test v1", "cargo-test-sbf -p compressed-token-test --test mint"]'
-          - program: compressed-token-batched-tree
-            sub-tests: '["cargo-test-sbf -p compressed-token-test -- test_transfer_with_photon_and_batched_tree"]'
-          - program: system-cpi-test
-            sub-tests:
-              '["cargo-test-sbf -p system-cpi-test", "cargo test -p light-system-program-pinocchio",
-              "cargo-test-sbf -p system-cpi-v2-test -- --skip functional_ --skip event::parse", "cargo-test-sbf -p system-cpi-v2-test -- event::parse",
-              "cargo-test-sbf -p compressed-token-test --test transfer2"
-              ]'
-          - program: system-cpi-test-v2-functional-read-only
-            sub-tests: '["cargo-test-sbf -p system-cpi-v2-test -- functional_read_only"]'
-          - program: system-cpi-test-v2-functional-account-infos
-            sub-tests: '["cargo-test-sbf -p system-cpi-v2-test -- functional_account_infos"]'
+        test-group:
+          - account-compression-and-registry
+          - system-address
+          - system-compression
+          - compressed-token-and-e2e
+          - compressed-token-batched-tree
+          - system-cpi
+          - system-cpi-v2-functional-read-only
+          - system-cpi-v2-functional-account-infos
+
     steps:
       - name: Checkout sources
         uses: actions/checkout@v6
@@ -87,34 +78,7 @@ jobs:
         run: |
           just cli build
 
-      - name: ${{ matrix.program }}
+      - name: Run tests
+        working-directory: program-tests
         run: |
-
-          IFS=',' read -r -a sub_tests <<< "${{ join(fromJSON(matrix['sub-tests']), ', ') }}"
-          for subtest in "${sub_tests[@]}"
-          do
-            echo "$subtest"
-
-            # Retry logic for flaky batched-tree test
-            if [[ "$subtest" == *"test_transfer_with_photon_and_batched_tree"* ]]; then
-              echo "Running flaky test with retry logic (max 3 attempts)..."
-              attempt=1
-              max_attempts=3
-              until RUSTFLAGS="-D warnings" eval "$subtest"; do
-                attempt=$((attempt + 1))
-                if [ $attempt -gt $max_attempts ]; then
-                  echo "Test failed after $max_attempts attempts"
-                  exit 1
-                fi
-                echo "Attempt $attempt/$max_attempts failed, retrying..."
-                sleep 5
-              done
-              echo "Test passed on attempt $attempt"
-            else
-              RUSTFLAGS="-D warnings" eval "$subtest"
-              if [ "$subtest" == "cargo-test-sbf -p e2e-test" ]; then
-                just programs build-compressed-token-small
-                RUSTFLAGS="-D warnings" eval "$subtest -- --test test_10_all"
-              fi
-            fi
-          done
+          just ci-${{ matrix.test-group }}

.mise.toml

@@ -0,0 +1,4 @@
+# Disable mise's Go management for this project.
+# We use our own Go installation via devenv.sh.
+[settings]
+disable_tools = ["go"]

Cargo.toml

@@ -147,6 +147,7 @@ pinocchio-pubkey = { version = "0.3.0" }
 pinocchio-system = { version = "0.3.0" }
 bs58 = "^0.5.1"
 sha2 = "0.10"
+hex = "0.4"
 litesvm = "0.7"
 # Anchor
 anchor-lang = { version = "0.31.1" }

cli/src/commands/test-validator/index.ts

@@ -43,6 +43,22 @@ class SetupCommand extends Command {
         "Runs a test validator without starting a new prover service.",
       default: false,
     }),
+    forester: Flags.boolean({
+      description:
+        "Start the forester service for auto-compression of compressible accounts.",
+      default: false,
+    }),
+    "forester-port": Flags.integer({
+      description: "Port for the forester API server.",
+      required: false,
+      default: 8080,
+    }),
+    "compressible-pda-program": Flags.string({
+      description:
+        "Compressible PDA programs to track. Format: 'program_id:discriminator_base58'. Can be specified multiple times.",
+      required: false,
+      multiple: true,
+    }),
     "skip-system-accounts": Flags.boolean({
       description:
         "Runs a test validator without initialized light system accounts.",
@@ -147,6 +163,13 @@ class SetupCommand extends Command {
       default: true,
       allowNo: true,
     }),
+    "account-dir": Flags.string({
+      description:
+        "Additional directory containing account JSON files to preload. Can be specified multiple times.",
+      required: false,
+      multiple: true,
+      summary: "Usage: --account-dir <path/to/accounts/>",
+    }),
   };
 
   validatePrograms(
@@ -216,6 +239,7 @@ class SetupCommand extends Command {
       await stopTestEnv({
         indexer: !flags["skip-indexer"],
         prover: !flags["skip-prover"],
+        forester: flags.forester,
       });
       this.log("\nTest validator stopped successfully \x1b[32m✔\x1b[0m");
     } else {
@@ -268,6 +292,9 @@ class SetupCommand extends Command {
         indexerPort: flags["indexer-port"],
         proverPort: flags["prover-port"],
         prover: !flags["skip-prover"],
+        forester: flags.forester,
+        foresterPort: flags["forester-port"],
+        compressiblePdaPrograms: flags["compressible-pda-program"],
         skipSystemAccounts: flags["skip-system-accounts"],
         geyserConfig: flags["geyser-config"],
         validatorArgs: flags["validator-args"],
@@ -279,6 +306,7 @@ class SetupCommand extends Command {
         verbose: flags.verbose,
         skipReset: flags["skip-reset"],
         useSurfpool: flags["use-surfpool"],
+        additionalAccountDirs: flags["account-dir"],
       });
       this.log("\nSetup tasks completed successfully \x1b[32m✔\x1b[0m");
     }

cli/src/utils/constants.ts

@@ -19,6 +19,7 @@ export const SOLANA_VALIDATOR_PROCESS_NAME = "solana-test-validator";
 export const SURFPOOL_PROCESS_NAME = "surfpool";
 export const LIGHT_PROVER_PROCESS_NAME = "light-prover";
 export const INDEXER_PROCESS_NAME = "photon";
+export const FORESTER_PROCESS_NAME = "forester";
 
 export const SURFPOOL_VERSION = "1.0.1";
 export const SURFPOOL_RELEASE_TAG = "v1.0.1-light";

cli/src/utils/initTestEnv.ts

@@ -24,6 +24,11 @@ import {
 } from "./process";
 import { killProver, startProver } from "./processProverServer";
 import { killIndexer, startIndexer } from "./processPhotonIndexer";
+import {
+  killForester,
+  startForester,
+  getPayerForForester,
+} from "./processForester";
 import { Connection, PublicKey } from "@solana/web3.js";
 import { execSync } from "child_process";
 
@@ -101,8 +106,10 @@ async function getProgramOwnedAccounts(
 export async function stopTestEnv(options: {
   indexer: boolean;
   prover: boolean;
+  forester?: boolean;
 }) {
   const processesToKill = [
+    { name: "forester", condition: options.forester ?? false, killFunction: killForester },
     { name: "photon", condition: options.indexer, killFunction: killIndexer },
     { name: "prover", condition: options.prover, killFunction: killProver },
     {
@@ -135,9 +142,11 @@ export async function initTestEnv({
   skipSystemAccounts,
   indexer = true,
   prover = true,
+  forester = false,
   rpcPort = 8899,
   indexerPort = 8784,
   proverPort = 3001,
+  foresterPort = 8080,
   gossipHost = "127.0.0.1",
   checkPhotonVersion = true,
   photonDatabaseUrl,
@@ -148,6 +157,8 @@ export async function initTestEnv({
   verbose,
   skipReset,
   useSurfpool,
+  compressiblePdaPrograms,
+  additionalAccountDirs,
 }: {
   additionalPrograms?: { address: string; path: string }[];
   upgradeablePrograms?: {
@@ -158,9 +169,11 @@ export async function initTestEnv({
   skipSystemAccounts?: boolean;
   indexer: boolean;
   prover: boolean;
+  forester?: boolean;
   rpcPort?: number;
   indexerPort?: number;
   proverPort?: number;
+  foresterPort?: number;
   gossipHost?: string;
   checkPhotonVersion?: boolean;
   photonDatabaseUrl?: string;
@@ -171,6 +184,8 @@ export async function initTestEnv({
   verbose?: boolean;
   skipReset?: boolean;
   useSurfpool?: boolean;
+  compressiblePdaPrograms?: string[];
+  additionalAccountDirs?: string[];
 }) {
   if (useSurfpool) {
     // For surfpool we can await startTestValidator because spawnBinary returns
@@ -189,6 +204,7 @@ export async function initTestEnv({
       verbose,
       skipReset,
       useSurfpool,
+      additionalAccountDirs,
     });
     // Surfpool only supports JSON-RPC POST, not GET /health.
     await confirmRpcReadiness(`http://127.0.0.1:${rpcPort}`);
@@ -207,6 +223,7 @@ export async function initTestEnv({
       verbose,
       skipReset,
       useSurfpool,
+      additionalAccountDirs,
     });
     await waitForServers([{ port: rpcPort, path: "/health" }]);
     await confirmServerStability(`http://127.0.0.1:${rpcPort}/health`);
@@ -250,6 +267,48 @@ export async function initTestEnv({
       startSlot,
     );
   }
+
+  if (forester) {
+    if (!indexer || !prover) {
+      throw new Error("Forester requires both indexer and prover to be running");
+    }
+    try {
+      const payer = getPayerForForester();
+      await startForester({
+        rpcUrl: `http://127.0.0.1:${rpcPort}`,
+        wsRpcUrl: `ws://127.0.0.1:${rpcPort + 1}`,
+        indexerUrl: `http://127.0.0.1:${indexerPort}`,
+        proverUrl: `http://127.0.0.1:${proverPort}`,
+        payer,
+        foresterPort,
+        compressiblePdaPrograms,
+      });
+    } catch (error) {
+      console.error("Failed to start forester:", error);
+      throw error;
+    }
+  }
+
+  if (forester) {
+    if (!indexer || !prover) {
+      throw new Error("Forester requires both indexer and prover to be running");
+    }
+    try {
+      const payer = getPayerForForester();
+      await startForester({
+        rpcUrl: `http://127.0.0.1:${rpcPort}`,
+        wsRpcUrl: `ws://127.0.0.1:${rpcPort + 1}`,
+        indexerUrl: `http://127.0.0.1:${indexerPort}`,
+        proverUrl: `http://127.0.0.1:${proverPort}`,
+        payer,
+        foresterPort,
+        compressiblePdaPrograms,
+      });
+    } catch (error) {
+      console.error("Failed to start forester:", error);
+      throw error;
+    }
+  }
 }
 
 export async function initTestEnvIfNeeded({
@@ -448,6 +507,7 @@ export async function getSurfpoolArgs({
   rpcPort,
   gossipHost,
   downloadBinaries = true,
+  additionalAccountDirs,
 }: {
   additionalPrograms?: { address: string; path: string }[];
   upgradeablePrograms?: {
@@ -459,6 +519,7 @@ export async function getSurfpoolArgs({
   rpcPort?: number;
   gossipHost?: string;
   downloadBinaries?: boolean;
+  additionalAccountDirs?: string[];
 }): Promise<Array<string>> {
   const dirPath = programsDirPath();
 
@@ -508,6 +569,13 @@ export async function getSurfpoolArgs({
     args.push("--account-dir", accountsPath);
   }
 
+  // Load additional account directories
+  if (additionalAccountDirs) {
+    for (const accountDir of additionalAccountDirs) {
+      args.push("--account-dir", path.resolve(accountDir));
+    }
+  }
+
   return args;
 }
 
@@ -603,6 +671,7 @@ export async function startTestValidator({
   verbose,
   skipReset,
   useSurfpool,
+  additionalAccountDirs,
 }: {
   additionalPrograms?: { address: string; path: string }[];
   upgradeablePrograms?: {
@@ -620,6 +689,7 @@ export async function startTestValidator({
   verbose?: boolean;
   skipReset?: boolean;
   useSurfpool?: boolean;
+  additionalAccountDirs?: string[];
 }) {
   if (useSurfpool) {
     const command = await ensureSurfpoolBinary();
@@ -629,6 +699,7 @@ export async function startTestValidator({
       skipSystemAccounts,
       rpcPort,
       gossipHost,
+      additionalAccountDirs,
     });
 
     await killTestValidator(rpcPort);