diff --git a/.github/actions/setup-and-build-nocheck/action.yml b/.github/actions/setup-and-build-nocheck/action.yml index 739174ad6d..66aa83fe11 100644 --- a/.github/actions/setup-and-build-nocheck/action.yml +++ b/.github/actions/setup-and-build-nocheck/action.yml @@ -20,7 +20,10 @@ runs: path: | .local prover/server/proving-keys - key: ${{ runner.os }}-local-${{ hashFiles('scripts/install.sh') }} + key: ${{ runner.os }}-local-v3-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }} + restore-keys: | + ${{ runner.os }}-local-v3- + ${{ runner.os }}-local-v2- - name: Install dependencies if: steps.restore-local-cache.outputs.cache-hit != 'true' @@ -28,13 +31,49 @@ runs: run: | ./scripts/install.sh + - name: Verify and repair installation + shell: bash + run: | + # Even if cache was restored, run install script to check and install missing components + # The install script now checks for actual file existence, so it will only install what's missing + echo "=== Verifying installation integrity ===" + ./scripts/install.sh --no-reset + + - name: Validate environment setup + shell: bash + run: | + echo "=== Validating environment setup ===" + source ./scripts/devenv.sh + + # Quick validation of critical components + [ -f "$GOROOT/bin/go" ] || { echo "✗ Go not found"; exit 1; } + [ -f "$CARGO_HOME/bin/cargo" ] || { echo "✗ Cargo not found"; exit 1; } + [ -d "prover/server/proving-keys" ] || { echo "✗ Proving keys not found"; exit 1; } + + echo "✓ All critical components found" + - name: Set local environment shell: bash run: | source ./scripts/devenv.sh + # Export critical environment variables for subsequent steps + echo "GOROOT=$GOROOT" >> $GITHUB_ENV + echo "CARGO_HOME=$CARGO_HOME" >> $GITHUB_ENV + echo "RUSTUP_HOME=$RUSTUP_HOME" >> $GITHUB_ENV + echo "PATH=$PATH" >> $GITHUB_ENV + echo "CARGO_FEATURES=$CARGO_FEATURES" >> $GITHUB_ENV + echo "REDIS_URL=$REDIS_URL" >> $GITHUB_ENV - name: Rust cache uses: swatinem/rust-cache@v2 + with: + cache-all-crates: true + cache-on-failure: true + # Add workspace target directory + workspaces: | + . -> target + cli -> cli/target + examples -> examples/target - name: Setup pnpm uses: pnpm/action-setup@v4 @@ -42,35 +81,59 @@ runs: run_install: false - name: Get pnpm store directory + id: pnpm-store shell: bash run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + # Get the store path before any pnpm operations + STORE_PATH=$(pnpm store path --silent) + echo "STORE_PATH=${STORE_PATH}" >> $GITHUB_ENV + echo "path=${STORE_PATH}" >> $GITHUB_OUTPUT - - uses: actions/cache@v4 - name: Setup pnpm cache + - name: Setup pnpm cache + id: pnpm-cache + uses: actions/cache@v4 with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store- - name: Install dependencies shell: bash - run: pnpm install + run: | + # Install dependencies + pnpm install --frozen-lockfile + + # Validate node_modules was created + if [ ! -d "node_modules" ] || [ -z "$(ls -A node_modules 2>/dev/null)" ]; then + echo "Error: node_modules not created after pnpm install" + exit 1 + fi + + - name: Save pnpm cache + # Save cache even on failure to speed up retries + if: steps.pnpm-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} - name: Build and prepare for tests shell: bash run: | source ./scripts/devenv.sh mkdir -p /home/runner/.config/solana/ - solana-keygen new --no-bip39-passphrase -o /home/runner/.config/solana/id.json + if [ ! -f /home/runner/.config/solana/id.json ]; then + solana-keygen new --no-bip39-passphrase -o /home/runner/.config/solana/id.json + fi npx nx build @lightprotocol/programs npx nx build @lightprotocol/zk-compression-cli - - name: Cache .local directory - uses: actions/cache@v4 + - name: Save .local directory cache + if: steps.restore-local-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 with: path: | .local prover/server/proving-keys - key: ${{ runner.os }}-local-${{ hashFiles('scripts/install.sh') }} + key: ${{ runner.os }}-local-v3-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }} diff --git a/.github/actions/setup-and-build/action.yml b/.github/actions/setup-and-build/action.yml index 460362f5fc..fb5e7469da 100644 --- a/.github/actions/setup-and-build/action.yml +++ b/.github/actions/setup-and-build/action.yml @@ -1,6 +1,12 @@ name: Setup and build description: Checkout sources, install dependencies, build and prepare for tests +inputs: + skip-components: + description: 'Comma-separated list of components to skip (e.g., "redis,go"). If not specified, all components are installed.' + required: false + default: "" + runs: using: "composite" steps: @@ -22,52 +28,160 @@ runs: path: | .local prover/server/proving-keys - key: ${{ runner.os }}-local-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }} + key: ${{ runner.os }}-local-v3-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }} + restore-keys: | + ${{ runner.os }}-local-v3- + ${{ runner.os }}-local-v2- - - name: Install dependencies + - name: Install system dependencies shell: bash run: | sudo apt-get update - sudo apt-get install -y libudev-dev pkg-config + sudo apt-get install -y libudev-dev pkg-config build-essential - name: Install dependencies if: steps.restore-local-cache.outputs.cache-hit != 'true' shell: bash run: | - ./scripts/install.sh + if [ -n "${{ inputs.skip-components }}" ]; then + ./scripts/install.sh --skip-components "${{ inputs.skip-components }}" + else + ./scripts/install.sh + fi + + - name: Verify and repair installation + shell: bash + run: | + # Even if cache was restored, run install script to check and install missing components + # The install script now checks for actual file existence, so it will only install what's missing + echo "=== Verifying installation integrity ===" + if [ -n "${{ inputs.skip-components }}" ]; then + ./scripts/install.sh --no-reset --skip-components "${{ inputs.skip-components }}" + else + ./scripts/install.sh --no-reset + fi + + - name: Validate environment setup + shell: bash + run: | + echo "=== Validating environment setup ===" + source ./scripts/devenv.sh + + # Check critical binaries exist + echo "Checking Go installation..." + if [ -f "$GOROOT/bin/go" ]; then + echo "✓ Go found at: $GOROOT/bin/go" + $GOROOT/bin/go version || echo "⚠ Go binary exists but failed to run" + else + echo "✗ Go not found at expected location: $GOROOT/bin/go" + exit 1 + fi + + echo "Checking Rust installation..." + if [ -f "$CARGO_HOME/bin/cargo" ]; then + echo "✓ Cargo found at: $CARGO_HOME/bin/cargo" + $CARGO_HOME/bin/cargo --version || echo "⚠ Cargo binary exists but failed to run" + else + echo "✗ Cargo not found at expected location: $CARGO_HOME/bin/cargo" + exit 1 + fi + + echo "Checking Node installation..." + which node && node --version || echo "⚠ Node not in PATH" + + echo "Checking pnpm installation..." + which pnpm && pnpm --version || echo "⚠ pnpm not in PATH" + + echo "Checking Solana installation..." + which solana && solana --version || echo "⚠ Solana not in PATH" + + echo "Checking Anchor installation..." + which anchor && anchor --version || echo "⚠ Anchor not in PATH" + + # Check proving keys + if [ -d "prover/server/proving-keys" ] && [ -n "$(ls -A prover/server/proving-keys 2>/dev/null)" ]; then + echo "✓ Proving keys found: $(ls prover/server/proving-keys | wc -l) files" + else + echo "✗ Proving keys directory missing or empty" + exit 1 + fi + + echo "=== Environment validation complete ===" - name: Set local environment shell: bash run: | source ./scripts/devenv.sh + # Export critical environment variables for subsequent steps + echo "GOROOT=$GOROOT" >> $GITHUB_ENV + echo "CARGO_HOME=$CARGO_HOME" >> $GITHUB_ENV + echo "RUSTUP_HOME=$RUSTUP_HOME" >> $GITHUB_ENV + echo "PATH=$PATH" >> $GITHUB_ENV + echo "CARGO_FEATURES=$CARGO_FEATURES" >> $GITHUB_ENV + echo "REDIS_URL=$REDIS_URL" >> $GITHUB_ENV - name: Rust cache uses: swatinem/rust-cache@v2 + with: + cache-all-crates: true + cache-on-failure: true + # Add workspace target directory + workspaces: | + . -> target + cli -> cli/target + examples -> examples/target - name: Setup pnpm uses: pnpm/action-setup@v4 with: - run_install: true + run_install: false - name: Get pnpm store directory + id: pnpm-store shell: bash run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + # Get the store path before any pnpm operations + STORE_PATH=$(pnpm store path --silent) + echo "STORE_PATH=${STORE_PATH}" >> $GITHUB_ENV + echo "path=${STORE_PATH}" >> $GITHUB_OUTPUT - - uses: actions/cache@v4 - name: Setup pnpm cache + - name: Setup pnpm cache + id: pnpm-cache + uses: actions/cache@v4 with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store- + - name: Install dependencies + shell: bash + run: | + # Install dependencies with frozen lockfile for consistency + pnpm install --frozen-lockfile + + # Validate node_modules was created + if [ ! -d "node_modules" ] || [ -z "$(ls -A node_modules 2>/dev/null)" ]; then + echo "Error: node_modules not created after pnpm install" + exit 1 + fi + + - name: Save pnpm cache + # Save cache even on failure to speed up retries + if: steps.pnpm-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} + - name: Generate Solana keypair shell: bash run: | source ./scripts/devenv.sh mkdir -p /home/runner/.config/solana/ - solana-keygen new --no-bip39-passphrase -o /home/runner/.config/solana/id.json + if [ ! -f /home/runner/.config/solana/id.json ]; then + solana-keygen new --no-bip39-passphrase -o /home/runner/.config/solana/id.json + fi - name: Copy spl_noop.so to target/deploy shell: bash @@ -75,14 +189,25 @@ runs: mkdir -p ./target/deploy cp ./third-party/solana-program-library/spl_noop.so ./target/deploy/spl_noop.so - - name: Cache .local directory + - name: Build Rust programs + shell: bash + run: | + source ./scripts/devenv.sh + echo "Building Rust programs..." + npx nx build @lightprotocol/programs || { + echo "Failed to build programs, retrying with verbose output..." + npx nx build @lightprotocol/programs --verbose + exit 1 + } + + - name: Save .local directory cache if: steps.restore-local-cache.outputs.cache-hit != 'true' - uses: actions/cache@v4 + uses: actions/cache/save@v4 with: path: | .local prover/server/proving-keys - key: ${{ runner.os }}-local-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }} + key: ${{ runner.os }}-local-v3-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }} - name: Check for git changes shell: bash diff --git a/.github/workflows/cli-v1.yml b/.github/workflows/cli-v1.yml new file mode 100644 index 0000000000..018ac91842 --- /dev/null +++ b/.github/workflows/cli-v1.yml @@ -0,0 +1,87 @@ +on: + push: + branches: + - main + pull_request: + branches: + - "*" + types: + - opened + - synchronize + - reopened + - ready_for_review + +name: cli-tests-v1 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + cli-v1: + name: cli-v1 + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + + services: + redis: + image: redis:8.0.1 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + LIGHT_PROTOCOL_VERSION: V1 + REDIS_URL: redis://localhost:6379 + CI: true + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Cache nx + uses: actions/cache@v4 + with: + path: | + .nx/cache + node_modules/.cache/nx + key: nx-cli-v1-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'nx.json') }} + restore-keys: | + nx-cli-v1-${{ runner.os }}- + + - name: Setup and build + uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" + + - name: Build stateless.js with V1 + run: | + source ./scripts/devenv.sh + cd js/stateless.js + pnpm build:v1 + + - name: Build compressed-token with V1 + run: | + source ./scripts/devenv.sh + cd js/compressed-token + pnpm build:v1 + + - name: Build CLI with V1 + run: | + source ./scripts/devenv.sh + npx nx build @lightprotocol/zk-compression-cli + + - name: Run CLI tests with V1 + run: | + source ./scripts/devenv.sh + npx nx test @lightprotocol/zk-compression-cli + + - name: Display prover logs on failure + if: failure() + run: | + echo "=== Displaying prover logs ===" + find cli/test-ledger -name "*prover*.log" -type f -exec echo "=== Contents of {} ===" \; -exec cat {} \; -exec echo "=== End of {} ===" \; || echo "No prover logs found" diff --git a/.github/workflows/cli-v2.yml b/.github/workflows/cli-v2.yml new file mode 100644 index 0000000000..2f7fd728ef --- /dev/null +++ b/.github/workflows/cli-v2.yml @@ -0,0 +1,87 @@ +on: + push: + branches: + - main + pull_request: + branches: + - "*" + types: + - opened + - synchronize + - reopened + - ready_for_review + +name: cli-tests-v2 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + cli-v2: + name: cli-v2 + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + + services: + redis: + image: redis:8.0.1 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + LIGHT_PROTOCOL_VERSION: V2 + REDIS_URL: redis://localhost:6379 + CI: true + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Cache nx + uses: actions/cache@v4 + with: + path: | + .nx/cache + node_modules/.cache/nx + key: nx-cli-v2-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'nx.json') }} + restore-keys: | + nx-cli-v2-${{ runner.os }}- + + - name: Setup and build + uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" + + - name: Build stateless.js with V2 + run: | + source ./scripts/devenv.sh + cd js/stateless.js + pnpm build:v2 + + - name: Build compressed-token with V2 + run: | + source ./scripts/devenv.sh + cd js/compressed-token + pnpm build:v2 + + - name: Build CLI with V2 + run: | + source ./scripts/devenv.sh + npx nx build @lightprotocol/zk-compression-cli + + - name: Run CLI tests with V2 + run: | + source ./scripts/devenv.sh + npx nx test @lightprotocol/zk-compression-cli + + - name: Display prover logs on failure + if: failure() + run: | + echo "=== Displaying prover logs ===" + find . -path "*/test-ledger/*prover*.log" -type f -exec echo "=== Contents of {} ===" \; -exec cat {} \; -exec echo "=== End of {} ===" \; || echo "No prover logs found" diff --git a/.github/workflows/forester-tests.yml b/.github/workflows/forester-tests.yml index 29e2c29d32..d078ea52e8 100644 --- a/.github/workflows/forester-tests.yml +++ b/.github/workflows/forester-tests.yml @@ -96,6 +96,8 @@ jobs: - uses: actions/checkout@v4 - name: Setup and build uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" - name: Clean build artifacts before tests shell: bash run: | diff --git a/.github/workflows/js-v2.yml b/.github/workflows/js-v2.yml new file mode 100644 index 0000000000..e6ecbc8f89 --- /dev/null +++ b/.github/workflows/js-v2.yml @@ -0,0 +1,92 @@ +on: + push: + branches: + - main + pull_request: + branches: + - "*" + types: + - opened + - synchronize + - reopened + - ready_for_review + +name: js-tests-v2 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + stateless-js-v2: + name: stateless-js-v2 + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + + services: + redis: + image: redis:8.0.1 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + LIGHT_PROTOCOL_VERSION: V2 + REDIS_URL: redis://localhost:6379 + CI: true + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Cache nx + uses: actions/cache@v4 + with: + path: | + .nx/cache + node_modules/.cache/nx + key: nx-js-v2-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'nx.json') }} + restore-keys: | + nx-js-v2-${{ runner.os }}- + + - name: Setup and build + uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" + + - name: Build stateless.js with V2 + run: | + source ./scripts/devenv.sh + cd js/stateless.js + pnpm build:v2 + + - name: Build compressed-token with V2 + run: | + source ./scripts/devenv.sh + cd js/compressed-token + pnpm build:v2 + + - name: Build CLI + run: | + source ./scripts/devenv.sh + npx nx build @lightprotocol/zk-compression-cli + + - name: Run stateless.js tests with V2 + run: | + source ./scripts/devenv.sh + npx nx test @lightprotocol/stateless.js + + - name: Run compressed-token tests with V2 + run: | + source ./scripts/devenv.sh + npx nx test @lightprotocol/compressed-token + + - name: Display prover logs on failure + if: failure() + run: | + echo "=== Displaying prover logs ===" + find . -path "*/test-ledger/*prover*.log" -type f -exec echo "=== Contents of {} ===" \; -exec cat {} \; -exec echo "=== End of {} ===" \; || echo "No prover logs found" diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index f8da21ae13..077ddfa0d0 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -11,15 +11,15 @@ on: - reopened - ready_for_review -name: js-tests +name: js-tests-v1 concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - stateless-js: - name: stateless-js + stateless-js-v1: + name: stateless-js-v1 if: github.event.pull_request.draft == false runs-on: ubuntu-latest @@ -35,32 +35,52 @@ jobs: --health-retries 5 env: + LIGHT_PROTOCOL_VERSION: V1 REDIS_URL: redis://localhost:6379 + CI: true steps: - name: Checkout sources uses: actions/checkout@v4 + - name: Cache nx + uses: actions/cache@v4 + with: + path: | + .nx/cache + node_modules/.cache/nx + key: nx-js-v1-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'nx.json') }} + restore-keys: | + nx-js-v1-${{ runner.os }}- + - name: Setup and build uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" - - name: Build CLI + - name: Build stateless.js with V1 + run: | + source ./scripts/devenv.sh + cd js/stateless.js + pnpm build:v1 + + - name: Build compressed-token with V1 run: | source ./scripts/devenv.sh - npx nx build @lightprotocol/zk-compression-cli --skip-nx-cache + cd js/compressed-token + pnpm build:v1 - # Comment for breaking changes to Photon - - name: Run CLI tests + - name: Build CLI run: | source ./scripts/devenv.sh - npx nx test @lightprotocol/zk-compression-cli + npx nx build @lightprotocol/zk-compression-cli - - name: Run stateless.js tests + - name: Run stateless.js tests with V1 run: | source ./scripts/devenv.sh npx nx test @lightprotocol/stateless.js - - name: Run compressed-token tests + - name: Run compressed-token tests with V1 run: | source ./scripts/devenv.sh npx nx test @lightprotocol/compressed-token diff --git a/.github/workflows/light-examples-tests.yml b/.github/workflows/light-examples-tests.yml index 2c0d1d355a..ba06e8ae12 100644 --- a/.github/workflows/light-examples-tests.yml +++ b/.github/workflows/light-examples-tests.yml @@ -64,6 +64,8 @@ jobs: - name: Setup and build uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" - name: build-programs run: | diff --git a/.github/workflows/light-system-programs-tests.yml b/.github/workflows/light-system-programs-tests.yml index a0c38ad6a4..0be5d51a2b 100644 --- a/.github/workflows/light-system-programs-tests.yml +++ b/.github/workflows/light-system-programs-tests.yml @@ -65,8 +65,12 @@ jobs: sub-tests: '["cargo-test-sbf -p compressed-token-test"]' - program: system-cpi-test sub-tests: '["cargo-test-sbf -p system-cpi-test"]' - - program: system-cpi-test-v2 - sub-tests: '["cargo-test-sbf -p system-cpi-v2-test"]' + - program: system-cpi-test-v2-event + sub-tests: '["cargo-test-sbf -p system-cpi-v2-test -- event::parse"]' + - program: system-cpi-test-v2-functional + sub-tests: '["cargo-test-sbf -p system-cpi-v2-test -- functional_"]' + - program: system-cpi-test-v2-other + sub-tests: '["cargo-test-sbf -p system-cpi-v2-test -- --skip functional_ --skip event::parse"]' - program: random-e2e-test sub-tests: '["cargo-test-sbf -p e2e-test"]' steps: @@ -75,6 +79,8 @@ jobs: - name: Setup and build uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" - name: Build CLI run: | diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 62775c9ba6..efa24bff78 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -35,17 +35,19 @@ jobs: matrix: group: - name: concurrent-merkle-tree - packages: light-concurrent-merkle-tree light-batched-merkle-tree + packages: light-concurrent-merkle-tree test_cmd: | cargo test -p light-concurrent-merkle-tree + - name: batched-merkle-tree-simulate + packages: light-batched-merkle-tree + test_cmd: | RUST_LOG=light_prover_client=debug cargo test -p light-batched-merkle-tree --features test-only -- --test test_simulate_transactions - - name: program-libs + - name: program-libs-fast packages: - aligned-sized light-bloom-filter light-hasher light-compressed-account light-account-checks \ - light-verifier light-merkle-tree-metadata light-zero-copy light-hash-set light-indexed-merkle-tree light-batched-merkle-tree + aligned-sized light-hasher light-compressed-account light-account-checks \ + light-verifier light-merkle-tree-metadata light-zero-copy light-hash-set test_cmd: | cargo test -p aligned-sized - cargo test -p light-bloom-filter --all-features cargo test -p light-hasher --all-features cargo test -p light-compressed-account --all-features cargo test -p light-account-checks --all-features @@ -53,6 +55,10 @@ jobs: cargo test -p light-merkle-tree-metadata --all-features cargo test -p light-zero-copy --features std cargo test -p light-hash-set --all-features + - name: program-libs-slow + packages: light-bloom-filter light-indexed-merkle-tree light-batched-merkle-tree + test_cmd: | + cargo test -p light-bloom-filter --all-features cargo test -p light-indexed-merkle-tree --all-features cargo test -p light-batched-merkle-tree --all-features -- --test test_e2e - name: sdk-libs @@ -87,6 +93,8 @@ jobs: - name: Setup and build uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" - name: Build CLI run: | diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000000..f3b779164e --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,34 @@ +Running `install.sh` is the first thing you should do after cloning this monorepo. + +```bash +# Install all components (default) +./scripts/install.sh + +# Install with all proving keys +./scripts/install.sh --full-keys + +# Skip components (only use if you know what you are doing) +./scripts/install.sh --skip-components "redis,keys,go" +``` + +## Components + +- `go` - Golang +- `rust` - Rust toolchain +- `node` - Node.js runtime +- `pnpm` - Package manager +- `solana` - Solana CLI tools +- `anchor` - Anchor +- `jq` - JSON processor +- `keys` - Gnark proving keys +- `dependencies` - all PNPM deps +- `redis` - Redis server + +## CI Usage + +```yaml +- name: Setup and build + uses: ./.github/actions/setup-and-build + with: + skip-components: "redis" +``` diff --git a/cli/scripts/buildProver.sh b/cli/scripts/buildProver.sh index 81a15019bf..d71ec31974 100755 --- a/cli/scripts/buildProver.sh +++ b/cli/scripts/buildProver.sh @@ -14,6 +14,13 @@ if [ ! -e "$out_dir" ]; then mkdir -p "$out_dir" fi +# Check if proving keys exist before copying +if [ ! -d "${gnark_dir}/proving-keys" ] || [ -z "$(ls -A "${gnark_dir}/proving-keys" 2>/dev/null)" ]; then + echo "ERROR: Proving keys not found at ${gnark_dir}/proving-keys" + echo "Please run: ./prover/server/scripts/download_keys.sh light" + exit 1 +fi + cp -r "${gnark_dir}/proving-keys" "$out_dir" cd "$gnark_dir" diff --git a/cli/src/utils/initTestEnv.ts b/cli/src/utils/initTestEnv.ts index 47d16b6b35..a2356de889 100644 --- a/cli/src/utils/initTestEnv.ts +++ b/cli/src/utils/initTestEnv.ts @@ -149,7 +149,14 @@ export async function initTestEnv({ const config = getConfig(); config.proverUrl = `http://127.0.0.1:${proverPort}`; setConfig(config); - await startProver(proverPort, proverRunMode, circuits); + try { + // TODO: check if using redisUrl is better here. + await startProver(proverPort, proverRunMode, circuits); + } catch (error) { + console.error("Failed to start prover:", error); + // Prover logs will be automatically displayed by spawnBinary in process.ts + throw error; + } } } diff --git a/cli/src/utils/process.ts b/cli/src/utils/process.ts index ec70832a36..6c824fce94 100644 --- a/cli/src/utils/process.ts +++ b/cli/src/utils/process.ts @@ -7,6 +7,47 @@ import { promisify } from "util"; import axios from "axios"; const waitOn = require("wait-on"); +const readdir = promisify(fs.readdir); +const readFile = promisify(fs.readFile); + +/** + * Logs the contents of prover log files in test-ledger dir. + */ +export async function logProverFileContents() { + const testLedgerDir = path.join(__dirname, "../..", "test-ledger"); + + try { + if (!fs.existsSync(testLedgerDir)) { + console.log("test-ledger directory does not exist"); + return; + } + + const files = await readdir(testLedgerDir); + + const proverFiles = files.filter((file) => file.includes("prover")); + + if (proverFiles.length === 0) { + console.log("No prover log files found in test-ledger directory"); + return; + } + + for (const file of proverFiles) { + const filePath = path.join(testLedgerDir, file); + console.log(`\n========== Contents of ${file} ==========`); + + try { + const contents = await readFile(filePath, "utf8"); + console.log(contents); + console.log(`========== End of ${file} ==========\n`); + } catch (error) { + console.error(`Error reading ${file}:`, error); + } + } + } catch (error) { + console.error("Error accessing test-ledger directory:", error); + } +} + export async function killProcess(processName: string) { const processList = await find("name", processName); @@ -22,7 +63,6 @@ export async function killProcess(processName: string) { } } - // Double-check if processes are still running const remainingProcesses = await find("name", processName); if (remainingProcesses.length > 0) { console.warn( @@ -174,8 +214,12 @@ export function spawnBinary(command: string, args: string[] = []) { detached: true, }); - spawnedProcess.on("close", (code) => { + spawnedProcess.on("close", async (code) => { console.log(`${binaryName} process exited with code ${code}`); + if (code !== 0 && binaryName.includes("prover")) { + console.error(`Prover process failed with exit code ${code}`); + await logProverFileContents(); + } }); return spawnedProcess; diff --git a/js/compressed-token/package.json b/js/compressed-token/package.json index 9b0fab25b5..f8fc8bad4c 100644 --- a/js/compressed-token/package.json +++ b/js/compressed-token/package.json @@ -78,8 +78,12 @@ }, "scripts": { "test": "pnpm test:e2e:all", + "test:v1": "LIGHT_PROTOCOL_VERSION=V1 pnpm test", + "test:v2": "LIGHT_PROTOCOL_VERSION=V2 pnpm test", "test-all": "vitest run", "test:unit:all": "EXCLUDE_E2E=true vitest run", + "test:unit:all:v1": "LIGHT_PROTOCOL_VERSION=V1 vitest run tests/unit --reporter=verbose", + "test:unit:all:v2": "LIGHT_PROTOCOL_VERSION=V2 vitest run tests/unit --reporter=verbose", "test-all:verbose": "vitest run --reporter=verbose", "test-validator": "./../../cli/test_bin/run test-validator --prover-run-mode rpc", "test-validator-skip-prover": "./../../cli/test_bin/run test-validator --skip-prover", @@ -102,8 +106,12 @@ "test:e2e:multi-pool": "pnpm test-validator && vitest run tests/e2e/multi-pool.test.ts --reporter=verbose", "test:e2e:all": "pnpm test-validator && vitest run tests/e2e/create-mint.test.ts && vitest run tests/e2e/mint-to.test.ts && vitest run tests/e2e/transfer.test.ts && vitest run tests/e2e/delegate.test.ts && vitest run tests/e2e/transfer-delegated.test.ts && vitest run tests/e2e/multi-pool.test.ts && vitest run tests/e2e/decompress-delegated.test.ts && pnpm test-validator-skip-prover && vitest run tests/e2e/compress.test.ts && vitest run tests/e2e/compress-spl-token-account.test.ts && vitest run tests/e2e/decompress.test.ts && vitest run tests/e2e/create-token-pool.test.ts && vitest run tests/e2e/approve-and-mint-to.test.ts && vitest run tests/e2e/rpc-token-interop.test.ts && vitest run tests/e2e/rpc-multi-trees.test.ts && vitest run tests/e2e/layout.test.ts && vitest run tests/e2e/select-accounts.test.ts", "pull-idl": "../../scripts/push-compressed-token-idl.sh", - "build": "rimraf dist && pnpm build:bundle", - "build:bundle": "rollup -c", + "build": "if [ \"$LIGHT_PROTOCOL_VERSION\" = \"V2\" ]; then LIGHT_PROTOCOL_VERSION=V2 pnpm build:bundle; else LIGHT_PROTOCOL_VERSION=V1 pnpm build:bundle; fi", + "build:bundle": "rimraf dist && rollup -c", + "build:v1": "LIGHT_PROTOCOL_VERSION=V1 pnpm build:stateless:v1 && LIGHT_PROTOCOL_VERSION=V1 pnpm build:bundle", + "build:v2": "LIGHT_PROTOCOL_VERSION=V2 pnpm build:stateless:v2 && LIGHT_PROTOCOL_VERSION=V2 pnpm build:bundle", + "build:stateless:v1": "cd ../stateless.js && pnpm build:v1", + "build:stateless:v2": "cd ../stateless.js && pnpm build:v2", "format": "prettier --write .", "lint": "eslint ." }, diff --git a/js/compressed-token/rollup.config.js b/js/compressed-token/rollup.config.js index 4fac8ae5ef..f19a4b3c29 100644 --- a/js/compressed-token/rollup.config.js +++ b/js/compressed-token/rollup.config.js @@ -1,3 +1,4 @@ +/* global process */ import typescript from '@rollup/plugin-typescript'; import nodePolyfills from 'rollup-plugin-polyfill-node'; import dts from 'rollup-plugin-dts'; @@ -6,6 +7,7 @@ import commonjs from '@rollup/plugin-commonjs'; import alias from '@rollup/plugin-alias'; import json from '@rollup/plugin-json'; import terser from '@rollup/plugin-terser'; +import replace from '@rollup/plugin-replace'; const rolls = (fmt, env) => ({ input: 'src/index.ts', @@ -22,6 +24,14 @@ const rolls = (fmt, env) => ({ '@lightprotocol/stateless.js', ], plugins: [ + replace({ + preventAssignment: true, + values: { + __BUILD_VERSION__: JSON.stringify( + process.env.LIGHT_PROTOCOL_VERSION || 'V1', + ), + }, + }), json(), typescript({ target: fmt === 'es' ? 'ES2022' : 'ES2017', diff --git a/js/compressed-token/src/types.ts b/js/compressed-token/src/types.ts index 65aa9fd6b1..09ff12f1cd 100644 --- a/js/compressed-token/src/types.ts +++ b/js/compressed-token/src/types.ts @@ -73,7 +73,6 @@ export type BatchCompressInstructionData = { bump: number; }; - export type MintToInstructionData = { recipients: PublicKey[]; amounts: BN[]; diff --git a/js/compressed-token/src/utils/version-check.ts b/js/compressed-token/src/utils/version-check.ts new file mode 100644 index 0000000000..41b0bc4f83 --- /dev/null +++ b/js/compressed-token/src/utils/version-check.ts @@ -0,0 +1,40 @@ +import { featureFlags, VERSION } from '@lightprotocol/stateless.js'; + +/** + * Validates that the built version of stateless.js matches the expected version. + * Throws an error if there's a mismatch. + * + * @param expectedVersion - The version expected (defaults to LIGHT_PROTOCOL_VERSION env var or V1) + * @throws Error if the versions don't match + */ +export function validateVersionConsistency(expectedVersion?: string): void { + const expected = + expectedVersion || process.env.LIGHT_PROTOCOL_VERSION || VERSION.V1; + const actual = featureFlags.version.replace(/['"]/g, ''); + + if (actual !== expected) { + throw new Error( + `Version mismatch detected!\n` + + `Expected: ${expected} (from ${expectedVersion ? 'parameter' : 'LIGHT_PROTOCOL_VERSION env var'})\n` + + `Actual: ${actual} (from built stateless.js)\n\n` + + `This means stateless.js was built with ${actual} but you're trying to use ${expected}.\n` + + `Please rebuild both packages with the same version:\n` + + ` pnpm build:${expected.toLowerCase()}\n` + + `This command will automatically build both stateless.js and compressed-token with ${expected}.`, + ); + } +} + +/** + * Gets the current version from stateless.js + */ +export function getCurrentVersion(): string { + return featureFlags.version.replace(/['"]/g, ''); +} + +/** + * Checks if the current version is V2 + */ +export function isV2(): boolean { + return featureFlags.isV2(); +} diff --git a/js/compressed-token/tests/setup/version-check.ts b/js/compressed-token/tests/setup/version-check.ts new file mode 100644 index 0000000000..9c1fdaedba --- /dev/null +++ b/js/compressed-token/tests/setup/version-check.ts @@ -0,0 +1,16 @@ +import { validateVersionConsistency } from '../../src/utils/version-check'; + +// Only used in tests. +export default function setup() { + console.log('Checking version consistency...'); + + try { + validateVersionConsistency(); + const expectedVersion = process.env.LIGHT_PROTOCOL_VERSION || 'V1'; + console.log(`✅ Version check passed: Using ${expectedVersion}`); + } catch (error) { + console.error('❌ Version check failed:'); + console.error(error.message); + process.exit(1); + } +} diff --git a/js/compressed-token/tests/unit/version.test.ts b/js/compressed-token/tests/unit/version.test.ts new file mode 100644 index 0000000000..fd6b369116 --- /dev/null +++ b/js/compressed-token/tests/unit/version.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect, beforeAll } from 'vitest'; +import { featureFlags, VERSION } from '@lightprotocol/stateless.js'; + +describe('Versioning', () => { + beforeAll(() => { + const expectedVersion = + process.env.LIGHT_PROTOCOL_VERSION || VERSION.V1; + const actualVersion = featureFlags.version.replace(/['"]/g, ''); + + if (actualVersion !== expectedVersion) { + throw new Error( + `Version mismatch detected!\n` + + `Expected: ${expectedVersion} (from LIGHT_PROTOCOL_VERSION env var)\n` + + `Actual: ${actualVersion} (from built stateless.js)\n\n` + + `This means stateless.js was built with ${actualVersion} but you're trying to test with ${expectedVersion}.\n` + + `Please rebuild stateless.js with the correct version:\n` + + ` cd ../stateless.js && pnpm build:${expectedVersion.toLowerCase()}\n` + + `Or use the compressed-token build command that handles this automatically:\n` + + ` pnpm build:${expectedVersion.toLowerCase()}`, + ); + } + }); + + it('should use version from stateless.js', () => { + console.log('Current version from stateless.js:', featureFlags.version); + console.log('isV2() from stateless.js:', featureFlags.isV2()); + console.log( + 'Environment variable:', + process.env.LIGHT_PROTOCOL_VERSION, + ); + + expect(featureFlags.version).toBeDefined(); + const actualVersion = featureFlags.version.replace(/['"]/g, ''); + expect([VERSION.V1, VERSION.V2]).toContain(actualVersion); + }); + + it('should respect LIGHT_PROTOCOL_VERSION environment variable', () => { + const expectedVersion = + process.env.LIGHT_PROTOCOL_VERSION || VERSION.V1; + const actualVersion = featureFlags.version.replace(/['"]/g, ''); + expect(actualVersion).toBe(expectedVersion); + }); + + it('isV2() should return correct value', () => { + const actualVersion = featureFlags.version.replace(/['"]/g, ''); + const expectedIsV2 = actualVersion === VERSION.V2; + expect(featureFlags.isV2()).toBe(expectedIsV2); + }); + + it('compressed-token should use the same version as stateless.js', () => { + const actualVersion = featureFlags.version.replace(/['"]/g, ''); + const isV2 = featureFlags.isV2(); + + if (process.env.LIGHT_PROTOCOL_VERSION === 'V2') { + expect(actualVersion).toBe(VERSION.V2); + expect(isV2).toBe(true); + } else { + expect(actualVersion).toBe(VERSION.V1); + expect(isV2).toBe(false); + } + }); +}); diff --git a/js/compressed-token/vitest.config.ts b/js/compressed-token/vitest.config.ts index 26a728edbe..3bbbf1ad7c 100644 --- a/js/compressed-token/vitest.config.ts +++ b/js/compressed-token/vitest.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ testTimeout: 350000, hookTimeout: 100000, reporters: ['verbose'], + globalSetup: './tests/setup/version-check.ts', }, define: { 'import.meta.vitest': false, diff --git a/js/stateless.js/.eslintignore b/js/stateless.js/.eslintignore index d2939734e4..26caad229d 100644 --- a/js/stateless.js/.eslintignore +++ b/js/stateless.js/.eslintignore @@ -1,3 +1,4 @@ node_modules lib dist +test-version.js diff --git a/js/stateless.js/BUILD.md b/js/stateless.js/BUILD.md new file mode 100644 index 0000000000..a3a6a37a08 --- /dev/null +++ b/js/stateless.js/BUILD.md @@ -0,0 +1,16 @@ +Stateless.js compiles with V1 API by default. To switch over to V2 endpoints (with backward compatibility for V1 state), run: + +```bash +pnpm build:v2 +# or +LIGHT_PROTOCOL_VERSION=V2 pnpm build +``` + +## Usage in Code + +```typescript +// From rpc.ts +const endpoint = featureFlags.isV2() + ? versionedEndpoint('getCompressedAccountV2') + : versionedEndpoint('getCompressedAccount'); +``` diff --git a/js/stateless.js/package.json b/js/stateless.js/package.json index f5430e0bb9..b8a18226a2 100644 --- a/js/stateless.js/package.json +++ b/js/stateless.js/package.json @@ -87,7 +87,11 @@ "scripts": { "test": "pnpm test:unit:all && pnpm test:e2e:all", "test-all": "vitest run", + "test:v1": "LIGHT_PROTOCOL_VERSION=V1 pnpm test", + "test:v2": "LIGHT_PROTOCOL_VERSION=V2 pnpm test", "test:unit:all": "vitest run tests/unit --reporter=verbose", + "test:unit:all:v1": "LIGHT_PROTOCOL_VERSION=V1 vitest run tests/unit --reporter=verbose", + "test:unit:all:v2": "LIGHT_PROTOCOL_VERSION=V2 vitest run tests/unit --reporter=verbose", "test:unit:tree-info": "vitest run tests/unit/utils/tree-info.test.ts --reporter=verbose", "test:conversions": "vitest run tests/unit/utils/conversion.test.ts --reporter=verbose", "test-validator": "./../../cli/test_bin/run test-validator --prover-run-mode rpc", @@ -96,7 +100,7 @@ "test:e2e:compress": "pnpm test-validator && vitest run tests/e2e/compress.test.ts --reporter=verbose", "test:e2e:test-rpc": "pnpm test-validator && vitest run tests/e2e/test-rpc.test.ts --reporter=verbose --bail=1", "test:e2e:rpc-interop": "pnpm test-validator && vitest run tests/e2e/rpc-interop.test.ts --reporter=verbose --bail=1", - "test:e2e:rpc-multi-trees": "pnpm test-validator && vitest run tests/e2e/rpc-multi-trees.test.ts", + "test:e2e:rpc-multi-trees": "pnpm test-validator && vitest run tests/e2e/rpc-multi-trees.test.ts --reporter=verbose --bail=1", "test:e2e:browser": "pnpm playwright test", "test:e2e:all": "pnpm test-validator && vitest run tests/e2e/test-rpc.test.ts && vitest run tests/e2e/compress.test.ts && vitest run tests/e2e/transfer.test.ts && vitest run tests/e2e/rpc-interop.test.ts && pnpm test-validator-skip-prover && vitest run tests/e2e/rpc-multi-trees.test.ts && vitest run tests/e2e/layout.test.ts && vitest run tests/e2e/safe-conversion.test.ts", "test:index": "vitest run tests/e2e/program.test.ts", @@ -105,8 +109,10 @@ "test:verbose": "vitest run --reporter=verbose", "test:testnet": "vitest run tests/e2e/testnet.test.ts --reporter=verbose", "pull-idls": "../../scripts/push-stateless-js-idls.sh && ../../scripts/push-compressed-token-idl.sh", - "build": "rimraf dist && pnpm build:bundle", - "build:bundle": "rollup -c", + "build": "if [ \"$LIGHT_PROTOCOL_VERSION\" = \"V2\" ]; then LIGHT_PROTOCOL_VERSION=V2 pnpm build:bundle; else LIGHT_PROTOCOL_VERSION=V1 pnpm build:bundle; fi", + "build:bundle": "rimraf dist && rollup -c", + "build:v1": "LIGHT_PROTOCOL_VERSION=V1 pnpm build:bundle", + "build:v2": "LIGHT_PROTOCOL_VERSION=V2 pnpm build:bundle", "format": "prettier --write .", "lint": "eslint ." }, diff --git a/js/stateless.js/rollup.config.js b/js/stateless.js/rollup.config.js index c78a4a1ea1..c42b978ea9 100644 --- a/js/stateless.js/rollup.config.js +++ b/js/stateless.js/rollup.config.js @@ -1,11 +1,13 @@ +/* global process */ import typescript from '@rollup/plugin-typescript'; import nodePolyfills from 'rollup-plugin-polyfill-node'; import dts from 'rollup-plugin-dts'; import resolve from '@rollup/plugin-node-resolve'; import commonjs from '@rollup/plugin-commonjs'; import terser from '@rollup/plugin-terser'; - +import replace from '@rollup/plugin-replace'; import json from '@rollup/plugin-json'; + const rolls = (fmt, env) => ({ input: 'src/index.ts', output: { @@ -16,6 +18,14 @@ const rolls = (fmt, env) => ({ }, external: ['@solana/web3.js'], plugins: [ + replace({ + preventAssignment: true, + values: { + __BUILD_VERSION__: JSON.stringify( + process.env.LIGHT_PROTOCOL_VERSION || 'V1', + ), + }, + }), typescript({ target: fmt === 'es' ? 'ES2022' : 'ES2017', outDir: `dist/${fmt}/${env}`, diff --git a/js/stateless.js/src/constants.ts b/js/stateless.js/src/constants.ts index af9041e6f4..b34cfaaff5 100644 --- a/js/stateless.js/src/constants.ts +++ b/js/stateless.js/src/constants.ts @@ -14,8 +14,24 @@ export enum VERSION { * Feature flags. Only use if you know what you are doing. */ export const featureFlags = { - version: VERSION.V1, - isV2: () => featureFlags.version.toUpperCase() === 'V2', + version: ((): VERSION => { + // Check if we're in a build environment (replaced by rollup) + // eslint-disable-next-line no-constant-condition + if ('__BUILD_VERSION__' !== '__BUILD_' + 'VERSION__') { + return '__BUILD_VERSION__' as VERSION; + } + // Otherwise, check runtime environment variable (for tests) + if ( + typeof process !== 'undefined' && + process.env?.LIGHT_PROTOCOL_VERSION + ) { + return process.env.LIGHT_PROTOCOL_VERSION as VERSION; + } + // Default to V1 + return VERSION.V1; + })(), + isV2: () => + featureFlags.version.replace(/['"]/g, '').toUpperCase() === 'V2', }; /** @@ -24,7 +40,7 @@ export const featureFlags = { * or 'getCompressedAccountV2' (V2) */ export const versionedEndpoint = (base: string) => - featureFlags.version.toUpperCase() === 'V1' ? base : `${base}V2`; + featureFlags.isV2() ? `${base}V2` : base; export const FIELD_SIZE = new BN( '21888242871839275222246405745257275088548364400416034343698204186575808495617', diff --git a/js/stateless.js/src/globals.d.ts b/js/stateless.js/src/globals.d.ts new file mode 100644 index 0000000000..b2ba4b3072 --- /dev/null +++ b/js/stateless.js/src/globals.d.ts @@ -0,0 +1 @@ +declare const __BUILD_VERSION__: 'V1' | 'V2'; diff --git a/js/stateless.js/src/rpc.ts b/js/stateless.js/src/rpc.ts index 9d57305700..9bc3660e5f 100644 --- a/js/stateless.js/src/rpc.ts +++ b/js/stateless.js/src/rpc.ts @@ -339,7 +339,7 @@ export const proverRequest = async ( method: 'inclusion' | 'new-address' | 'combined', params: any = [], log = false, - publicInputHash: BN | undefined = undefined, + _publicInputHash: BN | undefined = undefined, // Not supported. ): Promise => { let logMsg: string = ''; @@ -1065,7 +1065,7 @@ export class Rpc extends Connection implements CompressionApiInterface { stateTreeInfo, bn(item.hash.toArray('be', 32)), item.leafIndex, - false, + featureFlags.isV2() ? item.proveByIndex : false, ), item.owner, bn(item.lamports), @@ -1892,7 +1892,7 @@ export class Rpc extends Connection implements CompressionApiInterface { .concat(value.addresses.map((r: any) => r.rootIndex)), proveByIndices: value.accounts .map((r: any) => r.rootIndex.proveByIndex) - .concat(value.addresses.map((r: any) => false)), + .concat(value.addresses.map((r: any) => false)), // addresses.proveByIndex is always false. treeInfos: value.accounts .map((r: any) => r.merkleContext) .concat( diff --git a/js/stateless.js/test-version.js b/js/stateless.js/test-version.js new file mode 100644 index 0000000000..9f76abe3ec --- /dev/null +++ b/js/stateless.js/test-version.js @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +// Test script to verify the compiled version is correctly set +import { featureFlags } from './dist/cjs/node/index.cjs'; + +console.log('Testing build version...'); +console.log('featureFlags.version:', featureFlags.version); +console.log('featureFlags.isV2():', featureFlags.isV2()); + +const expectedVersion = process.env.EXPECTED_VERSION || 'V1'; +const actualVersion = featureFlags.version.replace(/['"]/g, ''); +if (actualVersion === expectedVersion) { + console.log(`✅ Success: Version is correctly set to ${expectedVersion}`); + process.exit(0); +} else { + console.error( + `❌ Error: Expected version ${expectedVersion} but got ${actualVersion}`, + ); + process.exit(1); +} diff --git a/js/stateless.js/tests/unit/version.test.ts b/js/stateless.js/tests/unit/version.test.ts new file mode 100644 index 0000000000..76285d96fd --- /dev/null +++ b/js/stateless.js/tests/unit/version.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect } from 'vitest'; +import { featureFlags, VERSION } from '../../src/constants'; + +describe('Version System', () => { + it('should have version set', () => { + console.log('Current version:', featureFlags.version); + console.log('isV2():', featureFlags.isV2()); + console.log( + 'Environment variable:', + process.env.LIGHT_PROTOCOL_VERSION, + ); + + expect(featureFlags.version).toBeDefined(); + expect([VERSION.V1, VERSION.V2]).toContain(featureFlags.version); + }); + + it('should respect LIGHT_PROTOCOL_VERSION environment variable', () => { + const expectedVersion = + process.env.LIGHT_PROTOCOL_VERSION || VERSION.V1; + expect(featureFlags.version).toBe(expectedVersion); + }); + + it('isV2() should return correct value', () => { + const expectedIsV2 = featureFlags.version === VERSION.V2; + expect(featureFlags.isV2()).toBe(expectedIsV2); + }); +}); diff --git a/nx.json b/nx.json index 3c8bc1b96c..e5f23b2f79 100644 --- a/nx.json +++ b/nx.json @@ -9,7 +9,7 @@ }, "targetDefaults": { "build": { - "cache": false, + "cache": true, "inputs": [ "noMarkdown", "^noMarkdown", @@ -22,11 +22,14 @@ "outputs": [ "{workspaceRoot}/target/deploy", "{workspaceRoot}/target/idl", - "{workspaceRoot}/target/types" + "{workspaceRoot}/target/types", + "{projectRoot}/dist", + "{projectRoot}/lib", + "{projectRoot}/bin" ] }, "test": { - "cache": false, + "cache": true, "inputs": [ "noMarkdown", "^noMarkdown", @@ -65,5 +68,17 @@ "^noTestLedger" ] } + }, + "daemon": { + "enabled": false + }, + "tasksRunnerOptions": { + "default": { + "runner": "nx/tasks-runners/default", + "options": { + "cacheableOperations": ["build", "test"], + "cacheDirectory": ".nx/cache" + } + } } } diff --git a/scripts/devenv.sh b/scripts/devenv.sh index 66c3867751..0349400d9c 100755 --- a/scripts/devenv.sh +++ b/scripts/devenv.sh @@ -2,7 +2,10 @@ # Command to deactivate the devenv. It sets the old environment variables. deactivate () { - redis-stop + # Only try to stop redis if the command exists + if command -v redis-stop >/dev/null 2>&1; then + redis-stop 2>/dev/null || true + fi PS1="${LIGHT_PROTOCOL_OLD_PS1}" RUSTUP_HOME="${LIGHT_PROTOCOL_OLD_RUSTUP_HOME}" @@ -24,15 +27,27 @@ deactivate () { if [ -z "${LIGHT_PROTOCOL_DEVENV:-}" ]; then LIGHT_PROTOCOL_DEVENV=1 else - return + return 2>/dev/null || exit 0 fi # The root of the git repository. -LIGHT_PROTOCOL_TOPLEVEL="`git rev-parse --show-toplevel`" +LIGHT_PROTOCOL_TOPLEVEL="$(git rev-parse --show-toplevel 2>/dev/null)" +if [ -z "$LIGHT_PROTOCOL_TOPLEVEL" ]; then + echo "Error: Not in a git repository" >&2 + return 1 2>/dev/null || exit 1 +fi + +# Verify the .local directory exists +if [ ! -d "${LIGHT_PROTOCOL_TOPLEVEL}/.local" ]; then + echo "Error: .local directory not found. Please run ./scripts/install.sh first" >&2 + return 1 2>/dev/null || exit 1 +fi -# Shell prompt. -LIGHT_PROTOCOL_OLD_PS1="${PS1:-}" -PS1="[🧢 Light Protocol devenv] ${PS1:-}" +# Shell prompt (only set if PS1 exists - not in CI) +if [ -n "${PS1:-}" ]; then + LIGHT_PROTOCOL_OLD_PS1="${PS1}" + PS1="[🧢 Light Protocol devenv] ${PS1}" +fi # Ensure that our rustup environment is used. LIGHT_PROTOCOL_OLD_RUSTUP_HOME="${RUSTUP_HOME:-}" @@ -54,8 +69,10 @@ PATH="${LIGHT_PROTOCOL_TOPLEVEL}/.local/npm-global/bin:${PATH}" # Remove the original Rust-related PATH entries PATH=$(echo "$PATH" | tr ':' '\n' | grep -vE "/.rustup/|/.cargo/" | tr '\n' ':' | sed 's/:$//') -# Define alias of `light` to use the CLI built from source. -alias light="${LIGHT_PROTOCOL_TOPLEVEL}/cli/test_bin/run" +# Define alias of `light` to use the CLI built from source (only if not in CI) +if [ -z "${CI:-}" ]; then + alias light="${LIGHT_PROTOCOL_TOPLEVEL}/cli/test_bin/run" +fi # Define GOROOT for Go. export GOROOT="${LIGHT_PROTOCOL_TOPLEVEL}/.local/go" @@ -63,15 +80,35 @@ export GOROOT="${LIGHT_PROTOCOL_TOPLEVEL}/.local/go" # Ensure Rust binaries are in PATH PATH="${CARGO_HOME}/bin:${PATH}" -# Export the modified PATH +# Export all critical environment variables export PATH +export RUSTUP_HOME +export CARGO_HOME +export NPM_CONFIG_PREFIX +export LIGHT_PROTOCOL_TOPLEVEL +export LIGHT_PROTOCOL_DEVENV -export REDIS_URL="redis://localhost:6379" +# Set Redis URL if not already set +export REDIS_URL="${REDIS_URL:-redis://localhost:6379}" # Enable small_ix feature by default in devenv -export CARGO_FEATURES="small_ix" +export CARGO_FEATURES="${CARGO_FEATURES:-small_ix}" +# macOS-specific settings if [[ "$(uname)" == "Darwin" ]]; then LIGHT_PROTOCOL_OLD_CPATH="${CPATH:-}" export CPATH="/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include:${CPATH:-}" fi + +# Validate critical tools are available (only warn, don't fail) +if [ -z "${LIGHT_PROTOCOL_SKIP_VALIDATION:-}" ]; then + if ! command -v cargo >/dev/null 2>&1; then + echo "Warning: cargo not found in PATH. Run ./scripts/install.sh to install Rust." >&2 + fi + if ! command -v go >/dev/null 2>&1; then + echo "Warning: go not found in PATH. Run ./scripts/install.sh to install Go." >&2 + fi + if ! command -v node >/dev/null 2>&1; then + echo "Warning: node not found in PATH. Run ./scripts/install.sh to install Node.js." >&2 + fi +fi diff --git a/scripts/install.sh b/scripts/install.sh index bfed6da5a3..08ae190633 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -81,7 +81,8 @@ download() { } install_go() { - if ! is_installed "go"; then + # Check if Go is actually installed, not just logged + if [ ! -d "${PREFIX}/go" ] || [ ! -f "${PREFIX}/go/bin/go" ]; then echo "Installing Go..." local version=$(get_version "go") local suffix=$(get_suffix "go") @@ -90,11 +91,14 @@ install_go() { tar -xzf "${PREFIX}/go.tar.gz" -C "${PREFIX}" rm "${PREFIX}/go.tar.gz" log "go" + else + echo "Go already installed, skipping..." fi } install_rust() { - if ! is_installed "rust"; then + # Check if Rust is actually installed + if [ ! -d "${PREFIX}/rustup" ] || [ ! -d "${PREFIX}/cargo" ] || [ ! -f "${PREFIX}/cargo/bin/cargo" ]; then echo "Installing Rust..." export RUSTUP_HOME="${PREFIX}/rustup" export CARGO_HOME="${PREFIX}/cargo" @@ -105,11 +109,14 @@ install_rust() { cargo install cargo-expand --locked cargo install --git https://github.com/helius-labs/photon.git --rev cf58facb4e0521843e3afd21d09d8e7e7f772140 --locked log "rust" + else + echo "Rust already installed, skipping..." fi } install_node() { - if ! is_installed "node"; then + # Check if Node is actually installed + if [ ! -f "${PREFIX}/bin/node" ] || [ ! -f "${PREFIX}/bin/npm" ]; then echo "Installing Node.js..." local version=$(get_version "node") local suffix=$(get_suffix "node") @@ -118,11 +125,14 @@ install_node() { tar -xzf "${PREFIX}/node.tar.gz" -C "${PREFIX}" --strip-components 1 rm "${PREFIX}/node.tar.gz" log "node" + else + echo "Node.js already installed, skipping..." fi } install_pnpm() { - if ! is_installed "pnpm"; then + # Check if pnpm is actually installed + if [ ! -f "${PREFIX}/bin/pnpm" ]; then echo "Installing pnpm..." local version=$(get_version "pnpm") local suffix=$(get_suffix "pnpm") @@ -130,11 +140,14 @@ install_pnpm() { download "$url" "${PREFIX}/bin/pnpm" chmod +x "${PREFIX}/bin/pnpm" log "pnpm" + else + echo "pnpm already installed, skipping..." fi } install_solana() { - if ! is_installed "solana"; then + # Check if Solana is actually installed + if [ ! -f "${PREFIX}/bin/solana" ] || [ ! -f "${PREFIX}/bin/solana-keygen" ]; then echo "Installing Solana..." local version=$(get_version "solana") local suffix=$(get_suffix "solana") @@ -143,52 +156,67 @@ install_solana() { tar -xjf "${PREFIX}/solana-release.tar.bz2" -C "${PREFIX}/bin" --strip-components 2 rm "${PREFIX}/solana-release.tar.bz2" log "solana" + else + echo "Solana already installed, skipping..." fi } install_anchor() { - if ! is_installed "anchor"; then + # Check if Anchor is actually installed + if [ ! -f "${PREFIX}/bin/anchor" ]; then echo "Installing Anchor..." local version=$(get_version "anchor") local suffix=$(get_suffix "anchor") local url="https://github.com/Lightprotocol/binaries/releases/download/${version}/anchor-${suffix}" download "$url" "${PREFIX}/bin/anchor" log "anchor" + else + echo "Anchor already installed, skipping..." fi } install_jq() { - if ! is_installed "jq"; then + # Check if jq is actually installed + if [ ! -f "${PREFIX}/bin/jq" ]; then echo "Installing jq..." local version=$(get_version "jq") local suffix=$(get_suffix "jq") local url="https://github.com/jqlang/jq/releases/download/${version}/${suffix}" download "$url" "${PREFIX}/bin/jq" log "jq" + else + echo "jq already installed, skipping..." fi } download_gnark_keys() { - if ! is_installed "gnark_keys"; then + ROOT_DIR="$(git rev-parse --show-toplevel)" + # Always check if keys actually exist, not just the install log + if [ ! -d "${ROOT_DIR}/prover/server/proving-keys" ] || [ -z "$(ls -A "${ROOT_DIR}/prover/server/proving-keys" 2>/dev/null)" ]; then echo "Downloading gnark keys..." - ROOT_DIR="$(git rev-parse --show-toplevel)" - "${ROOT_DIR}/prover/server/scripts/download_keys.sh" "$key_type" + "${ROOT_DIR}/prover/server/scripts/download_keys.sh" "$1" log "gnark_keys" + else + echo "Gnark keys already exist, skipping download..." fi } install_dependencies() { - if ! is_installed "dependencies"; then + # Check if node_modules exists and has content + if [ ! -d "node_modules" ] || [ -z "$(ls -A node_modules 2>/dev/null)" ]; then echo "Installing dependencies..." export PATH="${PREFIX}/bin:${PATH}" pnpm install log "dependencies" + else + echo "Dependencies already installed, skipping..." fi } install_redis() { - if ! is_installed "redis"; then + # Check if Redis is actually installed + if [ ! -f "${PREFIX}/bin/redis-server" ] || [ ! -f "${PREFIX}/bin/redis-cli" ]; then echo "Installing Redis..." local version=$(get_version "redis") local url="http://download.redis.io/releases/redis-${version}.tar.gz" @@ -326,6 +354,8 @@ EOF chmod +x "${PREFIX}/bin/redis-start" "${PREFIX}/bin/redis-stop" "${PREFIX}/bin/redis-status" mkdir -p "${PREFIX}/etc" "${PREFIX}/var" log "redis" + else + echo "Redis already installed, skipping..." fi } @@ -337,6 +367,8 @@ main() { # Parse command line arguments local key_type="light" local reset_log=true + local skip_components="" + while [[ $# -gt 0 ]]; do case $1 in --full-keys) @@ -347,8 +379,18 @@ main() { reset_log=false shift ;; + --skip-components) + if [ -z "$2" ] || [[ "$2" == --* ]]; then + echo "Error: --skip-components requires a value" + exit 1 + fi + skip_components="$2" + shift 2 + ;; *) echo "Unknown option: $1" + echo "Usage: $0 [--full-keys] [--no-reset] [--skip-components ]" + echo "Components that can be skipped: go,rust,node,pnpm,solana,anchor,jq,keys,dependencies,redis" exit 1 ;; esac @@ -358,18 +400,28 @@ main() { rm -f "$INSTALL_LOG" fi - install_go - install_rust - install_node - install_pnpm - install_solana - install_anchor - install_jq - download_gnark_keys "$key_type" - install_dependencies - install_redis + # Helper function to check if component should be skipped + should_skip() { + local component=$1 + [[ ",$skip_components," == *",$component,"* ]] + } + + # Install components unless explicitly skipped + should_skip "go" || install_go + should_skip "rust" || install_rust + should_skip "node" || install_node + should_skip "pnpm" || install_pnpm + should_skip "solana" || install_solana + should_skip "anchor" || install_anchor + should_skip "jq" || install_jq + should_skip "keys" || download_gnark_keys "$key_type" + should_skip "dependencies" || install_dependencies + should_skip "redis" || install_redis echo "✨ Light Protocol development dependencies installed" + if [ -n "$skip_components" ]; then + echo " Skipped components: $skip_components" + fi } -main +main "$@"