From 1c0af0dae29c026559a049bd5535af32c84719f2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 2 Feb 2026 13:48:08 +0100 Subject: [PATCH 1/2] workflow adjustments for rocksdb precompile --- .github/verify-rocksdb-static.sh | 3 + .github/workflows/ci.yml | 75 +++++++++++++++++++++++ .github/workflows/nightly.yml | 31 ++++++++++ .github/workflows/publish-debian-all.yml | 5 ++ .github/workflows/publish-debian.yml | 5 ++ .github/workflows/rocksdb-static.yml | 74 ++++++++++++++++++++++ Cargo.lock | 1 + crates/proto/Cargo.toml | 9 +-- crates/proto/build.rs | 2 + crates/rocksdb-cxx-linkage-fix/src/lib.rs | 3 +- 10 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 .github/verify-rocksdb-static.sh create mode 100644 .github/workflows/rocksdb-static.yml diff --git a/.github/verify-rocksdb-static.sh b/.github/verify-rocksdb-static.sh new file mode 100644 index 0000000000..8764e91dcb --- /dev/null +++ b/.github/verify-rocksdb-static.sh @@ -0,0 +1,3 @@ +set -euo pipefail +test -f rocksdb-artifact/librocksdb.a +test -f rocksdb-artifact/liblz4.a diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 016aeba77a..9c21685432 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,12 @@ env: RUST_CACHE_KEY: rust-cache-2026.02.02 # Reduce cache usage by removing debug information. CARGO_PROFILE_DEV_DEBUG: 0 + ROCKSDB_STATIC: true + ROCKSDB_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + ROCKSDB_COMPILE: false + LZ4_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + LZ4_STATIC: true + LIBZ_SYS_STATIC: 1 # Limits workflow concurrency to only the latest commit in the PR. concurrency: @@ -36,6 +42,12 @@ concurrency: cancel-in-progress: true jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ubuntu-24.04 + checkout-ref: ${{ github.ref }} + # =============================================================================================== # Conventional builds, lints and tests that re-use a single cache for efficiency # =============================================================================================== @@ -43,12 +55,20 @@ jobs: # Normal cargo build that populates a cache for all subsequent jobs to re-use. build: runs-on: ubuntu-24.04 + needs: [rocksdb-static] steps: - uses: actions/checkout@v6 - name: Cleanup large tools for build space uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -65,6 +85,15 @@ jobs: needs: [build] steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -81,6 +110,15 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: taiki-e/install-action@v2 @@ -105,6 +143,15 @@ jobs: - uses: actions/checkout@v6 - name: Cleanup large tools for build space uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -124,6 +171,15 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - name: Install protobuf @@ -148,6 +204,15 @@ jobs: DATA_DIR: /tmp/store steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -194,8 +259,18 @@ jobs: client-wasm: name: wasm targets runs-on: ubuntu-24.04 + needs: [rocksdb-static] steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - name: cargo build diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a5d6e3cae1..806271bcbd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,12 +11,27 @@ on: permissions: contents: read +env: + ROCKSDB_STATIC: true + ROCKSDB_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + ROCKSDB_COMPILE: false + LZ4_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + LZ4_STATIC: true + LIBZ_SYS_STATIC: 1 + jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ubuntu-24.04 + checkout-ref: refs/heads/next + # Run tests on the beta channel to provide feedback for Rust team. beta-test: name: test on beta channel runs-on: ubuntu-24.04 timeout-minutes: 30 + needs: rocksdb-static steps: - uses: actions/checkout@v6 with: @@ -25,6 +40,13 @@ jobs: uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup install beta && rustup default beta - uses: taiki-e/install-action@v2 @@ -39,6 +61,7 @@ jobs: check-features: name: feature combinations runs-on: ubuntu-24.04 + needs: rocksdb-static steps: - uses: actions/checkout@v6 with: @@ -47,6 +70,13 @@ jobs: uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Install rust run: rustup update --no-self-update - name: Install cargo-hack @@ -58,6 +88,7 @@ jobs: msrv: name: msrv check runs-on: ubuntu-24.04 + needs: rocksdb-static steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/publish-debian-all.yml b/.github/workflows/publish-debian-all.yml index 76e65d0eb7..e45ddc65b1 100644 --- a/.github/workflows/publish-debian-all.yml +++ b/.github/workflows/publish-debian-all.yml @@ -19,6 +19,11 @@ permissions: contents: write jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + checkout-ref: ${{ github.ref }} publish-node: name: Publish Node ${{ matrix.arch }} Debian strategy: diff --git a/.github/workflows/publish-debian.yml b/.github/workflows/publish-debian.yml index d17d065325..1085121681 100644 --- a/.github/workflows/publish-debian.yml +++ b/.github/workflows/publish-debian.yml @@ -47,6 +47,11 @@ permissions: contents: write jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + checkout-ref: ${{ github.ref }} publish: name: Publish ${{ inputs.package }} ${{ matrix.arch }} Debian strategy: diff --git a/.github/workflows/rocksdb-static.yml b/.github/workflows/rocksdb-static.yml new file mode 100644 index 0000000000..5bb30c39da --- /dev/null +++ b/.github/workflows/rocksdb-static.yml @@ -0,0 +1,74 @@ +name: rocksdb-static + +on: + workflow_call: + inputs: + runs-on: + required: true + type: string + checkout-ref: + required: false + type: string + +permissions: + contents: read + +env: + # Reduce cache usage by removing debug information. + CARGO_PROFILE_DEV_DEBUG: 0 + +jobs: + rocksdb-static: + name: rocksdb-static + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 30 + env: + ROCKSDB_RUST_VERSION: "" + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.checkout-ref || github.ref }} + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Rustup + run: rustup update --no-self-update + - name: Resolve RocksDB version + run: | + set -euo pipefail + rocksdb_version=$(cargo tree --locked --depth=0 -i librocksdb-sys | tail -n 1 | awk '{print $2}' | sed 's/^v//') + if [ -z "$rocksdb_version" ]; then + echo "::error::Unable to determine librocksdb-sys version from cargo tree." + exit 1 + fi + echo "ROCKSDB_RUST_VERSION=$rocksdb_version" >> $GITHUB_ENV + - name: RocksDB cache date + id: rocksdb-date + run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT + - name: Cache RocksDB static library + id: rocksdb-cache + uses: actions/cache@v4 + with: + path: rocksdb-artifact + key: rocksdb-${{ runner.os }}-${{ runner.arch }}-${{ env.ROCKSDB_RUST_VERSION }}-${{ steps.rocksdb-date.outputs.date }}-${{ hashFiles('Cargo.lock') }} + - name: Build rocksdb.a + if: steps.rocksdb-cache.outputs.cache-hit != 'true' + run: | + set -euo pipefail + CARGO_TARGET_DIR=rocksdb-target cargo build -p librocksdb-sys@${{ env.ROCKSDB_RUST_VERSION }} --locked + mkdir -p rocksdb-artifact + for lib in librocksdb.a liblz4.a; do + lib_path="$(find rocksdb-target -name "$lib" -print -quit)" + if [ -z "$lib_path" ]; then + echo "::error::Missing $lib." + exit 1 + fi + cp "$lib_path" "rocksdb-artifact/$lib" + done + - name: Upload RocksDB static library + uses: actions/upload-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + if-no-files-found: error diff --git a/Cargo.lock b/Cargo.lock index 65360401b2..d41133072d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2850,6 +2850,7 @@ dependencies = [ "http", "miden-node-grpc-error-macro", "miden-node-proto-build", + "miden-node-rocksdb-cxx-linkage-fix", "miden-node-utils", "miden-protocol", "miden-standards", diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index 6d3589ca3d..2e9767f887 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -33,7 +33,8 @@ assert_matches = { workspace = true } proptest = { version = "1.7" } [build-dependencies] -fs-err = { workspace = true } -miden-node-proto-build = { features = ["internal"], workspace = true } -miette = { version = "7.6" } -tonic-prost-build = { workspace = true } +fs-err = { workspace = true } +miden-node-proto-build = { features = ["internal"], workspace = true } +miden-node-rocksdb-cxx-linkage-fix = { workspace = true } +miette = { version = "7.6" } +tonic-prost-build = { workspace = true } diff --git a/crates/proto/build.rs b/crates/proto/build.rs index b0ac773a72..5a39c1d546 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -22,6 +22,8 @@ fn main() -> miette::Result<()> { println!("cargo::rerun-if-changed=../../proto/proto"); println!("cargo::rerun-if-env-changed=BUILD_PROTO"); + miden_node_rocksdb_cxx_linkage_fix::configure(); + // Skip this build script in BUILD_PROTO environment variable is not set to `1`. if env::var("BUILD_PROTO").unwrap_or("0".to_string()) == "0" { return Ok(()); diff --git a/crates/rocksdb-cxx-linkage-fix/src/lib.rs b/crates/rocksdb-cxx-linkage-fix/src/lib.rs index 9eaae82fd3..a20f550bd9 100644 --- a/crates/rocksdb-cxx-linkage-fix/src/lib.rs +++ b/crates/rocksdb-cxx-linkage-fix/src/lib.rs @@ -17,7 +17,8 @@ pub fn configure() { fn should_link_cpp_stdlib() -> bool { let rocksdb_compile = env::var("ROCKSDB_COMPILE").unwrap_or_default(); - let rocksdb_compile_disabled = matches!(rocksdb_compile.as_str(), "0" | "false" | "FALSE"); + let rocksdb_compile_disabled = + rocksdb_compile.is_empty() || matches!(rocksdb_compile.as_str(), "0" | "false" | "FALSE"); let rocksdb_static = env::var("ROCKSDB_STATIC").is_ok(); let rocksdb_lib_dir_set = env::var("ROCKSDB_LIB_DIR").is_ok(); From 2e83cb2c078830983f78c12ed691961a58726d9f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 10 Feb 2026 15:27:31 +0100 Subject: [PATCH 2/2] unused script --- .github/verify-rocksdb-static.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .github/verify-rocksdb-static.sh diff --git a/.github/verify-rocksdb-static.sh b/.github/verify-rocksdb-static.sh deleted file mode 100644 index 8764e91dcb..0000000000 --- a/.github/verify-rocksdb-static.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -euo pipefail -test -f rocksdb-artifact/librocksdb.a -test -f rocksdb-artifact/liblz4.a