From 51681adad9ba51ca98257bb029300d36053c6394 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 09:53:06 +0300 Subject: [PATCH 01/10] Integrate ib_linux / ib_console for distributed Bun builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps the Ninja-driven Bun build with Incredibuild's ib_console so the ~2,000 C/C++ TUs (Bun core + boringssl + WebKit + libuv + libarchive + …) and 107-crate Rust workspace can be distributed across an IB grid. - .incredibuild/ib_profile.xml: process intercept/distribution rules. clang/clang++/gcc/g++ are local_only with exclude_args="-c:-S:-E" so compile-only invocations distribute while link stays local. ninja / cmake / cargo / bun are intercepted recursive so LD_PRELOAD propagates through the whole subtree. rustc is allow_remote with build_script_build excluded (build.rs inspects the host). - .incredibuild/build.sh: wrapper that runs `ib_console -- bun scripts/build.ts …` with sane defaults and BUN_INCREDIBUILD_WRAPPED=1 to prevent the inner ninja spawn from re-wrapping. - scripts/build.ts: opt-in BUN_USE_INCREDIBUILD=1 wraps the inner ninja spawn with ib_console. No-op when unset or when already wrapped. - .github/workflows/incredibuild-linux.yml: self-hosted runner workflow gated on the [self-hosted, linux, incredibuild] label set. - .incredibuild/README.md: setup + verification + tunables. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/incredibuild-linux.yml | 114 +++++++++++++++++++++++ .incredibuild/README.md | 98 +++++++++++++++++++ .incredibuild/build.sh | 57 ++++++++++++ .incredibuild/ib_profile.xml | 89 ++++++++++++++++++ scripts/build.ts | 22 ++++- 5 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/incredibuild-linux.yml create mode 100644 .incredibuild/README.md create mode 100755 .incredibuild/build.sh create mode 100644 .incredibuild/ib_profile.xml diff --git a/.github/workflows/incredibuild-linux.yml b/.github/workflows/incredibuild-linux.yml new file mode 100644 index 00000000000..fcf3b2152a4 --- /dev/null +++ b/.github/workflows/incredibuild-linux.yml @@ -0,0 +1,114 @@ +name: incredibuild-linux + +# Build Bun on Linux via Incredibuild (ib_console). +# +# Targets self-hosted runners that already have ib_linux installed and are +# registered with an Incredibuild coordinator. The runner must carry the +# labels: [self-hosted, linux, incredibuild]. Anything else will skip. +# +# Trigger: +# - workflow_dispatch (manual) +# - push to branches matching incredibuild/** +# - PRs that touch .incredibuild/** or this workflow itself +on: + workflow_dispatch: + inputs: + bun_profile: + description: "Bun build profile (debug | release | ci-cpp-only | ci-release)" + required: true + default: "ci-cpp-only" + type: string + force_remote: + description: "Pass --force-remote to ib_console" + required: false + default: "true" + type: string + max_local_cores: + description: "Limit local cores (blank = no limit)" + required: false + default: "" + type: string + push: + branches: + - "incredibuild/**" + pull_request: + paths: + - ".incredibuild/**" + - ".github/workflows/incredibuild-linux.yml" + - "scripts/build.ts" + +concurrency: + group: incredibuild-linux-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: ib-build (${{ inputs.bun_profile || 'ci-cpp-only' }}) + runs-on: [self-hosted, linux, incredibuild] + timeout-minutes: 120 + + env: + BUN_PROFILE: ${{ inputs.bun_profile || 'ci-cpp-only' }} + IB_FORCE_REMOTE: ${{ inputs.force_remote || 'true' == 'true' && '1' || '0' }} + IB_MAX_LOCAL: ${{ inputs.max_local_cores || '' }} + IB_PROFILE: ${{ github.workspace }}/.incredibuild/ib_profile.xml + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Sanity-check ib_linux + run: | + set -euo pipefail + if ! command -v ib_console >/dev/null 2>&1; then + echo "::error::ib_console not on PATH. ib_linux must be installed and the runner registered with a coordinator." + exit 127 + fi + ib_console --version + ib_console --check-license + test -f "$IB_PROFILE" || { echo "::error::missing $IB_PROFILE"; exit 1; } + + - name: Install Bun (bootstrap runtime) + run: | + set -euo pipefail + if ! command -v bun >/dev/null 2>&1; then + curl -fsSL https://bun.sh/install | bash + echo "$HOME/.bun/bin" >> "$GITHUB_PATH" + fi + # Toolchain assumed pre-installed on the runner image: + # clang, clang++, lld, ninja, cmake, cargo, rustc, nasm, python3. + for t in clang clang++ ninja cmake cargo rustc; do + command -v "$t" >/dev/null || { echo "::error::missing $t on runner"; exit 1; } + done + + - name: Configure (emit ninja graph) + run: | + bun scripts/build.ts --profile="$BUN_PROFILE" --configure-only + + - name: Build under Incredibuild + run: | + set -euo pipefail + ib_args=(--profile "$IB_PROFILE" --caption "gh-bun-$BUN_PROFILE") + [[ "$IB_FORCE_REMOTE" == "1" ]] && ib_args+=(--force-remote) + [[ -n "$IB_MAX_LOCAL" ]] && ib_args+=(--max-local-cores "$IB_MAX_LOCAL") + + # Outer ib_console wraps the whole bun → ninja → clang/rustc tree. + # BUN_INCREDIBUILD_WRAPPED tells scripts/build.ts not to nest another + # ib_console around the inner ninja spawn. + BUN_INCREDIBUILD_WRAPPED=1 \ + ib_console "${ib_args[@]}" -- bun scripts/build.ts --profile="$BUN_PROFILE" + + - name: Upload build dir on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: build-failure-${{ env.BUN_PROFILE }} + path: | + build/**/CMakeFiles/CMakeOutput.log + build/**/CMakeFiles/CMakeError.log + build/**/build.ninja + build/**/*.log + if-no-files-found: ignore + retention-days: 7 diff --git a/.incredibuild/README.md b/.incredibuild/README.md new file mode 100644 index 00000000000..97845755f1e --- /dev/null +++ b/.incredibuild/README.md @@ -0,0 +1,98 @@ +# Incredibuild integration for Bun + +Wraps the Ninja-driven Bun build with [`ib_console`][ib_linux] so the +~2,000 C/C++ TUs (Bun core + boringssl + WebKit + libuv + libarchive + …) +and the 107-crate Rust workspace get distributed across an Incredibuild grid. + +[ib_linux]: https://github.com/Incredibuild-RND/ib_linux + +## Layout + +``` +.incredibuild/ +├── ib_profile.xml IB profile: which procs intercept, distribute, stay local +├── build.sh Wrapper: `ib_console -- bun scripts/build.ts …` +└── README.md This file +``` + +The build.ts side has an opt-in patch: when `BUN_USE_INCREDIBUILD=1`, the +inner `ninja` spawn is wrapped with `ib_console`. When `build.sh` is used, +it sets `BUN_INCREDIBUILD_WRAPPED=1` so we don't nest ib_console twice. + +## Profile in one paragraph + +`clang` / `clang++` / `gcc` / `g++` are listed as `local_only` with +`exclude_args="-c:-S:-E"`. This is the canonical IB pattern: the rule +doesn't fire when a compile flag is present, so `clang -c foo.cpp` falls +through to the default distributable path while `clang foo.o bar.o -o out` +(linking, no -c) stays pinned to the initiator. `ninja`, `cmake`, `cargo`, +and `bun` itself are `intercepted recursive` so the LD_PRELOAD interceptor +propagates through the whole child tree. `rustc` is `allow_remote` with +`build_script_build` excluded — build.rs scripts inspect the local host and +must run locally. + +## Local use + +```bash +# Debug build +.incredibuild/build.sh + +# CI C++ phase (matches the Buildkite cpp-only step) +.incredibuild/build.sh --profile=ci-cpp-only + +# Full release with aggressive remote distribution +IB_FORCE_REMOTE=1 IB_MAX_LOCAL=4 .incredibuild/build.sh --profile=release +``` + +Knobs (all env-var): + +| Var | Default | Purpose | +| --- | --- | --- | +| `IB_CONSOLE` | `ib_console` | path to the binary | +| `IB_PROFILE` | `.incredibuild/ib_profile.xml` | profile XML | +| `IB_FORCE_REMOTE` | unset | `1` adds `--force-remote` | +| `IB_MAX_LOCAL` | unset | int → `--max-local-cores N` | +| `IB_CAPTION` | `bun-` | name in the IB monitor | +| `IB_EXTRA_ARGS` | unset | appended verbatim to ib_console | + +## Use from `bun scripts/build.ts` directly + +```bash +BUN_USE_INCREDIBUILD=1 IB_FORCE_REMOTE=1 \ + bun scripts/build.ts --profile=ci-cpp-only +``` + +This path is what `.github/workflows/incredibuild-linux.yml` exercises on +self-hosted runners tagged `[self-hosted, linux, incredibuild]`. + +## Prerequisites on the build machine + +1. ib_linux installed → `ib_console` on `PATH` (usually at + `/opt/incredibuild/bin/ib_console`). +2. Machine registered with an Incredibuild coordinator (initiator role). +3. A pool of helpers on the same network — these execute the distributed + `clang -c`, `rustc`, parser, and assembler jobs. +4. Bun toolchain: `clang`, `clang++`, `lld`, `ninja`, `cmake`, `cargo`, + `rustc`, `nasm`, `python3`, plus `bun` itself. + +## How to verify it's actually distributing + +```bash +.incredibuild/build.sh --profile=ci-cpp-only 2>&1 | tee /tmp/ib.log +grep -E "remote|helper|task" /tmp/ib.log | head +``` + +ib_console prints per-task placement (local vs helper N) when +`monitor_enable="true"` in the profile (which is the default here). + +## What's distributable vs not + +| Phase | TUs (≈) | Distributable | Notes | +| --- | --- | --- | --- | +| BoringSSL | 600 | yes | mostly `clang -c` | +| WebKit (local build) | thousands | yes | giant win when not using prebuilt | +| libuv / c-ares / libarchive / zstd / zlib / libdeflate | ~250 total | yes | plain C `clang -c` | +| Bun C++ core (`src/**/*.cpp`) | 554 | yes | uses PCH; PCH itself stays local | +| Rust crates | 107 | partial | rustc is `allow_remote`; build.rs is local | +| Final link | 1 | no | always local; LTO needs all object files on box | +| Codegen scripts (JS) | many | no (intercepted, runs local) | `bun` runs them | diff --git a/.incredibuild/build.sh b/.incredibuild/build.sh new file mode 100755 index 00000000000..f5d10138027 --- /dev/null +++ b/.incredibuild/build.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Wrapper: run a Bun build through Incredibuild (ib_console). +# +# Usage: +# .incredibuild/build.sh # debug profile +# .incredibuild/build.sh --profile=ci-cpp-only # CI C++ phase +# .incredibuild/build.sh --profile=release --build-dir=build/release +# +# Env knobs: +# IB_CONSOLE path to ib_console (default: ib_console on PATH) +# IB_PROFILE path to profile XML (default: .incredibuild/ib_profile.xml) +# IB_FORCE_REMOTE "1" to pass --force-remote (push allow_remote off-box) +# IB_MAX_LOCAL integer; pass --max-local-cores N +# IB_CAPTION build name for the IB monitor (default: bun-) +# IB_EXTRA_ARGS extra args appended verbatim to ib_console + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +IB_CONSOLE="${IB_CONSOLE:-ib_console}" +IB_PROFILE="${IB_PROFILE:-$repo_root/.incredibuild/ib_profile.xml}" + +if ! command -v "$IB_CONSOLE" >/dev/null 2>&1; then + echo "error: '$IB_CONSOLE' not on PATH." >&2 + echo " install ib_linux from https://github.com/Incredibuild-RND/ib_linux" >&2 + echo " or set IB_CONSOLE=/opt/incredibuild/bin/ib_console" >&2 + exit 127 +fi + +if [[ ! -f "$IB_PROFILE" ]]; then + echo "error: IB profile not found: $IB_PROFILE" >&2 + exit 1 +fi + +# Default to debug if no --profile passed. (Bun build profile, not IB profile.) +bun_profile="debug" +for arg in "$@"; do + case "$arg" in + --profile=*) bun_profile="${arg#--profile=}" ;; + esac +done + +ib_args=( + --profile "$IB_PROFILE" + --caption "${IB_CAPTION:-bun-$bun_profile}" +) +[[ "${IB_FORCE_REMOTE:-0}" == "1" ]] && ib_args+=(--force-remote) +[[ -n "${IB_MAX_LOCAL:-}" ]] && ib_args+=(--max-local-cores "$IB_MAX_LOCAL") +[[ -n "${IB_EXTRA_ARGS:-}" ]] && read -r -a extra <<<"$IB_EXTRA_ARGS" && ib_args+=("${extra[@]}") + +# Tell scripts/build.ts not to re-wrap ninja under another ib_console — +# the outer ib_console here already intercepts the whole subtree. +export BUN_INCREDIBUILD_WRAPPED=1 + +exec "$IB_CONSOLE" "${ib_args[@]}" -- bun scripts/build.ts "$@" diff --git a/.incredibuild/ib_profile.xml b/.incredibuild/ib_profile.xml new file mode 100644 index 00000000000..81215fd8397 --- /dev/null +++ b/.incredibuild/ib_profile.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/build.ts b/scripts/build.ts index e92aa652db3..a405e21b1f1 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -97,6 +97,24 @@ async function main(): Promise { const ninjaArgv = (cfg: { buildDir: string }) => ["-C", cfg.buildDir, ...args.ninjaArgs, ...args.ninjaTargets]; const ninjaEnv = (env: Record) => ({ ...process.env, ...env }); + // Incredibuild integration. Opt-in via BUN_USE_INCREDIBUILD=1. Skipped when + // BUN_INCREDIBUILD_WRAPPED=1 (set by .incredibuild/build.sh, which already + // wraps the whole bun-build subtree under ib_console). + const useIB = process.env.BUN_USE_INCREDIBUILD === "1" && process.env.BUN_INCREDIBUILD_WRAPPED !== "1"; + const ibConsole = process.env.IB_CONSOLE || "ib_console"; + const ibProfile = process.env.IB_PROFILE || join(process.cwd(), ".incredibuild", "ib_profile.xml"); + const ibArgs = useIB + ? [ + "--profile", ibProfile, + "--caption", `bun-${args.profile ?? "build"}`, + ...(process.env.IB_FORCE_REMOTE === "1" ? ["--force-remote"] : []), + ...(process.env.IB_MAX_LOCAL ? ["--max-local-cores", process.env.IB_MAX_LOCAL] : []), + "--", + ] + : []; + const ninjaCmd = useIB ? ibConsole : "ninja"; + const ninjaFullArgv = (cfg: { buildDir: string }) => (useIB ? [...ibArgs, "ninja", ...ninjaArgv(cfg)] : ninjaArgv(cfg)); + if (isCI) { // CI: machine/env dump + collapsible groups + annotation-on-failure. printEnvironment(); @@ -109,7 +127,7 @@ async function main(): Promise { } await startGroup("Build", () => - spawnWithAnnotations("ninja", ninjaArgv(result.cfg), { label: "ninja", env: ninjaEnv(result.env) }), + spawnWithAnnotations(ninjaCmd, ninjaFullArgv(result.cfg), { label: "ninja", env: ninjaEnv(result.env) }), ); // cpp-only/rust-only: upload build outputs for downstream link-only. @@ -180,7 +198,7 @@ async function main(): Promise { if (!quiet && interactive) { stdio[STREAM_FD] = 2; } - const ninja = spawnSync("ninja", ninjaArgv(result.cfg), { + const ninja = spawnSync(ninjaCmd, ninjaFullArgv(result.cfg), { stdio, env: ninjaEnv(result.env), // cargo's compile output (now part of the ninja graph via emitRust) can From 7dbbeb9dda662600c9f319a078fe0fe960f6ebaf Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 10:06:03 +0300 Subject: [PATCH 02/10] .incredibuild: fix XML comments, add XSD validation, wire it into CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ib_profile.xml comments contained `--` (XML-illegal double-hyphen inside comments), causing the file to fail libxml2 parsing entirely. Rewrote the comments to avoid the sequence. The profile now validates clean against ib_linux/data/ib_profile.xsd — the same XSD the real ib_console uses internally to load ProfileConfig. Added .incredibuild/validate.sh: auto-locates the XSD (env var, installed ib_linux, sibling checkout) and runs xmllint. Anyone can sanity-check the profile without spinning up a coordinator + helper grid. incredibuild-linux.yml: new `validate-profile-xsd` job on ubuntu-latest that fetches the XSD from Incredibuild-RND/ib_linux@develop and validates the profile on every PR. Gated `build` on `validate` so a broken profile fails fast instead of consuming a self-hosted runner slot. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/incredibuild-linux.yml | 24 +++++++++++++++ .incredibuild/ib_profile.xml | 34 +++++++++++----------- .incredibuild/validate.sh | 37 ++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 17 deletions(-) create mode 100755 .incredibuild/validate.sh diff --git a/.github/workflows/incredibuild-linux.yml b/.github/workflows/incredibuild-linux.yml index fcf3b2152a4..1992a60b5d4 100644 --- a/.github/workflows/incredibuild-linux.yml +++ b/.github/workflows/incredibuild-linux.yml @@ -42,7 +42,31 @@ concurrency: cancel-in-progress: true jobs: + validate: + name: validate-profile-xsd + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install xmllint + run: sudo apt-get update && sudo apt-get install -y libxml2-utils + - name: Fetch ib_profile.xsd from Incredibuild-RND/ib_linux + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p .ib_xsd + # Public-fallback: try unauthenticated raw first, then gh api (works + # for org members on the internal repo). + if ! curl -fsSL https://raw.githubusercontent.com/Incredibuild-RND/ib_linux/develop/data/ib_profile.xsd \ + -o .ib_xsd/ib_profile.xsd 2>/dev/null; then + gh api repos/Incredibuild-RND/ib_linux/contents/data/ib_profile.xsd?ref=develop \ + --jq '.content' | base64 -d > .ib_xsd/ib_profile.xsd + fi + test -s .ib_xsd/ib_profile.xsd + - name: Validate .incredibuild/ib_profile.xml + run: IB_XSD="$PWD/.ib_xsd/ib_profile.xsd" ./.incredibuild/validate.sh + build: + needs: validate name: ib-build (${{ inputs.bun_profile || 'ci-cpp-only' }}) runs-on: [self-hosted, linux, incredibuild] timeout-minutes: 120 diff --git a/.incredibuild/ib_profile.xml b/.incredibuild/ib_profile.xml index 81215fd8397..bed0eeb5992 100644 --- a/.incredibuild/ib_profile.xml +++ b/.incredibuild/ib_profile.xml @@ -2,21 +2,21 @@ + enables build-cache integration. --> @@ -75,9 +75,9 @@ - + diff --git a/.incredibuild/validate.sh b/.incredibuild/validate.sh new file mode 100755 index 00000000000..ff08acb58f8 --- /dev/null +++ b/.incredibuild/validate.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Validate .incredibuild/ib_profile.xml against ib_linux's official XSD. +# This is the same XSD the real ib_console's ProfileConfig parser uses. +# +# ./.incredibuild/validate.sh # auto-locates the XSD +# IB_XSD=/path/to/ib_profile.xsd ./.incredibuild/validate.sh +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +profile="$repo_root/.incredibuild/ib_profile.xml" + +# Locate the XSD. Order: $IB_XSD, installed ib_linux, ../ib_linux checkout. +xsd="" +for candidate in \ + "${IB_XSD:-}" \ + /opt/incredibuild/data/ib_profile.xsd \ + "$repo_root/../ib_linux/data/ib_profile.xsd" \ + /tmp/ib_linux/data/ib_profile.xsd +do + [[ -n "$candidate" && -f "$candidate" ]] && { xsd="$candidate"; break; } +done + +if [[ -z "$xsd" ]]; then + echo "error: ib_profile.xsd not found." >&2 + echo " Set IB_XSD=/path/to/ib_profile.xsd, install ib_linux to /opt/incredibuild," >&2 + echo " or clone https://github.com/Incredibuild-RND/ib_linux next to this repo." >&2 + exit 1 +fi + +if ! command -v xmllint >/dev/null 2>&1; then + echo "error: xmllint not on PATH (apt install libxml2-utils / brew install libxml2)" >&2 + exit 1 +fi + +echo "profile: $profile" +echo "xsd: $xsd" +xmllint --noout --schema "$xsd" "$profile" From 98fa0bd9059ac25ca3ecc723b518fc64d2fe9ed2 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 13:10:00 +0300 Subject: [PATCH 03/10] .incredibuild: vendor ib_profile.xsd so CI works without cross-repo auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The validate-profile-xsd job was fetching the XSD from Incredibuild-RND/ib_linux at run time, which 404s on ubuntu-latest because ib_linux is INTERNAL and the workflow's GITHUB_TOKEN doesn't have cross-repo access. Vendor data/ib_profile.xsd (97 lines, ~4 KB) into .incredibuild/. The profile schema is versioned and changes rarely; bumping the vendored copy when ib_linux releases a new schema is a 1-line diff. validate.sh now prefers the vendored copy and falls back to env override / installed ib_linux / sibling checkout. The CI step is now a single `./.incredibuild/validate.sh` — no network calls, no auth, no flake. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/incredibuild-linux.yml | 15 +--- .incredibuild/README.md | 2 + .incredibuild/ib_profile.xsd | 97 ++++++++++++++++++++++++ .incredibuild/validate.sh | 5 +- 4 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 .incredibuild/ib_profile.xsd diff --git a/.github/workflows/incredibuild-linux.yml b/.github/workflows/incredibuild-linux.yml index 1992a60b5d4..87ee8e37f92 100644 --- a/.github/workflows/incredibuild-linux.yml +++ b/.github/workflows/incredibuild-linux.yml @@ -49,21 +49,8 @@ jobs: - uses: actions/checkout@v4 - name: Install xmllint run: sudo apt-get update && sudo apt-get install -y libxml2-utils - - name: Fetch ib_profile.xsd from Incredibuild-RND/ib_linux - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - mkdir -p .ib_xsd - # Public-fallback: try unauthenticated raw first, then gh api (works - # for org members on the internal repo). - if ! curl -fsSL https://raw.githubusercontent.com/Incredibuild-RND/ib_linux/develop/data/ib_profile.xsd \ - -o .ib_xsd/ib_profile.xsd 2>/dev/null; then - gh api repos/Incredibuild-RND/ib_linux/contents/data/ib_profile.xsd?ref=develop \ - --jq '.content' | base64 -d > .ib_xsd/ib_profile.xsd - fi - test -s .ib_xsd/ib_profile.xsd - name: Validate .incredibuild/ib_profile.xml - run: IB_XSD="$PWD/.ib_xsd/ib_profile.xsd" ./.incredibuild/validate.sh + run: ./.incredibuild/validate.sh build: needs: validate diff --git a/.incredibuild/README.md b/.incredibuild/README.md index 97845755f1e..f896f2b9101 100644 --- a/.incredibuild/README.md +++ b/.incredibuild/README.md @@ -11,7 +11,9 @@ and the 107-crate Rust workspace get distributed across an Incredibuild grid. ``` .incredibuild/ ├── ib_profile.xml IB profile: which procs intercept, distribute, stay local +├── ib_profile.xsd Vendored copy of ib_linux/data/ib_profile.xsd for offline validation ├── build.sh Wrapper: `ib_console -- bun scripts/build.ts …` +├── validate.sh Validate the profile against the XSD via xmllint └── README.md This file ``` diff --git a/.incredibuild/ib_profile.xsd b/.incredibuild/ib_profile.xsd new file mode 100644 index 00000000000..47c00e8b3d3 --- /dev/null +++ b/.incredibuild/ib_profile.xsd @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.incredibuild/validate.sh b/.incredibuild/validate.sh index ff08acb58f8..c52eb6be624 100755 --- a/.incredibuild/validate.sh +++ b/.incredibuild/validate.sh @@ -9,10 +9,13 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" profile="$repo_root/.incredibuild/ib_profile.xml" -# Locate the XSD. Order: $IB_XSD, installed ib_linux, ../ib_linux checkout. +# Locate the XSD. Order: $IB_XSD, vendored copy, installed ib_linux, +# sibling ib_linux checkout. Vendored is the default so CI works without +# cross-repo access to the internal Incredibuild-RND/ib_linux repo. xsd="" for candidate in \ "${IB_XSD:-}" \ + "$repo_root/.incredibuild/ib_profile.xsd" \ /opt/incredibuild/data/ib_profile.xsd \ "$repo_root/../ib_linux/data/ib_profile.xsd" \ /tmp/ib_linux/data/ib_profile.xsd From 3d7ec38d58ad25a05e53db0f6b262b586b681cdf Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 13:24:31 +0300 Subject: [PATCH 04/10] ci: composite actions, build cache, metrics summary, side-by-side bench MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts the IB invocation into two reusable composite actions and adds a benchmark workflow so we can measure speedup objectively. - .github/actions/ib-build/action.yml: single source of truth for `ib_console -- bun scripts/build.ts`. Inputs cover profile, force-remote, max-local-cores, cache-dir, caption, extra-args. Outputs the log path the metrics action consumes. - .github/actions/ib-metrics/action.yml: parses ib_console's monitor log (task counts, local vs helper distribution, cache hits, acceleration factor) and writes a markdown table to $GITHUB_STEP_SUMMARY. Falls back to placement-line counting if structured fields aren't emitted. - .github/workflows/incredibuild-linux.yml: wires actions/cache@v4 to ib_console's --build-cache-local-user dir (cache key on profile + Cargo.lock + package.json + scripts/build/** + build.rs files), uses the composite actions, uploads the ib log on every run + build dir on failure. Also adds shellcheck of the wrapper scripts to the validate job. - .github/workflows/incredibuild-compare.yml: manual-dispatch benchmark. Builds the same profile N times with and without IB, reports median wall time, delta, and speedup factor. Designed as the canonical acceleration-regression check. - .github/actionlint.yaml: registers the `incredibuild` self-hosted label so linters don't flag it as a typo. Local validation: actionlint → exit 0 shellcheck → silent (clean) on build.sh + validate.sh xmllint → profile validates against vendored XSD Co-Authored-By: Claude Opus 4.7 --- .github/actionlint.yaml | 5 + .github/actions/ib-build/action.yml | 118 +++++++++++++++++++ .github/actions/ib-metrics/action.yml | 79 +++++++++++++ .github/workflows/incredibuild-compare.yml | 121 +++++++++++++++++++ .github/workflows/incredibuild-linux.yml | 130 +++++++++++---------- 5 files changed, 393 insertions(+), 60 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/actions/ib-build/action.yml create mode 100644 .github/actions/ib-metrics/action.yml create mode 100644 .github/workflows/incredibuild-compare.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000..4a668b76212 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +# actionlint config — registers labels that exist on our self-hosted runners +# but not on GH-hosted ones, so the linter stops flagging them as typos. +self-hosted-runner: + labels: + - incredibuild diff --git a/.github/actions/ib-build/action.yml b/.github/actions/ib-build/action.yml new file mode 100644 index 00000000000..3012b201342 --- /dev/null +++ b/.github/actions/ib-build/action.yml @@ -0,0 +1,118 @@ +name: ib-build +description: | + Build Bun under Incredibuild's ib_console. Single source of truth for the + ib_console invocation across all CI jobs. Wraps `bun scripts/build.ts` so + the inner ninja → clang/cargo subtree is distributed across an IB grid. +author: Incredibuild-RND + +inputs: + bun-profile: + description: "Bun build profile (debug | release | ci-cpp-only | ci-release | …)" + required: true + force-remote: + description: "Pass --force-remote to ib_console (push allow_remote off-box)" + required: false + default: "true" + max-local-cores: + description: "Cap local cores used by ib_console (blank = no cap)" + required: false + default: "" + cache-dir: + description: "ib_console --build-cache-basedir (must be absolute). Empty disables cache." + required: false + default: "" + caption: + description: "Build name shown in the IB monitor" + required: false + default: "" + extra-args: + description: "Extra args appended verbatim to ib_console (e.g. --compress 5)" + required: false + default: "" + +outputs: + log-path: + description: "Path to the ib_console monitor log written by this build" + value: ${{ steps.run.outputs.log-path }} + build-dir: + description: "Bun build directory ninja wrote to" + value: ${{ steps.run.outputs.build-dir }} + +runs: + using: composite + steps: + - name: Sanity-check ib_console + shell: bash + run: | + set -euo pipefail + if ! command -v ib_console >/dev/null 2>&1; then + echo "::error::ib_console not on PATH. ib_linux must be installed on this runner." + exit 127 + fi + ib_console --version + ib_console --check-license + test -f "${{ github.workspace }}/.incredibuild/ib_profile.xml" \ + || { echo "::error::missing .incredibuild/ib_profile.xml"; exit 1; } + + - name: Install Bun (if not pre-installed on runner) + shell: bash + run: | + set -euo pipefail + if ! command -v bun >/dev/null 2>&1; then + curl -fsSL https://bun.sh/install | bash + echo "$HOME/.bun/bin" >> "$GITHUB_PATH" + fi + for t in clang clang++ ninja cmake cargo rustc; do + command -v "$t" >/dev/null || { echo "::error::missing $t on runner"; exit 1; } + done + + - name: Build under Incredibuild + id: run + shell: bash + env: + BUN_PROFILE: ${{ inputs.bun-profile }} + IB_FORCE_REMOTE_IN: ${{ inputs.force-remote }} + IB_MAX_LOCAL_IN: ${{ inputs.max-local-cores }} + IB_CACHE_DIR_IN: ${{ inputs.cache-dir }} + IB_CAPTION_IN: ${{ inputs.caption }} + IB_EXTRA_ARGS_IN: ${{ inputs.extra-args }} + run: | + set -euo pipefail + log_path="${RUNNER_TEMP:-/tmp}/ib_console-${BUN_PROFILE}.log" + echo "log-path=$log_path" >> "$GITHUB_OUTPUT" + echo "build-dir=build/${BUN_PROFILE}" >> "$GITHUB_OUTPUT" + + ib_args=( + --profile "${{ github.workspace }}/.incredibuild/ib_profile.xml" + --caption "${IB_CAPTION_IN:-gh-bun-$BUN_PROFILE}" + ) + + # Acceleration knobs. + [[ "$IB_FORCE_REMOTE_IN" == "true" ]] && ib_args+=(--force-remote) + [[ -n "$IB_MAX_LOCAL_IN" ]] && ib_args+=(--max-local-cores "$IB_MAX_LOCAL_IN") + + # Build cache: ib_console's own per-user cache backed by GH's + # actions/cache (set up by the caller, restored before this step). + if [[ -n "$IB_CACHE_DIR_IN" ]]; then + mkdir -p "$IB_CACHE_DIR_IN" + ib_args+=( + --build-cache-local-user + --build-cache-basedir "$IB_CACHE_DIR_IN" + --build-cache-local-logfile "$log_path.cache" + ) + fi + + # Pass arbitrary extras through (e.g., --compress 5). + if [[ -n "$IB_EXTRA_ARGS_IN" ]]; then + read -r -a extras <<<"$IB_EXTRA_ARGS_IN" + ib_args+=("${extras[@]}") + fi + + # BUN_INCREDIBUILD_WRAPPED stops scripts/build.ts from wrapping the + # inner ninja in a second ib_console — we wrap the whole subtree here. + export BUN_INCREDIBUILD_WRAPPED=1 + + # Tee to file so the metrics step can parse placement data. + ib_console "${ib_args[@]}" -- \ + bun scripts/build.ts --profile="$BUN_PROFILE" 2>&1 \ + | tee "$log_path" diff --git a/.github/actions/ib-metrics/action.yml b/.github/actions/ib-metrics/action.yml new file mode 100644 index 00000000000..d81ab2574bb --- /dev/null +++ b/.github/actions/ib-metrics/action.yml @@ -0,0 +1,79 @@ +name: ib-metrics +description: | + Parse an ib_console monitor log and write a markdown summary to + $GITHUB_STEP_SUMMARY: task counts, local vs helper distribution, + parallelism factor, cache hit rate. Treats missing fields gracefully. + +inputs: + log-path: + description: "Path to the ib_console log produced by ib-build" + required: true + label: + description: "Heading for the summary block" + required: false + default: "Incredibuild metrics" + +runs: + using: composite + steps: + - name: Write summary + shell: bash + env: + LOG_PATH: ${{ inputs.log-path }} + LABEL: ${{ inputs.label }} + run: | + set -uo pipefail + if [[ ! -s "$LOG_PATH" ]]; then + { + echo "## $LABEL" + echo + echo "_no log captured at \`$LOG_PATH\`_" + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # Heuristic extraction. ib_console with monitor_enable="true" emits + # lines like: + # [task] helper N: clang -c ... + # [task] local : ld ... + # plus a final block: + # Total tasks: X + # Tasks executed locally: Y + # Tasks executed remotely: Z + # Parallel acceleration: K.K x + total_tasks=$(grep -Ei "Total tasks" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+' | head -1 || true) + local_tasks=$(grep -Ei "(executed locally|local tasks)" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+' | head -1 || true) + remote_tasks=$(grep -Ei "(executed remotely|remote tasks)" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+' | head -1 || true) + accel=$(grep -Ei "(acceleration|speedup)" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+(\.[0-9]+)?' | head -1 || true) + wall=$(grep -Ei "(wall.?time|build duration|total time)" "$LOG_PATH" | tail -1 || true) + + # Fallback if the structured block isn't found: count placement lines. + if [[ -z "$total_tasks" ]]; then + local_tasks=$(grep -cE '\[(task|exec)\][^]]*local' "$LOG_PATH" || echo 0) + remote_tasks=$(grep -cE '\[(task|exec)\][^]]*(helper|remote)' "$LOG_PATH" || echo 0) + total_tasks=$(( local_tasks + remote_tasks )) + fi + + cache_hits=$(grep -ciE "cache.?hit" "$LOG_PATH" || echo 0) + cache_miss=$(grep -ciE "cache.?miss" "$LOG_PATH" || echo 0) + + { + echo "## $LABEL" + echo + echo "| metric | value |" + echo "| --- | --- |" + echo "| total tasks | ${total_tasks:-?} |" + echo "| local tasks | ${local_tasks:-?} |" + echo "| remote tasks | ${remote_tasks:-?} |" + if [[ -n "$accel" ]]; then echo "| parallel acceleration | ${accel}x |"; fi + if [[ -n "$wall" ]]; then echo "| wall time | \`$wall\` |"; fi + echo "| cache hits | $cache_hits |" + echo "| cache misses | $cache_miss |" + echo + echo "
log tail (last 50 lines)" + echo + echo "\`\`\`" + tail -n 50 "$LOG_PATH" + echo "\`\`\`" + echo "
" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/incredibuild-compare.yml b/.github/workflows/incredibuild-compare.yml new file mode 100644 index 00000000000..8812ca51efe --- /dev/null +++ b/.github/workflows/incredibuild-compare.yml @@ -0,0 +1,121 @@ +name: incredibuild-compare + +# Side-by-side acceleration benchmark: build the same Bun profile twice +# on the same self-hosted runner — once with Incredibuild, once stock — +# and report the wall-time delta to the job summary. +# +# Manual dispatch only. Designed to be the canonical "is IB still helping?" +# regression check after profile changes or compiler upgrades. + +on: + workflow_dispatch: + inputs: + bun_profile: + description: "Bun build profile to benchmark" + required: true + default: "ci-cpp-only" + type: choice + options: + - debug + - release + - ci-cpp-only + - ci-rust-only + - ci-release + iterations: + description: "How many times to run each variant (median is reported)" + required: false + default: "1" + type: string + clean_between: + description: "Wipe the build dir between iterations (cold build) vs reuse (warm)" + required: false + default: true + type: boolean + +concurrency: + group: incredibuild-compare-${{ github.ref }} + cancel-in-progress: false + +jobs: + benchmark: + runs-on: [self-hosted, linux, incredibuild] + timeout-minutes: 240 + env: + BUN_PROFILE: ${{ inputs.bun_profile }} + ITERATIONS: ${{ inputs.iterations }} + CLEAN: ${{ inputs.clean_between }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Warm dependencies once (excluded from timing) + run: | + set -euo pipefail + if ! command -v bun >/dev/null 2>&1; then + curl -fsSL https://bun.sh/install | bash + echo "$HOME/.bun/bin" >> "$GITHUB_PATH" + fi + bun scripts/build.ts --profile="$BUN_PROFILE" --configure-only + + - name: Run stock builds + id: stock + run: | + set -euo pipefail + times_file=/tmp/stock-times.txt + : > "$times_file" + for i in $(seq 1 "$ITERATIONS"); do + [[ "$CLEAN" == "true" ]] && rm -rf "build/$BUN_PROFILE" + t0=$(date +%s.%N) + bun scripts/build.ts --profile="$BUN_PROFILE" + t1=$(date +%s.%N) + dt=$(awk -v a="$t0" -v b="$t1" 'BEGIN{print b-a}') + echo "$dt" >> "$times_file" + echo "iteration $i: ${dt}s" >&2 + done + median=$(sort -n "$times_file" | awk 'NR==int(NR/2)+1' ) + echo "median=$median" >> "$GITHUB_OUTPUT" + cat "$times_file" + + - name: Run IB-wrapped builds + id: ib + run: | + set -euo pipefail + times_file=/tmp/ib-times.txt + : > "$times_file" + for i in $(seq 1 "$ITERATIONS"); do + [[ "$CLEAN" == "true" ]] && rm -rf "build/$BUN_PROFILE" + t0=$(date +%s.%N) + BUN_INCREDIBUILD_WRAPPED=1 \ + ib_console --profile "$PWD/.incredibuild/ib_profile.xml" \ + --force-remote \ + --caption "bench-$BUN_PROFILE-$i" \ + -- bun scripts/build.ts --profile="$BUN_PROFILE" + t1=$(date +%s.%N) + dt=$(awk -v a="$t0" -v b="$t1" 'BEGIN{print b-a}') + echo "$dt" >> "$times_file" + echo "iteration $i: ${dt}s" >&2 + done + median=$(sort -n "$times_file" | awk 'NR==int(NR/2)+1' ) + echo "median=$median" >> "$GITHUB_OUTPUT" + cat "$times_file" + + - name: Report + run: | + stock=${{ steps.stock.outputs.median }} + ib=${{ steps.ib.outputs.median }} + speedup=$(awk -v s="$stock" -v i="$ib" 'BEGIN{ if (i>0) printf "%.2f", s/i; else print "n/a"}') + delta=$(awk -v s="$stock" -v i="$ib" 'BEGIN{printf "%.2f", s-i}') + { + echo "## Incredibuild speedup — $BUN_PROFILE" + echo + echo "| variant | median wall (s) |" + echo "| --- | --- |" + echo "| stock | $stock |" + echo "| ib_console --force-remote | $ib |" + echo + echo "**delta:** ${delta}s saved · **speedup:** ${speedup}×" + echo + echo "_(${ITERATIONS} iteration(s), clean-between=${CLEAN})_" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/incredibuild-linux.yml b/.github/workflows/incredibuild-linux.yml index 87ee8e37f92..d04db4291ab 100644 --- a/.github/workflows/incredibuild-linux.yml +++ b/.github/workflows/incredibuild-linux.yml @@ -2,38 +2,51 @@ name: incredibuild-linux # Build Bun on Linux via Incredibuild (ib_console). # -# Targets self-hosted runners that already have ib_linux installed and are -# registered with an Incredibuild coordinator. The runner must carry the -# labels: [self-hosted, linux, incredibuild]. Anything else will skip. -# -# Trigger: -# - workflow_dispatch (manual) -# - push to branches matching incredibuild/** -# - PRs that touch .incredibuild/** or this workflow itself +# Two jobs: +# 1. validate-profile-xsd (ubuntu-latest) — always runs, no IB needed. +# 2. ib-build (self-hosted) — needs runner with ib_linux +# installed + coordinator +# registered, label +# `[self-hosted, linux, +# incredibuild]`. on: workflow_dispatch: inputs: bun_profile: - description: "Bun build profile (debug | release | ci-cpp-only | ci-release)" + description: "Bun build profile" required: true default: "ci-cpp-only" - type: string + type: choice + options: + - debug + - release + - ci-cpp-only + - ci-rust-only + - ci-link-only + - ci-release force_remote: description: "Pass --force-remote to ib_console" required: false - default: "true" - type: string + default: true + type: boolean max_local_cores: - description: "Limit local cores (blank = no limit)" + description: "Cap local cores (blank = no cap)" required: false default: "" type: string + use_cache: + description: "Use IB build cache (actions/cache-backed)" + required: false + default: true + type: boolean push: branches: - "incredibuild/**" pull_request: paths: - ".incredibuild/**" + - ".github/actions/ib-build/**" + - ".github/actions/ib-metrics/**" - ".github/workflows/incredibuild-linux.yml" - "scripts/build.ts" @@ -42,8 +55,7 @@ concurrency: cancel-in-progress: true jobs: - validate: - name: validate-profile-xsd + validate-profile-xsd: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -51,65 +63,63 @@ jobs: run: sudo apt-get update && sudo apt-get install -y libxml2-utils - name: Validate .incredibuild/ib_profile.xml run: ./.incredibuild/validate.sh + - name: Shellcheck wrapper scripts + run: | + sudo apt-get install -y shellcheck + shellcheck .incredibuild/build.sh .incredibuild/validate.sh - build: - needs: validate - name: ib-build (${{ inputs.bun_profile || 'ci-cpp-only' }}) + ib-build: + needs: validate-profile-xsd + if: ${{ github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/incredibuild/') }} runs-on: [self-hosted, linux, incredibuild] timeout-minutes: 120 env: BUN_PROFILE: ${{ inputs.bun_profile || 'ci-cpp-only' }} - IB_FORCE_REMOTE: ${{ inputs.force_remote || 'true' == 'true' && '1' || '0' }} - IB_MAX_LOCAL: ${{ inputs.max_local_cores || '' }} - IB_PROFILE: ${{ github.workspace }}/.incredibuild/ib_profile.xml steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Sanity-check ib_linux - run: | - set -euo pipefail - if ! command -v ib_console >/dev/null 2>&1; then - echo "::error::ib_console not on PATH. ib_linux must be installed and the runner registered with a coordinator." - exit 127 - fi - ib_console --version - ib_console --check-license - test -f "$IB_PROFILE" || { echo "::error::missing $IB_PROFILE"; exit 1; } - - - name: Install Bun (bootstrap runtime) - run: | - set -euo pipefail - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - echo "$HOME/.bun/bin" >> "$GITHUB_PATH" - fi - # Toolchain assumed pre-installed on the runner image: - # clang, clang++, lld, ninja, cmake, cargo, rustc, nasm, python3. - for t in clang clang++ ninja cmake cargo rustc; do - command -v "$t" >/dev/null || { echo "::error::missing $t on runner"; exit 1; } - done - - - name: Configure (emit ninja graph) - run: | - bun scripts/build.ts --profile="$BUN_PROFILE" --configure-only + - name: Restore IB build cache + if: ${{ inputs.use_cache != false }} + id: ib-cache + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/ib-cache + # Key on profile + lockfiles that influence outputs. Adjust as the + # Bun build's input set evolves. + key: ib-cache-${{ runner.os }}-${{ env.BUN_PROFILE }}-${{ hashFiles('Cargo.lock', 'package.json', 'scripts/build/**', 'src/**/build.rs') }} + restore-keys: | + ib-cache-${{ runner.os }}-${{ env.BUN_PROFILE }}- + ib-cache-${{ runner.os }}- - name: Build under Incredibuild - run: | - set -euo pipefail - ib_args=(--profile "$IB_PROFILE" --caption "gh-bun-$BUN_PROFILE") - [[ "$IB_FORCE_REMOTE" == "1" ]] && ib_args+=(--force-remote) - [[ -n "$IB_MAX_LOCAL" ]] && ib_args+=(--max-local-cores "$IB_MAX_LOCAL") + id: build + uses: ./.github/actions/ib-build + with: + bun-profile: ${{ env.BUN_PROFILE }} + force-remote: ${{ inputs.force_remote == true && 'true' || 'false' }} + max-local-cores: ${{ inputs.max_local_cores }} + cache-dir: ${{ inputs.use_cache != false && format('{0}/ib-cache', runner.temp) || '' }} + caption: "gh-${{ env.BUN_PROFILE }}-${{ github.run_id }}" - # Outer ib_console wraps the whole bun → ninja → clang/rustc tree. - # BUN_INCREDIBUILD_WRAPPED tells scripts/build.ts not to nest another - # ib_console around the inner ninja spawn. - BUN_INCREDIBUILD_WRAPPED=1 \ - ib_console "${ib_args[@]}" -- bun scripts/build.ts --profile="$BUN_PROFILE" + - name: Acceleration metrics + if: always() + uses: ./.github/actions/ib-metrics + with: + log-path: ${{ steps.build.outputs.log-path }} + label: "Incredibuild — ${{ env.BUN_PROFILE }}" + + - name: Upload ib_console log + if: always() + uses: actions/upload-artifact@v4 + with: + name: ib-log-${{ env.BUN_PROFILE }} + path: ${{ steps.build.outputs.log-path }}* + if-no-files-found: ignore + retention-days: 14 - name: Upload build dir on failure if: failure() From 07e48d45adb392a343982f230da6377632d84d7b Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 13:26:38 +0300 Subject: [PATCH 05/10] ci: gate ib-build on workflow_dispatch only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pushes to incredibuild/** and PR triggers were queueing ib-build for the self-hosted runner indefinitely (no runner exists yet on the GH side). The validate job still auto-runs on every push/PR — that's where the per-commit signal lives. ib-build is for benchmarks and explicit kickoff once the runner is online. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/incredibuild-linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/incredibuild-linux.yml b/.github/workflows/incredibuild-linux.yml index d04db4291ab..ed3e6b02365 100644 --- a/.github/workflows/incredibuild-linux.yml +++ b/.github/workflows/incredibuild-linux.yml @@ -70,7 +70,9 @@ jobs: ib-build: needs: validate-profile-xsd - if: ${{ github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/incredibuild/') }} + # Self-hosted only: runs on manual dispatch. Pushes/PRs would queue + # indefinitely if no runner is registered. + if: ${{ github.event_name == 'workflow_dispatch' }} runs-on: [self-hosted, linux, incredibuild] timeout-minutes: 120 From 53416940c310c57aa3c05e89945caf72415fa675 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 13:41:17 +0300 Subject: [PATCH 06/10] scripts/build.ts: apply prettier formatting flagged by autofix.ci The IB integration block had: - paired array items on a single line ("--profile", ibProfile,) - one-line ternary spanning >120 chars Both violate the project's prettier config. autofix.ci flagged them on PR #1; applying its exact suggested diff. Co-Authored-By: Claude Opus 4.7 --- scripts/build.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/build.ts b/scripts/build.ts index a405e21b1f1..e5f3d0364f4 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -105,15 +105,18 @@ async function main(): Promise { const ibProfile = process.env.IB_PROFILE || join(process.cwd(), ".incredibuild", "ib_profile.xml"); const ibArgs = useIB ? [ - "--profile", ibProfile, - "--caption", `bun-${args.profile ?? "build"}`, + "--profile", + ibProfile, + "--caption", + `bun-${args.profile ?? "build"}`, ...(process.env.IB_FORCE_REMOTE === "1" ? ["--force-remote"] : []), ...(process.env.IB_MAX_LOCAL ? ["--max-local-cores", process.env.IB_MAX_LOCAL] : []), "--", ] : []; const ninjaCmd = useIB ? ibConsole : "ninja"; - const ninjaFullArgv = (cfg: { buildDir: string }) => (useIB ? [...ibArgs, "ninja", ...ninjaArgv(cfg)] : ninjaArgv(cfg)); + const ninjaFullArgv = (cfg: { buildDir: string }) => + useIB ? [...ibArgs, "ninja", ...ninjaArgv(cfg)] : ninjaArgv(cfg); if (isCI) { // CI: machine/env dump + collapsible groups + annotation-on-failure. From d901b42b9931a66f297334049793c5115c750489 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 13:46:35 +0300 Subject: [PATCH 07/10] ci: realign for vnext-processing-engine IB runner architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vnext-processing-engine runner image (nscr.io/.../incredibuild-runner:latest) ships an ib-accel PATH shim that intercepts `ninja`/`cmake`/`cargo` and transparently wraps them with `ib_console --standalone --build-cache-local-shared --build-cache-force --build-cache-basedir=$PWD`. So on `runs-on: incredibuild-runner` we must NOT also call ib_console ourselves — it would double-wrap and the shim guards (__IB_NINJA_WRAPPED, IB_CONSOLE_SKIP) would interact badly. What changed: - Renamed runner label to `incredibuild-runner` (single descriptive label; `self-hosted` is auto-applied by GitHub) — matches vnext's JIT pool registration. - Dropped .github/actions/ib-build/action.yml. The composite action existed to do the ib_console wrap, which is now redundant. The build.ts opt-in patch + .incredibuild/build.sh remain for off-grid use (workstation with ib_linux, traditional self-hosted). - ib-build job now: cp .incredibuild/ib_profile.xml ib_profile.xml (highest-priority slot in ib_console's default load order, since the shim invokes ib_console without --profile) → `bun scripts/build.ts --profile=…` → done. IB_ACCEL_DEBUG=1 surfaces every shim transformation when the workflow_dispatch input is set. - compare workflow uses IB_CONSOLE_SKIP=1 as the "stock" variant (the shim recognizes this and falls back to plain ninja), avoiding any divergence between variants other than the wrap itself. - README documents both paths (vnext runner = default; off-grid = set BUN_USE_INCREDIBUILD=1). Co-Authored-By: Claude Opus 4.7 --- .github/actionlint.yaml | 2 +- .github/actions/ib-build/action.yml | 118 --------------------- .github/workflows/incredibuild-compare.yml | 61 +++++------ .github/workflows/incredibuild-linux.yml | 98 ++++++++--------- .incredibuild/README.md | 28 ++++- 5 files changed, 101 insertions(+), 206 deletions(-) delete mode 100644 .github/actions/ib-build/action.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 4a668b76212..20ebee076c1 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,4 +2,4 @@ # but not on GH-hosted ones, so the linter stops flagging them as typos. self-hosted-runner: labels: - - incredibuild + - incredibuild-runner diff --git a/.github/actions/ib-build/action.yml b/.github/actions/ib-build/action.yml deleted file mode 100644 index 3012b201342..00000000000 --- a/.github/actions/ib-build/action.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: ib-build -description: | - Build Bun under Incredibuild's ib_console. Single source of truth for the - ib_console invocation across all CI jobs. Wraps `bun scripts/build.ts` so - the inner ninja → clang/cargo subtree is distributed across an IB grid. -author: Incredibuild-RND - -inputs: - bun-profile: - description: "Bun build profile (debug | release | ci-cpp-only | ci-release | …)" - required: true - force-remote: - description: "Pass --force-remote to ib_console (push allow_remote off-box)" - required: false - default: "true" - max-local-cores: - description: "Cap local cores used by ib_console (blank = no cap)" - required: false - default: "" - cache-dir: - description: "ib_console --build-cache-basedir (must be absolute). Empty disables cache." - required: false - default: "" - caption: - description: "Build name shown in the IB monitor" - required: false - default: "" - extra-args: - description: "Extra args appended verbatim to ib_console (e.g. --compress 5)" - required: false - default: "" - -outputs: - log-path: - description: "Path to the ib_console monitor log written by this build" - value: ${{ steps.run.outputs.log-path }} - build-dir: - description: "Bun build directory ninja wrote to" - value: ${{ steps.run.outputs.build-dir }} - -runs: - using: composite - steps: - - name: Sanity-check ib_console - shell: bash - run: | - set -euo pipefail - if ! command -v ib_console >/dev/null 2>&1; then - echo "::error::ib_console not on PATH. ib_linux must be installed on this runner." - exit 127 - fi - ib_console --version - ib_console --check-license - test -f "${{ github.workspace }}/.incredibuild/ib_profile.xml" \ - || { echo "::error::missing .incredibuild/ib_profile.xml"; exit 1; } - - - name: Install Bun (if not pre-installed on runner) - shell: bash - run: | - set -euo pipefail - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - echo "$HOME/.bun/bin" >> "$GITHUB_PATH" - fi - for t in clang clang++ ninja cmake cargo rustc; do - command -v "$t" >/dev/null || { echo "::error::missing $t on runner"; exit 1; } - done - - - name: Build under Incredibuild - id: run - shell: bash - env: - BUN_PROFILE: ${{ inputs.bun-profile }} - IB_FORCE_REMOTE_IN: ${{ inputs.force-remote }} - IB_MAX_LOCAL_IN: ${{ inputs.max-local-cores }} - IB_CACHE_DIR_IN: ${{ inputs.cache-dir }} - IB_CAPTION_IN: ${{ inputs.caption }} - IB_EXTRA_ARGS_IN: ${{ inputs.extra-args }} - run: | - set -euo pipefail - log_path="${RUNNER_TEMP:-/tmp}/ib_console-${BUN_PROFILE}.log" - echo "log-path=$log_path" >> "$GITHUB_OUTPUT" - echo "build-dir=build/${BUN_PROFILE}" >> "$GITHUB_OUTPUT" - - ib_args=( - --profile "${{ github.workspace }}/.incredibuild/ib_profile.xml" - --caption "${IB_CAPTION_IN:-gh-bun-$BUN_PROFILE}" - ) - - # Acceleration knobs. - [[ "$IB_FORCE_REMOTE_IN" == "true" ]] && ib_args+=(--force-remote) - [[ -n "$IB_MAX_LOCAL_IN" ]] && ib_args+=(--max-local-cores "$IB_MAX_LOCAL_IN") - - # Build cache: ib_console's own per-user cache backed by GH's - # actions/cache (set up by the caller, restored before this step). - if [[ -n "$IB_CACHE_DIR_IN" ]]; then - mkdir -p "$IB_CACHE_DIR_IN" - ib_args+=( - --build-cache-local-user - --build-cache-basedir "$IB_CACHE_DIR_IN" - --build-cache-local-logfile "$log_path.cache" - ) - fi - - # Pass arbitrary extras through (e.g., --compress 5). - if [[ -n "$IB_EXTRA_ARGS_IN" ]]; then - read -r -a extras <<<"$IB_EXTRA_ARGS_IN" - ib_args+=("${extras[@]}") - fi - - # BUN_INCREDIBUILD_WRAPPED stops scripts/build.ts from wrapping the - # inner ninja in a second ib_console — we wrap the whole subtree here. - export BUN_INCREDIBUILD_WRAPPED=1 - - # Tee to file so the metrics step can parse placement data. - ib_console "${ib_args[@]}" -- \ - bun scripts/build.ts --profile="$BUN_PROFILE" 2>&1 \ - | tee "$log_path" diff --git a/.github/workflows/incredibuild-compare.yml b/.github/workflows/incredibuild-compare.yml index 8812ca51efe..fd7ada5c045 100644 --- a/.github/workflows/incredibuild-compare.yml +++ b/.github/workflows/incredibuild-compare.yml @@ -1,11 +1,9 @@ name: incredibuild-compare -# Side-by-side acceleration benchmark: build the same Bun profile twice -# on the same self-hosted runner — once with Incredibuild, once stock — -# and report the wall-time delta to the job summary. -# -# Manual dispatch only. Designed to be the canonical "is IB still helping?" -# regression check after profile changes or compiler upgrades. +# Side-by-side acceleration benchmark on a vnext IB runner: builds the +# same Bun profile twice on the same runner, once with the IB ninja shim +# active and once with IB_CONSOLE_SKIP=1 (the shim recognizes this and +# falls back to plain ninja). Reports the wall-time delta. on: workflow_dispatch: @@ -22,12 +20,12 @@ on: - ci-rust-only - ci-release iterations: - description: "How many times to run each variant (median is reported)" + description: "Iterations per variant (median reported)" required: false default: "1" type: string clean_between: - description: "Wipe the build dir between iterations (cold build) vs reuse (warm)" + description: "Wipe build dir between iterations (cold) vs reuse (warm)" required: false default: true type: boolean @@ -38,7 +36,7 @@ concurrency: jobs: benchmark: - runs-on: [self-hosted, linux, incredibuild] + runs-on: incredibuild-runner timeout-minutes: 240 env: BUN_PROFILE: ${{ inputs.bun_profile }} @@ -50,56 +48,51 @@ jobs: with: submodules: recursive + - name: Activate Bun-specific IB profile + run: cp .incredibuild/ib_profile.xml ib_profile.xml + - name: Warm dependencies once (excluded from timing) run: | set -euo pipefail - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - echo "$HOME/.bun/bin" >> "$GITHUB_PATH" - fi bun scripts/build.ts --profile="$BUN_PROFILE" --configure-only - - name: Run stock builds + - name: Stock builds (IB_CONSOLE_SKIP=1 disables the ninja shim) id: stock + env: + IB_CONSOLE_SKIP: "1" run: | set -euo pipefail - times_file=/tmp/stock-times.txt - : > "$times_file" + times=/tmp/stock-times.txt + : > "$times" for i in $(seq 1 "$ITERATIONS"); do [[ "$CLEAN" == "true" ]] && rm -rf "build/$BUN_PROFILE" t0=$(date +%s.%N) bun scripts/build.ts --profile="$BUN_PROFILE" t1=$(date +%s.%N) dt=$(awk -v a="$t0" -v b="$t1" 'BEGIN{print b-a}') - echo "$dt" >> "$times_file" - echo "iteration $i: ${dt}s" >&2 + echo "$dt" >> "$times" + echo "stock iter $i: ${dt}s" >&2 done - median=$(sort -n "$times_file" | awk 'NR==int(NR/2)+1' ) + median=$(sort -n "$times" | awk 'NR==int(NR/2)+1') echo "median=$median" >> "$GITHUB_OUTPUT" - cat "$times_file" - - name: Run IB-wrapped builds + - name: IB-accelerated builds (shim active) id: ib run: | set -euo pipefail - times_file=/tmp/ib-times.txt - : > "$times_file" + times=/tmp/ib-times.txt + : > "$times" for i in $(seq 1 "$ITERATIONS"); do [[ "$CLEAN" == "true" ]] && rm -rf "build/$BUN_PROFILE" t0=$(date +%s.%N) - BUN_INCREDIBUILD_WRAPPED=1 \ - ib_console --profile "$PWD/.incredibuild/ib_profile.xml" \ - --force-remote \ - --caption "bench-$BUN_PROFILE-$i" \ - -- bun scripts/build.ts --profile="$BUN_PROFILE" + bun scripts/build.ts --profile="$BUN_PROFILE" t1=$(date +%s.%N) dt=$(awk -v a="$t0" -v b="$t1" 'BEGIN{print b-a}') - echo "$dt" >> "$times_file" - echo "iteration $i: ${dt}s" >&2 + echo "$dt" >> "$times" + echo "ib iter $i: ${dt}s" >&2 done - median=$(sort -n "$times_file" | awk 'NR==int(NR/2)+1' ) + median=$(sort -n "$times" | awk 'NR==int(NR/2)+1') echo "median=$median" >> "$GITHUB_OUTPUT" - cat "$times_file" - name: Report run: | @@ -112,8 +105,8 @@ jobs: echo echo "| variant | median wall (s) |" echo "| --- | --- |" - echo "| stock | $stock |" - echo "| ib_console --force-remote | $ib |" + echo "| stock (IB_CONSOLE_SKIP=1) | $stock |" + echo "| ib-accelerated | $ib |" echo echo "**delta:** ${delta}s saved · **speedup:** ${speedup}×" echo diff --git a/.github/workflows/incredibuild-linux.yml b/.github/workflows/incredibuild-linux.yml index ed3e6b02365..64c704cef06 100644 --- a/.github/workflows/incredibuild-linux.yml +++ b/.github/workflows/incredibuild-linux.yml @@ -1,14 +1,19 @@ name: incredibuild-linux -# Build Bun on Linux via Incredibuild (ib_console). +# Build Bun on a vnext-processing-engine Incredibuild runner. +# +# The runner image (nscr.io/.../incredibuild-runner:latest, from +# Incredibuild-RND/vnext-processing-engine) already PATH-shims `ninja`, +# `cmake`, `cargo`, etc. to invoke `ib_console --standalone +# --build-cache-local-shared --build-cache-force --build-cache-basedir=$PWD` +# transparently. So we just need to: +# 1. Drop our Bun-specific profile at $PWD/ib_profile.xml (highest +# priority in ib_console's default profile load order). +# 2. Run `bun scripts/build.ts` normally — the shim does the rest. # # Two jobs: -# 1. validate-profile-xsd (ubuntu-latest) — always runs, no IB needed. -# 2. ib-build (self-hosted) — needs runner with ib_linux -# installed + coordinator -# registered, label -# `[self-hosted, linux, -# incredibuild]`. +# 1. validate-profile-xsd (ubuntu-latest) — runs on every push/PR. +# 2. ib-build (incredibuild-runner) — workflow_dispatch only. on: workflow_dispatch: inputs: @@ -24,20 +29,10 @@ on: - ci-rust-only - ci-link-only - ci-release - force_remote: - description: "Pass --force-remote to ib_console" - required: false - default: true - type: boolean - max_local_cores: - description: "Cap local cores (blank = no cap)" + debug_shim: + description: "Set IB_ACCEL_DEBUG=1 so the shim logs every transformation" required: false - default: "" - type: string - use_cache: - description: "Use IB build cache (actions/cache-backed)" - required: false - default: true + default: false type: boolean push: branches: @@ -45,7 +40,6 @@ on: pull_request: paths: - ".incredibuild/**" - - ".github/actions/ib-build/**" - ".github/actions/ib-metrics/**" - ".github/workflows/incredibuild-linux.yml" - "scripts/build.ts" @@ -59,53 +53,55 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install xmllint - run: sudo apt-get update && sudo apt-get install -y libxml2-utils + - name: Install xmllint + shellcheck + run: sudo apt-get update && sudo apt-get install -y libxml2-utils shellcheck - name: Validate .incredibuild/ib_profile.xml run: ./.incredibuild/validate.sh - name: Shellcheck wrapper scripts - run: | - sudo apt-get install -y shellcheck - shellcheck .incredibuild/build.sh .incredibuild/validate.sh + run: shellcheck .incredibuild/build.sh .incredibuild/validate.sh ib-build: needs: validate-profile-xsd - # Self-hosted only: runs on manual dispatch. Pushes/PRs would queue - # indefinitely if no runner is registered. if: ${{ github.event_name == 'workflow_dispatch' }} - runs-on: [self-hosted, linux, incredibuild] + runs-on: incredibuild-runner timeout-minutes: 120 env: BUN_PROFILE: ${{ inputs.bun_profile || 'ci-cpp-only' }} + IB_ACCEL_DEBUG: ${{ inputs.debug_shim == true && '1' || '0' }} steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Restore IB build cache - if: ${{ inputs.use_cache != false }} - id: ib-cache - uses: actions/cache@v4 - with: - path: ${{ runner.temp }}/ib-cache - # Key on profile + lockfiles that influence outputs. Adjust as the - # Bun build's input set evolves. - key: ib-cache-${{ runner.os }}-${{ env.BUN_PROFILE }}-${{ hashFiles('Cargo.lock', 'package.json', 'scripts/build/**', 'src/**/build.rs') }} - restore-keys: | - ib-cache-${{ runner.os }}-${{ env.BUN_PROFILE }}- - ib-cache-${{ runner.os }}- + - name: Activate Bun-specific IB profile + run: | + set -euo pipefail + # vnext runner's ninja shim wraps with `ib_console --standalone …` + # without --profile, so ib_console resolves via the default load + # order. $PWD/ib_profile.xml is the highest-priority slot. + cp .incredibuild/ib_profile.xml ib_profile.xml + xmllint --noout --schema .incredibuild/ib_profile.xsd ib_profile.xml + echo "[ib] profile active at $PWD/ib_profile.xml" + + - name: Show effective IB env + run: | + echo "ib_console: $(command -v ib_console || echo 'not on PATH')" + ib_console --version 2>/dev/null || true + echo "ninja shim: $(command -v ninja)" + echo "IB_ACCEL_DIR=${IB_ACCEL_DIR:-}" + echo "IB_ACCEL_DEBUG=${IB_ACCEL_DEBUG:-}" - - name: Build under Incredibuild + - name: Build Bun id: build - uses: ./.github/actions/ib-build - with: - bun-profile: ${{ env.BUN_PROFILE }} - force-remote: ${{ inputs.force_remote == true && 'true' || 'false' }} - max-local-cores: ${{ inputs.max_local_cores }} - cache-dir: ${{ inputs.use_cache != false && format('{0}/ib-cache', runner.temp) || '' }} - caption: "gh-${{ env.BUN_PROFILE }}-${{ github.run_id }}" + run: | + set -o pipefail + log="${RUNNER_TEMP:-/tmp}/ib-build-${BUN_PROFILE}.log" + echo "log-path=$log" >> "$GITHUB_OUTPUT" + # The PATH shim does the ib_console wrapping for ninja transparently. + # We just need to invoke the normal Bun build entry point. + bun scripts/build.ts --profile="$BUN_PROFILE" 2>&1 | tee "$log" - name: Acceleration metrics if: always() @@ -114,12 +110,12 @@ jobs: log-path: ${{ steps.build.outputs.log-path }} label: "Incredibuild — ${{ env.BUN_PROFILE }}" - - name: Upload ib_console log + - name: Upload IB build log if: always() uses: actions/upload-artifact@v4 with: name: ib-log-${{ env.BUN_PROFILE }} - path: ${{ steps.build.outputs.log-path }}* + path: ${{ steps.build.outputs.log-path }} if-no-files-found: ignore retention-days: 14 diff --git a/.incredibuild/README.md b/.incredibuild/README.md index f896f2b9101..2db18d97ca3 100644 --- a/.incredibuild/README.md +++ b/.incredibuild/README.md @@ -64,11 +64,35 @@ BUN_USE_INCREDIBUILD=1 IB_FORCE_REMOTE=1 \ bun scripts/build.ts --profile=ci-cpp-only ``` -This path is what `.github/workflows/incredibuild-linux.yml` exercises on -self-hosted runners tagged `[self-hosted, linux, incredibuild]`. +This is for **off-grid** use (workstation with ib_linux installed, +traditional self-hosted setups). On a vnext runner you don't need it — +see below. + +## On a vnext-processing-engine IB runner + +`Incredibuild-RND/vnext-processing-engine`'s runner image ships an +[ib-accel](https://github.com/Incredibuild-RND/vnext-processing-engine/tree/main/src/runner_engine/build/ib-accel) +PATH shim that intercepts `ninja`/`cmake`/`cargo`/etc. and wraps them +with `ib_console --standalone --build-cache-local-shared +--build-cache-force --build-cache-basedir=$PWD` transparently. So on +`runs-on: incredibuild-runner`, the job only needs to: + +1. Drop our Bun-specific profile at `$PWD/ib_profile.xml` (highest + priority in `ib_console`'s default load order). +2. Run `bun scripts/build.ts --profile=…` normally. + +That's exactly what `.github/workflows/incredibuild-linux.yml` does. +**Do not** also set `BUN_USE_INCREDIBUILD=1` on a vnext runner — the +shim already wraps; double-wrapping breaks. Set `IB_ACCEL_DEBUG=1` if +you want to see every shim transformation. ## Prerequisites on the build machine +For a **vnext IB runner** (the recommended path): nothing — the image +ships ib_linux, ib_console, the PATH shim, the Bun toolchain, and the +coordinator/helper grid wiring out of the box. + +For **off-grid** use: 1. ib_linux installed → `ib_console` on `PATH` (usually at `/opt/incredibuild/bin/ib_console`). 2. Machine registered with an Incredibuild coordinator (initiator role). From 1e931416ff1ee6a0aa291fe0f0e60c1fe4a5cd7e Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 13:56:03 +0300 Subject: [PATCH 08/10] =?UTF-8?q?.incredibuild:=20add=20ACCELERATION.md=20?= =?UTF-8?q?=E2=80=94=20what,=20how,=20by=20how=20much?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Concrete model anchored in measurement, not vibes: - WHAT: 564 Bun-core TUs + ~1,300 vendored-dep TUs (boringssl 143, lsquic 69, libdeflate 11, highway 9, tinycc 6, +200 across array-based deps) + 105 Rust crates. Link/ar/strip stay local. - HOW: vnext runner PATH-shims ninja → ib_console intercepts each `clang -c` and ships it to a helper; link stays local. Off-grid uses the build.ts opt-in or .incredibuild/build.sh. Same profile both ways. - BY HOW MUCH: measured T_per_TU = 0.44s on x86_64 clang 18.1.3 (10 vCPU container), 5.36x on 10 local cores. Projects to ~24x on a 30-helper grid for cpp-only, asymptote ~40x bound by ar/link. Release-LTO is Amdahl-bound at link → 5-15x ceiling. Incremental re-runs land at 5-15s wall via --build-cache-local-shared. Numbers labeled as derived/projected vs measured; the real stock-vs-IB delta will come from the incredibuild-compare.yml workflow once a vnext runner is provisioned. Co-Authored-By: Claude Opus 4.7 --- .incredibuild/ACCELERATION.md | 138 ++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 .incredibuild/ACCELERATION.md diff --git a/.incredibuild/ACCELERATION.md b/.incredibuild/ACCELERATION.md new file mode 100644 index 00000000000..d7eb45c3eda --- /dev/null +++ b/.incredibuild/ACCELERATION.md @@ -0,0 +1,138 @@ +# Acceleration model — what, how, by how much + +What this integration accelerates on a Bun build, the mechanism, and the +expected wall-time delta. + +## What we accelerate + +| Category | Count (this fork) | Profile rule | Distributable | +| --- | ---: | --- | --- | +| **Bun core C++ TUs** (`src/**/*.cpp`) | 554 | `clang++` `local_only` + `exclude_args="-c"` | yes (compile-only) | +| **Bun core C TUs** (`src/**/*.c`) | 10 | `clang` `local_only` + `exclude_args="-c"` | yes | +| **boringssl** | 143 | `clang` (compile) | yes | +| **lsquic** (HTTP/3 QUIC) | 69 | `clang` (compile) | yes | +| **libdeflate** | 11 | `clang` (compile) | yes | +| **highway** (SIMD) | 9 | `clang++` (compile) | yes | +| **tinycc** | 6 | `clang` (compile) | yes | +| libuv (windows-only), libarchive, c-ares, brotli, sqlite, mimalloc, zstd, libwebp, libjpeg-turbo, libspng, lshpack, hdrhistogram, picohttpparser | ~200 (array-based source lists, vary by platform) | `clang`/`clang++` | yes | +| **Rust crates** (workspace) | 105 | `rustc` `allow_remote` minus `build_script_build` | yes (codegen units distributable) | +| Linker (`ld`/`ld.lld`) | 1 (per artifact) | `intercepted` | **no — stays local** (LTO needs all objects on one box) | +| `ar`/`ranlib`/`strip`/`objcopy` | per artifact | `intercepted` | **no — state consolidation** | +| Codegen scripts (TS via `bun`) | many | `intercepted recursive` | no (host-bound) | + +**Total distributable C/C++ TUs: ~1,300–1,500** depending on platform glob expansion. + +## How we accelerate + +Two integration modes — same profile, different invocation: + +### vnext IB runner (`runs-on: incredibuild-runner`) +1. Runner image + ([vnext-processing-engine/src/runner_engine/build](https://github.com/Incredibuild-RND/vnext-processing-engine/tree/main/src/runner_engine/build)) + sources `ib-accel/setup.sh` in its entrypoint. +2. PATH gets `…/ib-accel/bin` prepended; shim scripts shadow `ninja`, + `cmake`, `cargo`, `make`, `docker`, `npm`/`pnpm`/`yarn`. +3. Ninja shim invokes + `ib_console --standalone --build-cache-local-shared --build-cache-force --build-cache-basedir=$PWD -j $(nproc) "$@"`. +4. `ib_console` resolves the active profile via its default load order; + our `.incredibuild/ib_profile.xml` is copied to `$PWD/ib_profile.xml` + at job start so it wins. +5. The LD_PRELOAD interceptor inside `ib_console` matches each spawned + process against the profile; `clang -c` invocations are distributed + to helpers, `clang … -o bin` (link, no `-c`) stays local. + +### Off-grid (workstation, traditional self-hosted) +1. `BUN_USE_INCREDIBUILD=1 bun scripts/build.ts --profile=…` — our + `scripts/build.ts` patch wraps the inner `ninja` spawn with + `ib_console --profile … --`. +2. Same per-process interception logic as above. No shim involved. + +In both modes the build cache (ccache-backed, +`--build-cache-local-shared`) catches incremental rebuilds; the +distributed grid only sees cache misses. + +## By how much — measurement + model + +### Measured baseline + +Real `clang++ -O2 -std=c++17 -c` on a representative Bun-class template-heavy +C++ TU, x86_64 Ubuntu 24.04 + clang 18.1.3, 10 vCPU: + +| metric | value | +| --- | ---: | +| T per TU (median of 3, single TU) | **0.44 s** | +| Serial wall, 32 TUs `-j1` | 13.9 s (0.43 s/TU sustained) | +| Parallel wall, 32 TUs `-j10` (10 cores) | 2.60 s | +| Local parallel speedup | **5.36×** | + +The 5.36× / 10-core ratio (vs the 10× ceiling) is the usual local +contention loss — disk, malloc, frontend pipeline. IB's distributed +helpers don't share those resources, so the per-helper efficiency is +closer to 1. + +### Projected wall time, full cpp-only phase + +Assuming ~1,300 distributable C/C++ TUs at the measured 0.44 s/TU +(conservative — real Bun TUs with the WTF+JSC PCH average **2–4 s** +under `-O3`, so this is a floor): + +| K (effective helpers) | Compile wall | + ar/link | Total wall | Speedup vs serial | +| ---: | ---: | ---: | ---: | ---: | +| 1 (serial) | 572 s | 5 s | **577 s** (~9.6 min) | 1× | +| 8 (laptop) | 72 s | 5 s | **77 s** (~1.3 min) | 7.5× | +| 16 | 36 s | 5 s | **41 s** | 14× | +| 30 (modest grid) | 19 s | 5 s | **24 s** | 24× | +| 60 | 9.5 s | 5 s | **14.5 s** | 40× | +| 100 (dispatch-bound) | 5.7 s + 0.1×100 overhead | 5 s | **~21 s** | ~28× (asymptote) | + +For realistic Bun TUs (2 s/TU floor): multiply the compile column by ~4.5. +A 30-helper grid lands cpp-only at ~90 s instead of ~40 minutes serial. + +### Amdahl ceiling on full release builds + +The link with LTO is single-threaded and dominates as K grows: + +``` +T_total = T_compile / K + T_link +``` + +| profile | T_compile (serial) | T_link | Ceiling at K=∞ | +| --- | ---: | ---: | ---: | +| ci-cpp-only | ~10 min | ~5 s (ar) | **~5 s** | +| ci-rust-only | ~25 min | included | bounded by slowest crate's codegen | +| ci-release (LTO on) | cpp+rust ~35 min | 5–15 min LTO link | **5–15 min** | +| debug (no LTO) | ~15 min | ~30 s | **~30 s** | + +So a debug build benefits the most asymptotically; a release build with +LTO hits the link wall around 5–10×, no matter how big the grid. + +### What still needs measurement on real infra + +These numbers are derived from a real per-TU measurement + the published +IB distribution model. They are NOT yet validated end-to-end against a +real vnext IB runner because no `incredibuild-runner`-labeled instance +exists on the GH Actions side for this repo. Once one is provisioned: + +```bash +gh workflow run incredibuild-compare.yml \ + -F bun_profile=ci-cpp-only \ + -F iterations=3 \ + -F clean_between=true +``` + +That run produces a real (stock vs IB-accelerated) median wall delta +and writes it to `$GITHUB_STEP_SUMMARY` — the canonical answer. + +### Cache-hit re-runs (incremental) + +`ib_console --build-cache-local-shared --build-cache-force` records +each compile keyed on (compiler, flags, source content + transitive +header content). On a re-run with no source changes: + +- Bun cpp-only: ~99% cache hit → wall dominated by ninja's stat walk + + `ar` link → **~5–15 s total** regardless of grid size. +- Bun cpp-only with a single .cpp touched in `src/jsc/`: 1 cache miss + → 1 TU compiled → re-link → **~3–6 s on a 30-helper grid**. + +This is the warmest-path acceleration and the dominant case in CI on +PR pushes that touch a small file set. From 95ded93383207385d09f6d147b199a059117a776 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 13:59:58 +0300 Subject: [PATCH 09/10] =?UTF-8?q?.incredibuild:=20correct=20ACCELERATION.m?= =?UTF-8?q?d=20=E2=80=94=20cache-only,=20no=20helpers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous model assumed a remote-helper distribution grid (24x at K=30, 40x asymptote). That's WRONG for this deployment. vnext's ninja shim hard-codes `ib_console --standalone --build-cache-local-shared --build-cache-force` (line 65 of src/runner_engine/build/ib-accel/bin/ninja), so: - No distribution. Every compile runs locally. - Acceleration comes entirely from the content-addressed compile cache. - The cache lives in $PWD via --build-cache-basedir; vnext promotes cache/ across JIT pod runs via the /ib-workspace virtiofs mount. Corrected speedup curve as a function of cache HIT RATE (not helper count): 0% hit (cold first build) -> ~1.0x (~1% hashing overhead) 50% hit (rebase) -> ~1.8x 90% hit (small refactor) -> ~7x 99% hit (PR re-push, 1 file) -> ~15x 100% hit (CI re-run same SHA)-> ~30x (capped by ar/link) Per-profile ceilings: debug builds get the most (~30x best case), ci-release with LTO is bound by uncached link time (5-15x ceiling). Cold-cache builds get no speedup — that's the honest answer. The acceleration is a function of the cache hit rate, which is high in typical CI patterns (PR pushes touching small file sets) and low after large refactors or compiler upgrades. Real numbers still need a `gh workflow run incredibuild-compare.yml` on a vnext runner with the cache pre-warmed. Co-Authored-By: Claude Opus 4.7 --- .incredibuild/ACCELERATION.md | 240 +++++++++++++++++++--------------- 1 file changed, 134 insertions(+), 106 deletions(-) diff --git a/.incredibuild/ACCELERATION.md b/.incredibuild/ACCELERATION.md index d7eb45c3eda..dc3dd247a41 100644 --- a/.incredibuild/ACCELERATION.md +++ b/.incredibuild/ACCELERATION.md @@ -3,136 +3,164 @@ What this integration accelerates on a Bun build, the mechanism, and the expected wall-time delta. +> **Mode: cache-only (`--standalone`).** vnext-processing-engine's ninja +> shim invokes `ib_console --standalone --build-cache-local-shared +> --build-cache-force --build-cache-basedir=$PWD`. There is **no** remote +> distribution to helpers — `--standalone` explicitly disables that. The +> entire acceleration comes from the content-addressed compile cache. + ## What we accelerate -| Category | Count (this fork) | Profile rule | Distributable | -| --- | ---: | --- | --- | -| **Bun core C++ TUs** (`src/**/*.cpp`) | 554 | `clang++` `local_only` + `exclude_args="-c"` | yes (compile-only) | -| **Bun core C TUs** (`src/**/*.c`) | 10 | `clang` `local_only` + `exclude_args="-c"` | yes | -| **boringssl** | 143 | `clang` (compile) | yes | -| **lsquic** (HTTP/3 QUIC) | 69 | `clang` (compile) | yes | -| **libdeflate** | 11 | `clang` (compile) | yes | -| **highway** (SIMD) | 9 | `clang++` (compile) | yes | -| **tinycc** | 6 | `clang` (compile) | yes | -| libuv (windows-only), libarchive, c-ares, brotli, sqlite, mimalloc, zstd, libwebp, libjpeg-turbo, libspng, lshpack, hdrhistogram, picohttpparser | ~200 (array-based source lists, vary by platform) | `clang`/`clang++` | yes | -| **Rust crates** (workspace) | 105 | `rustc` `allow_remote` minus `build_script_build` | yes (codegen units distributable) | -| Linker (`ld`/`ld.lld`) | 1 (per artifact) | `intercepted` | **no — stays local** (LTO needs all objects on one box) | -| `ar`/`ranlib`/`strip`/`objcopy` | per artifact | `intercepted` | **no — state consolidation** | -| Codegen scripts (TS via `bun`) | many | `intercepted recursive` | no (host-bound) | - -**Total distributable C/C++ TUs: ~1,300–1,500** depending on platform glob expansion. - -## How we accelerate - -Two integration modes — same profile, different invocation: - -### vnext IB runner (`runs-on: incredibuild-runner`) -1. Runner image - ([vnext-processing-engine/src/runner_engine/build](https://github.com/Incredibuild-RND/vnext-processing-engine/tree/main/src/runner_engine/build)) - sources `ib-accel/setup.sh` in its entrypoint. -2. PATH gets `…/ib-accel/bin` prepended; shim scripts shadow `ninja`, - `cmake`, `cargo`, `make`, `docker`, `npm`/`pnpm`/`yarn`. -3. Ninja shim invokes - `ib_console --standalone --build-cache-local-shared --build-cache-force --build-cache-basedir=$PWD -j $(nproc) "$@"`. -4. `ib_console` resolves the active profile via its default load order; - our `.incredibuild/ib_profile.xml` is copied to `$PWD/ib_profile.xml` - at job start so it wins. -5. The LD_PRELOAD interceptor inside `ib_console` matches each spawned - process against the profile; `clang -c` invocations are distributed - to helpers, `clang … -o bin` (link, no `-c`) stays local. - -### Off-grid (workstation, traditional self-hosted) -1. `BUN_USE_INCREDIBUILD=1 bun scripts/build.ts --profile=…` — our - `scripts/build.ts` patch wraps the inner `ninja` spawn with - `ib_console --profile … --`. -2. Same per-process interception logic as above. No shim involved. - -In both modes the build cache (ccache-backed, -`--build-cache-local-shared`) catches incremental rebuilds; the -distributed grid only sees cache misses. - -## By how much — measurement + model +We accelerate **incremental rebuilds via cache hits**, not cold builds. +A from-scratch first build sees little to no speedup (every compile +misses, hashing adds 1–3% overhead). The win shows up the second time +the same content compiles — PR re-pushes, CI re-runs of the same SHA, +branch switches, base-branch rebases. + +The cached units are every process matched by the profile (see +`ib_profile.xml`): + +| Process | Profile rule | Cached? | +| --- | --- | --- | +| `clang -c` / `clang++ -c` / `gcc -c` / `g++ -c` | `local_only` + `exclude_args="-c:-S:-E"` + `cached="true"` | **yes** (compile only) | +| `clang … -o exe` (link, no `-c`) | matches `local_only` (no exclusion) | no — stays local, no cache | +| `rustc ` | `allow_remote` minus `build_script_build` | yes (ccache backend) | +| `cargo` / `ninja` / `cmake` / `bun` | `intercepted recursive` | not cached themselves (they orchestrate) | +| `bison` / `flex` / `m4` / `nasm` / `yasm` / `as` | `allow_remote` | yes | +| `ld` / `ld.lld` / `ar` / `ranlib` / `strip` / `objcopy` | `intercepted` | no (state consolidation) | + +## What gets counted + +Concrete TU counts in this fork: + +| Component | TUs | Cached on hit | +| --- | ---: | :---: | +| Bun core `src/**/*.cpp` | 554 | ✓ | +| Bun core `src/**/*.c` | 10 | ✓ | +| boringssl | 143 | ✓ | +| lsquic (HTTP/3 QUIC) | 69 | ✓ | +| libdeflate | 11 | ✓ | +| highway (SIMD) | 9 | ✓ | +| tinycc | 6 | ✓ | +| libuv (Windows-only), libarchive, c-ares, brotli, sqlite, mimalloc, zstd, libwebp, libjpeg-turbo, libspng, lshpack, hdrhistogram, picohttpparser | ~200 (array-based, platform-varying) | ✓ | +| Rust crates (workspace) | 105 | ✓ (codegen units) | +| Final link (one per artifact) | — | ✗ | + +**Total cacheable C/C++ TUs: ~1,300–1,500.** + +## How it accelerates + +ib_console computes a hash of each compile invocation: + +``` +hash = H(compiler-binary-content, + exact-flag-string, + source-file-content, + transitive-header-content) +``` + +On a hit, the cached `.o` is copied out instead of running the +compiler. On a miss, the compiler runs normally and the result is +written into the cache. `--build-cache-force` skips the default +"is-it-safe-to-cache" heuristics (it's safe — the hash already covers +every input). + +**Cache lifetime on vnext runners**: `--build-cache-basedir=$PWD` puts +the cache in the job's working directory. The vnext orchestrator +bind-mounts `/ib-workspace/cache` from the host via virtiofs and +promotes cache contents across runs (see +`vnext-processing-engine/src/runner_engine/build/entrypoint.sh` +line ~108 — `cache/` is the only directory promoted post-job). + +## How it integrates here + +Two paths, same `.incredibuild/ib_profile.xml`: + +- **vnext IB runner** (`runs-on: incredibuild-runner`) — the ib-accel + PATH shim invokes `ib_console --standalone --build-cache-local-shared + --build-cache-force …` automatically. Profile resolved from + `$PWD/ib_profile.xml` (we copy it there in the job's first step). +- **Off-grid** (`BUN_USE_INCREDIBUILD=1`) — `scripts/build.ts` wraps + the inner `ninja` spawn with explicit `ib_console --profile … --`. + Same per-process matching. + +## By how much ### Measured baseline -Real `clang++ -O2 -std=c++17 -c` on a representative Bun-class template-heavy -C++ TU, x86_64 Ubuntu 24.04 + clang 18.1.3, 10 vCPU: +`clang++ -O2 -std=c++17 -c` on a representative Bun-class +template-heavy TU, x86_64 Ubuntu 24.04 + clang 18.1.3, 10 vCPU: | metric | value | | --- | ---: | -| T per TU (median of 3, single TU) | **0.44 s** | -| Serial wall, 32 TUs `-j1` | 13.9 s (0.43 s/TU sustained) | -| Parallel wall, 32 TUs `-j10` (10 cores) | 2.60 s | -| Local parallel speedup | **5.36×** | +| T per TU (median of 3) | **0.44 s** | +| Serial wall, 32 TUs | 13.9 s | +| Parallel `-j10` wall, 32 TUs | 2.60 s (5.36× from native multi-core) | -The 5.36× / 10-core ratio (vs the 10× ceiling) is the usual local -contention loss — disk, malloc, frontend pipeline. IB's distributed -helpers don't share those resources, so the per-helper efficiency is -closer to 1. +Real Bun TUs with the WTF + JSC PCH on `-O3` are heavier — call it +2 s/TU floor. With ~1,300 cacheable TUs, a cold serial cpp-only is +~9.6–43 minutes; with ninja `-j$(nproc)` on a 16-core CI box it's +**~3–10 min**. That's the stock baseline IB has to beat. -### Projected wall time, full cpp-only phase +### Cache-hit speedup curve -Assuming ~1,300 distributable C/C++ TUs at the measured 0.44 s/TU -(conservative — real Bun TUs with the WTF+JSC PCH average **2–4 s** -under `-O3`, so this is a floor): +The cache lookup is dominated by hashing the source + transitive +headers. Typical per-file lookup is 5–20 ms. -| K (effective helpers) | Compile wall | + ar/link | Total wall | Speedup vs serial | +| hit rate | misses (compile) | hits (lookup) | wall (estimate) | speedup vs stock-parallel | | ---: | ---: | ---: | ---: | ---: | -| 1 (serial) | 572 s | 5 s | **577 s** (~9.6 min) | 1× | -| 8 (laptop) | 72 s | 5 s | **77 s** (~1.3 min) | 7.5× | -| 16 | 36 s | 5 s | **41 s** | 14× | -| 30 (modest grid) | 19 s | 5 s | **24 s** | 24× | -| 60 | 9.5 s | 5 s | **14.5 s** | 40× | -| 100 (dispatch-bound) | 5.7 s + 0.1×100 overhead | 5 s | **~21 s** | ~28× (asymptote) | - -For realistic Bun TUs (2 s/TU floor): multiply the compile column by ~4.5. -A 30-helper grid lands cpp-only at ~90 s instead of ~40 minutes serial. +| 0% (cold) | 1,300 × 2.0 s | 0 | **~10 min** (= stock + ~1% overhead) | ~1.0× | +| 50% (rebase) | 650 × 2.0 s | 650 × 0.01 s | **~5.5 min** | ~1.8× | +| 90% (small refactor) | 130 × 2.0 s | 1,170 × 0.01 s | **~80 s** | ~7× | +| 99% (PR re-push touching 1 file) | 13 × 2.0 s | 1,287 × 0.01 s | **~40 s** | ~15× | +| 100% (CI re-run same SHA) | 0 | 1,300 × 0.01 s | **~13 s + link** ≈ **20 s** | ~30× | -### Amdahl ceiling on full release builds +(Assuming the 16-core parallel stock baseline ≈ 10 min. With LTO link +included the percentages shift — link is uncached and dominates as +hit rate approaches 100%, capping the ceiling at the link time.) -The link with LTO is single-threaded and dominates as K grows: +### Per-profile expectations -``` -T_total = T_compile / K + T_link -``` +| Bun profile | Stock parallel | Best-case cached re-run | Notes | +| --- | ---: | ---: | --- | +| `debug` | 3–6 min | **20–40 s** | no LTO; link is fast; cache is everything | +| `ci-cpp-only` | 8–15 min | **15–30 s** | static-lib output; link is just `ar` | +| `ci-rust-only` | 15–30 min | **30–90 s** | rustc codegen units cached; build.rs not | +| `ci-release` (LTO on) | 30–60 min | **6–16 min** | LTO link uncached, sets the ceiling | -| profile | T_compile (serial) | T_link | Ceiling at K=∞ | -| --- | ---: | ---: | ---: | -| ci-cpp-only | ~10 min | ~5 s (ar) | **~5 s** | -| ci-rust-only | ~25 min | included | bounded by slowest crate's codegen | -| ci-release (LTO on) | cpp+rust ~35 min | 5–15 min LTO link | **5–15 min** | -| debug (no LTO) | ~15 min | ~30 s | **~30 s** | +LTO link time is the Amdahl wall for release builds. The cache pulls +compile to near zero; nothing accelerates the link. -So a debug build benefits the most asymptotically; a release build with -LTO hits the link wall around 5–10×, no matter how big the grid. +### What still needs measurement -### What still needs measurement on real infra +These numbers are derived from one measured `T_per_TU` (0.44 s on a +synthetic Bun-class TU) plus the documented `--build-cache` behavior. +Real Bun TUs are heavier (PCH + WebKit), and the cache hit rate in +practice depends on how many transitive headers change per commit — +a touch to `root-pch.h` invalidates the world. -These numbers are derived from a real per-TU measurement + the published -IB distribution model. They are NOT yet validated end-to-end against a -real vnext IB runner because no `incredibuild-runner`-labeled instance -exists on the GH Actions side for this repo. Once one is provisioned: +To get the real number, on a vnext runner with the cache pre-warmed: ```bash gh workflow run incredibuild-compare.yml \ -F bun_profile=ci-cpp-only \ -F iterations=3 \ - -F clean_between=true + -F clean_between=false # keep build dir for warm-cache measurement ``` -That run produces a real (stock vs IB-accelerated) median wall delta -and writes it to `$GITHUB_STEP_SUMMARY` — the canonical answer. - -### Cache-hit re-runs (incremental) - -`ib_console --build-cache-local-shared --build-cache-force` records -each compile keyed on (compiler, flags, source content + transitive -header content). On a re-run with no source changes: - -- Bun cpp-only: ~99% cache hit → wall dominated by ninja's stat walk + - `ar` link → **~5–15 s total** regardless of grid size. -- Bun cpp-only with a single .cpp touched in `src/jsc/`: 1 cache miss - → 1 TU compiled → re-link → **~3–6 s on a 30-helper grid**. - -This is the warmest-path acceleration and the dominant case in CI on -PR pushes that touch a small file set. +The "stock" arm sets `IB_CONSOLE_SKIP=1` so the shim falls back to +plain ninja — that gives the apples-to-apples delta. + +### What doesn't help + +- **Cold CI builds on a fresh branch with no warm cache** — same wall + as stock, ±1%. The acceleration story for this case requires either + pre-populating the cache from a recent main-branch build, or + switching to a remote-distribution config (not what vnext does + today). +- **Touching `root-pch.h` or a heavily-included header** — invalidates + the dependent TUs' cache entries by content hash. Worst case behaves + like a cold build. +- **Compiler / flag upgrades** — bump clang version or change `-O` + level and the cache misses on everything (correct behavior; the + hash includes compiler binary + flags). From 60055fb7e74e41814316ba1de86f8b7ddb45323b Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Wed, 20 May 2026 14:00:36 +0300 Subject: [PATCH 10/10] .incredibuild/README: lead with cache, not distribution vnext runs ib_console --standalone, so the distribution narrative was misleading. README now matches ACCELERATION.md: cache-only, ~1.5k cacheable TUs, big wins on incremental, ~1x on cold. Co-Authored-By: Claude Opus 4.7 --- .incredibuild/README.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.incredibuild/README.md b/.incredibuild/README.md index 2db18d97ca3..b43ea539658 100644 --- a/.incredibuild/README.md +++ b/.incredibuild/README.md @@ -1,8 +1,14 @@ # Incredibuild integration for Bun Wraps the Ninja-driven Bun build with [`ib_console`][ib_linux] so the -~2,000 C/C++ TUs (Bun core + boringssl + WebKit + libuv + libarchive + …) -and the 107-crate Rust workspace get distributed across an Incredibuild grid. +~1,500 C/C++ TUs (Bun core + boringssl + lsquic + libdeflate + …) and the +105-crate Rust workspace go through a **content-addressed compile cache**. +Re-runs on the same SHA, PR re-pushes, and small refactors get the big +wins; cold builds get ~1×. + +The deployment mode on vnext-processing-engine runners is `--standalone` +(cache only, no remote helpers) — see +[`ACCELERATION.md`](./ACCELERATION.md) for the full speedup curve. [ib_linux]: https://github.com/Incredibuild-RND/ib_linux @@ -24,14 +30,15 @@ it sets `BUN_INCREDIBUILD_WRAPPED=1` so we don't nest ib_console twice. ## Profile in one paragraph `clang` / `clang++` / `gcc` / `g++` are listed as `local_only` with -`exclude_args="-c:-S:-E"`. This is the canonical IB pattern: the rule -doesn't fire when a compile flag is present, so `clang -c foo.cpp` falls -through to the default distributable path while `clang foo.o bar.o -o out` -(linking, no -c) stays pinned to the initiator. `ninja`, `cmake`, `cargo`, -and `bun` itself are `intercepted recursive` so the LD_PRELOAD interceptor -propagates through the whole child tree. `rustc` is `allow_remote` with -`build_script_build` excluded — build.rs scripts inspect the local host and -must run locally. +`exclude_args="-c:-S:-E"` and `cached="true"`. The exclusion means the +rule doesn't fire when a compile flag is present, so `clang -c foo.cpp` +falls through to the cache path while `clang foo.o bar.o -o out` (link, +no `-c`) stays pinned and uncached. `ninja`, `cmake`, `cargo`, and `bun` +are `intercepted recursive` so the LD_PRELOAD interceptor propagates +through the whole child tree. `rustc` is `allow_remote` with +`build_script_build` excluded (build.rs scripts inspect the host and +must run locally) — in `--standalone` mode `allow_remote` is moot but +still enables caching for matched processes. ## Local use