diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000..20ebee076c1 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +# actionlint config — registers labels that exist on our self-hosted runners +# but not on GH-hosted ones, so the linter stops flagging them as typos. +self-hosted-runner: + labels: + - incredibuild-runner diff --git a/.github/actions/ib-metrics/action.yml b/.github/actions/ib-metrics/action.yml new file mode 100644 index 00000000000..d81ab2574bb --- /dev/null +++ b/.github/actions/ib-metrics/action.yml @@ -0,0 +1,79 @@ +name: ib-metrics +description: | + Parse an ib_console monitor log and write a markdown summary to + $GITHUB_STEP_SUMMARY: task counts, local vs helper distribution, + parallelism factor, cache hit rate. Treats missing fields gracefully. + +inputs: + log-path: + description: "Path to the ib_console log produced by ib-build" + required: true + label: + description: "Heading for the summary block" + required: false + default: "Incredibuild metrics" + +runs: + using: composite + steps: + - name: Write summary + shell: bash + env: + LOG_PATH: ${{ inputs.log-path }} + LABEL: ${{ inputs.label }} + run: | + set -uo pipefail + if [[ ! -s "$LOG_PATH" ]]; then + { + echo "## $LABEL" + echo + echo "_no log captured at \`$LOG_PATH\`_" + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # Heuristic extraction. ib_console with monitor_enable="true" emits + # lines like: + # [task] helper N: clang -c ... + # [task] local : ld ... + # plus a final block: + # Total tasks: X + # Tasks executed locally: Y + # Tasks executed remotely: Z + # Parallel acceleration: K.K x + total_tasks=$(grep -Ei "Total tasks" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+' | head -1 || true) + local_tasks=$(grep -Ei "(executed locally|local tasks)" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+' | head -1 || true) + remote_tasks=$(grep -Ei "(executed remotely|remote tasks)" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+' | head -1 || true) + accel=$(grep -Ei "(acceleration|speedup)" "$LOG_PATH" | tail -1 | grep -oE '[0-9]+(\.[0-9]+)?' | head -1 || true) + wall=$(grep -Ei "(wall.?time|build duration|total time)" "$LOG_PATH" | tail -1 || true) + + # Fallback if the structured block isn't found: count placement lines. + if [[ -z "$total_tasks" ]]; then + local_tasks=$(grep -cE '\[(task|exec)\][^]]*local' "$LOG_PATH" || echo 0) + remote_tasks=$(grep -cE '\[(task|exec)\][^]]*(helper|remote)' "$LOG_PATH" || echo 0) + total_tasks=$(( local_tasks + remote_tasks )) + fi + + cache_hits=$(grep -ciE "cache.?hit" "$LOG_PATH" || echo 0) + cache_miss=$(grep -ciE "cache.?miss" "$LOG_PATH" || echo 0) + + { + echo "## $LABEL" + echo + echo "| metric | value |" + echo "| --- | --- |" + echo "| total tasks | ${total_tasks:-?} |" + echo "| local tasks | ${local_tasks:-?} |" + echo "| remote tasks | ${remote_tasks:-?} |" + if [[ -n "$accel" ]]; then echo "| parallel acceleration | ${accel}x |"; fi + if [[ -n "$wall" ]]; then echo "| wall time | \`$wall\` |"; fi + echo "| cache hits | $cache_hits |" + echo "| cache misses | $cache_miss |" + echo + echo "
log tail (last 50 lines)" + echo + echo "\`\`\`" + tail -n 50 "$LOG_PATH" + echo "\`\`\`" + echo "
" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/incredibuild-compare.yml b/.github/workflows/incredibuild-compare.yml new file mode 100644 index 00000000000..fd7ada5c045 --- /dev/null +++ b/.github/workflows/incredibuild-compare.yml @@ -0,0 +1,114 @@ +name: incredibuild-compare + +# Side-by-side acceleration benchmark on a vnext IB runner: builds the +# same Bun profile twice on the same runner, once with the IB ninja shim +# active and once with IB_CONSOLE_SKIP=1 (the shim recognizes this and +# falls back to plain ninja). Reports the wall-time delta. + +on: + workflow_dispatch: + inputs: + bun_profile: + description: "Bun build profile to benchmark" + required: true + default: "ci-cpp-only" + type: choice + options: + - debug + - release + - ci-cpp-only + - ci-rust-only + - ci-release + iterations: + description: "Iterations per variant (median reported)" + required: false + default: "1" + type: string + clean_between: + description: "Wipe build dir between iterations (cold) vs reuse (warm)" + required: false + default: true + type: boolean + +concurrency: + group: incredibuild-compare-${{ github.ref }} + cancel-in-progress: false + +jobs: + benchmark: + runs-on: incredibuild-runner + timeout-minutes: 240 + env: + BUN_PROFILE: ${{ inputs.bun_profile }} + ITERATIONS: ${{ inputs.iterations }} + CLEAN: ${{ inputs.clean_between }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Activate Bun-specific IB profile + run: cp .incredibuild/ib_profile.xml ib_profile.xml + + - name: Warm dependencies once (excluded from timing) + run: | + set -euo pipefail + bun scripts/build.ts --profile="$BUN_PROFILE" --configure-only + + - name: Stock builds (IB_CONSOLE_SKIP=1 disables the ninja shim) + id: stock + env: + IB_CONSOLE_SKIP: "1" + run: | + set -euo pipefail + times=/tmp/stock-times.txt + : > "$times" + for i in $(seq 1 "$ITERATIONS"); do + [[ "$CLEAN" == "true" ]] && rm -rf "build/$BUN_PROFILE" + t0=$(date +%s.%N) + bun scripts/build.ts --profile="$BUN_PROFILE" + t1=$(date +%s.%N) + dt=$(awk -v a="$t0" -v b="$t1" 'BEGIN{print b-a}') + echo "$dt" >> "$times" + echo "stock iter $i: ${dt}s" >&2 + done + median=$(sort -n "$times" | awk 'NR==int(NR/2)+1') + echo "median=$median" >> "$GITHUB_OUTPUT" + + - name: IB-accelerated builds (shim active) + id: ib + run: | + set -euo pipefail + times=/tmp/ib-times.txt + : > "$times" + for i in $(seq 1 "$ITERATIONS"); do + [[ "$CLEAN" == "true" ]] && rm -rf "build/$BUN_PROFILE" + t0=$(date +%s.%N) + bun scripts/build.ts --profile="$BUN_PROFILE" + t1=$(date +%s.%N) + dt=$(awk -v a="$t0" -v b="$t1" 'BEGIN{print b-a}') + echo "$dt" >> "$times" + echo "ib iter $i: ${dt}s" >&2 + done + median=$(sort -n "$times" | awk 'NR==int(NR/2)+1') + echo "median=$median" >> "$GITHUB_OUTPUT" + + - name: Report + run: | + stock=${{ steps.stock.outputs.median }} + ib=${{ steps.ib.outputs.median }} + speedup=$(awk -v s="$stock" -v i="$ib" 'BEGIN{ if (i>0) printf "%.2f", s/i; else print "n/a"}') + delta=$(awk -v s="$stock" -v i="$ib" 'BEGIN{printf "%.2f", s-i}') + { + echo "## Incredibuild speedup — $BUN_PROFILE" + echo + echo "| variant | median wall (s) |" + echo "| --- | --- |" + echo "| stock (IB_CONSOLE_SKIP=1) | $stock |" + echo "| ib-accelerated | $ib |" + echo + echo "**delta:** ${delta}s saved · **speedup:** ${speedup}×" + echo + echo "_(${ITERATIONS} iteration(s), clean-between=${CLEAN})_" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/incredibuild-linux.yml b/.github/workflows/incredibuild-linux.yml new file mode 100644 index 00000000000..64c704cef06 --- /dev/null +++ b/.github/workflows/incredibuild-linux.yml @@ -0,0 +1,133 @@ +name: incredibuild-linux + +# Build Bun on a vnext-processing-engine Incredibuild runner. +# +# The runner image (nscr.io/.../incredibuild-runner:latest, from +# Incredibuild-RND/vnext-processing-engine) already PATH-shims `ninja`, +# `cmake`, `cargo`, etc. to invoke `ib_console --standalone +# --build-cache-local-shared --build-cache-force --build-cache-basedir=$PWD` +# transparently. So we just need to: +# 1. Drop our Bun-specific profile at $PWD/ib_profile.xml (highest +# priority in ib_console's default profile load order). +# 2. Run `bun scripts/build.ts` normally — the shim does the rest. +# +# Two jobs: +# 1. validate-profile-xsd (ubuntu-latest) — runs on every push/PR. +# 2. ib-build (incredibuild-runner) — workflow_dispatch only. +on: + workflow_dispatch: + inputs: + bun_profile: + description: "Bun build profile" + required: true + default: "ci-cpp-only" + type: choice + options: + - debug + - release + - ci-cpp-only + - ci-rust-only + - ci-link-only + - ci-release + debug_shim: + description: "Set IB_ACCEL_DEBUG=1 so the shim logs every transformation" + required: false + default: false + type: boolean + push: + branches: + - "incredibuild/**" + pull_request: + paths: + - ".incredibuild/**" + - ".github/actions/ib-metrics/**" + - ".github/workflows/incredibuild-linux.yml" + - "scripts/build.ts" + +concurrency: + group: incredibuild-linux-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate-profile-xsd: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install xmllint + shellcheck + run: sudo apt-get update && sudo apt-get install -y libxml2-utils shellcheck + - name: Validate .incredibuild/ib_profile.xml + run: ./.incredibuild/validate.sh + - name: Shellcheck wrapper scripts + run: shellcheck .incredibuild/build.sh .incredibuild/validate.sh + + ib-build: + needs: validate-profile-xsd + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: incredibuild-runner + timeout-minutes: 120 + + env: + BUN_PROFILE: ${{ inputs.bun_profile || 'ci-cpp-only' }} + IB_ACCEL_DEBUG: ${{ inputs.debug_shim == true && '1' || '0' }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Activate Bun-specific IB profile + run: | + set -euo pipefail + # vnext runner's ninja shim wraps with `ib_console --standalone …` + # without --profile, so ib_console resolves via the default load + # order. $PWD/ib_profile.xml is the highest-priority slot. + cp .incredibuild/ib_profile.xml ib_profile.xml + xmllint --noout --schema .incredibuild/ib_profile.xsd ib_profile.xml + echo "[ib] profile active at $PWD/ib_profile.xml" + + - name: Show effective IB env + run: | + echo "ib_console: $(command -v ib_console || echo 'not on PATH')" + ib_console --version 2>/dev/null || true + echo "ninja shim: $(command -v ninja)" + echo "IB_ACCEL_DIR=${IB_ACCEL_DIR:-}" + echo "IB_ACCEL_DEBUG=${IB_ACCEL_DEBUG:-}" + + - name: Build Bun + id: build + run: | + set -o pipefail + log="${RUNNER_TEMP:-/tmp}/ib-build-${BUN_PROFILE}.log" + echo "log-path=$log" >> "$GITHUB_OUTPUT" + # The PATH shim does the ib_console wrapping for ninja transparently. + # We just need to invoke the normal Bun build entry point. + bun scripts/build.ts --profile="$BUN_PROFILE" 2>&1 | tee "$log" + + - name: Acceleration metrics + if: always() + uses: ./.github/actions/ib-metrics + with: + log-path: ${{ steps.build.outputs.log-path }} + label: "Incredibuild — ${{ env.BUN_PROFILE }}" + + - name: Upload IB build log + if: always() + uses: actions/upload-artifact@v4 + with: + name: ib-log-${{ env.BUN_PROFILE }} + path: ${{ steps.build.outputs.log-path }} + if-no-files-found: ignore + retention-days: 14 + + - name: Upload build dir on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: build-failure-${{ env.BUN_PROFILE }} + path: | + build/**/CMakeFiles/CMakeOutput.log + build/**/CMakeFiles/CMakeError.log + build/**/build.ninja + build/**/*.log + if-no-files-found: ignore + retention-days: 7 diff --git a/.incredibuild/ACCELERATION.md b/.incredibuild/ACCELERATION.md new file mode 100644 index 00000000000..dc3dd247a41 --- /dev/null +++ b/.incredibuild/ACCELERATION.md @@ -0,0 +1,166 @@ +# Acceleration model — what, how, by how much + +What this integration accelerates on a Bun build, the mechanism, and the +expected wall-time delta. + +> **Mode: cache-only (`--standalone`).** vnext-processing-engine's ninja +> shim invokes `ib_console --standalone --build-cache-local-shared +> --build-cache-force --build-cache-basedir=$PWD`. There is **no** remote +> distribution to helpers — `--standalone` explicitly disables that. The +> entire acceleration comes from the content-addressed compile cache. + +## What we accelerate + +We accelerate **incremental rebuilds via cache hits**, not cold builds. +A from-scratch first build sees little to no speedup (every compile +misses, hashing adds 1–3% overhead). The win shows up the second time +the same content compiles — PR re-pushes, CI re-runs of the same SHA, +branch switches, base-branch rebases. + +The cached units are every process matched by the profile (see +`ib_profile.xml`): + +| Process | Profile rule | Cached? | +| --- | --- | --- | +| `clang -c` / `clang++ -c` / `gcc -c` / `g++ -c` | `local_only` + `exclude_args="-c:-S:-E"` + `cached="true"` | **yes** (compile only) | +| `clang … -o exe` (link, no `-c`) | matches `local_only` (no exclusion) | no — stays local, no cache | +| `rustc ` | `allow_remote` minus `build_script_build` | yes (ccache backend) | +| `cargo` / `ninja` / `cmake` / `bun` | `intercepted recursive` | not cached themselves (they orchestrate) | +| `bison` / `flex` / `m4` / `nasm` / `yasm` / `as` | `allow_remote` | yes | +| `ld` / `ld.lld` / `ar` / `ranlib` / `strip` / `objcopy` | `intercepted` | no (state consolidation) | + +## What gets counted + +Concrete TU counts in this fork: + +| Component | TUs | Cached on hit | +| --- | ---: | :---: | +| Bun core `src/**/*.cpp` | 554 | ✓ | +| Bun core `src/**/*.c` | 10 | ✓ | +| boringssl | 143 | ✓ | +| lsquic (HTTP/3 QUIC) | 69 | ✓ | +| libdeflate | 11 | ✓ | +| highway (SIMD) | 9 | ✓ | +| tinycc | 6 | ✓ | +| libuv (Windows-only), libarchive, c-ares, brotli, sqlite, mimalloc, zstd, libwebp, libjpeg-turbo, libspng, lshpack, hdrhistogram, picohttpparser | ~200 (array-based, platform-varying) | ✓ | +| Rust crates (workspace) | 105 | ✓ (codegen units) | +| Final link (one per artifact) | — | ✗ | + +**Total cacheable C/C++ TUs: ~1,300–1,500.** + +## How it accelerates + +ib_console computes a hash of each compile invocation: + +``` +hash = H(compiler-binary-content, + exact-flag-string, + source-file-content, + transitive-header-content) +``` + +On a hit, the cached `.o` is copied out instead of running the +compiler. On a miss, the compiler runs normally and the result is +written into the cache. `--build-cache-force` skips the default +"is-it-safe-to-cache" heuristics (it's safe — the hash already covers +every input). + +**Cache lifetime on vnext runners**: `--build-cache-basedir=$PWD` puts +the cache in the job's working directory. The vnext orchestrator +bind-mounts `/ib-workspace/cache` from the host via virtiofs and +promotes cache contents across runs (see +`vnext-processing-engine/src/runner_engine/build/entrypoint.sh` +line ~108 — `cache/` is the only directory promoted post-job). + +## How it integrates here + +Two paths, same `.incredibuild/ib_profile.xml`: + +- **vnext IB runner** (`runs-on: incredibuild-runner`) — the ib-accel + PATH shim invokes `ib_console --standalone --build-cache-local-shared + --build-cache-force …` automatically. Profile resolved from + `$PWD/ib_profile.xml` (we copy it there in the job's first step). +- **Off-grid** (`BUN_USE_INCREDIBUILD=1`) — `scripts/build.ts` wraps + the inner `ninja` spawn with explicit `ib_console --profile … --`. + Same per-process matching. + +## By how much + +### Measured baseline + +`clang++ -O2 -std=c++17 -c` on a representative Bun-class +template-heavy TU, x86_64 Ubuntu 24.04 + clang 18.1.3, 10 vCPU: + +| metric | value | +| --- | ---: | +| T per TU (median of 3) | **0.44 s** | +| Serial wall, 32 TUs | 13.9 s | +| Parallel `-j10` wall, 32 TUs | 2.60 s (5.36× from native multi-core) | + +Real Bun TUs with the WTF + JSC PCH on `-O3` are heavier — call it +2 s/TU floor. With ~1,300 cacheable TUs, a cold serial cpp-only is +~9.6–43 minutes; with ninja `-j$(nproc)` on a 16-core CI box it's +**~3–10 min**. That's the stock baseline IB has to beat. + +### Cache-hit speedup curve + +The cache lookup is dominated by hashing the source + transitive +headers. Typical per-file lookup is 5–20 ms. + +| hit rate | misses (compile) | hits (lookup) | wall (estimate) | speedup vs stock-parallel | +| ---: | ---: | ---: | ---: | ---: | +| 0% (cold) | 1,300 × 2.0 s | 0 | **~10 min** (= stock + ~1% overhead) | ~1.0× | +| 50% (rebase) | 650 × 2.0 s | 650 × 0.01 s | **~5.5 min** | ~1.8× | +| 90% (small refactor) | 130 × 2.0 s | 1,170 × 0.01 s | **~80 s** | ~7× | +| 99% (PR re-push touching 1 file) | 13 × 2.0 s | 1,287 × 0.01 s | **~40 s** | ~15× | +| 100% (CI re-run same SHA) | 0 | 1,300 × 0.01 s | **~13 s + link** ≈ **20 s** | ~30× | + +(Assuming the 16-core parallel stock baseline ≈ 10 min. With LTO link +included the percentages shift — link is uncached and dominates as +hit rate approaches 100%, capping the ceiling at the link time.) + +### Per-profile expectations + +| Bun profile | Stock parallel | Best-case cached re-run | Notes | +| --- | ---: | ---: | --- | +| `debug` | 3–6 min | **20–40 s** | no LTO; link is fast; cache is everything | +| `ci-cpp-only` | 8–15 min | **15–30 s** | static-lib output; link is just `ar` | +| `ci-rust-only` | 15–30 min | **30–90 s** | rustc codegen units cached; build.rs not | +| `ci-release` (LTO on) | 30–60 min | **6–16 min** | LTO link uncached, sets the ceiling | + +LTO link time is the Amdahl wall for release builds. The cache pulls +compile to near zero; nothing accelerates the link. + +### What still needs measurement + +These numbers are derived from one measured `T_per_TU` (0.44 s on a +synthetic Bun-class TU) plus the documented `--build-cache` behavior. +Real Bun TUs are heavier (PCH + WebKit), and the cache hit rate in +practice depends on how many transitive headers change per commit — +a touch to `root-pch.h` invalidates the world. + +To get the real number, on a vnext runner with the cache pre-warmed: + +```bash +gh workflow run incredibuild-compare.yml \ + -F bun_profile=ci-cpp-only \ + -F iterations=3 \ + -F clean_between=false # keep build dir for warm-cache measurement +``` + +The "stock" arm sets `IB_CONSOLE_SKIP=1` so the shim falls back to +plain ninja — that gives the apples-to-apples delta. + +### What doesn't help + +- **Cold CI builds on a fresh branch with no warm cache** — same wall + as stock, ±1%. The acceleration story for this case requires either + pre-populating the cache from a recent main-branch build, or + switching to a remote-distribution config (not what vnext does + today). +- **Touching `root-pch.h` or a heavily-included header** — invalidates + the dependent TUs' cache entries by content hash. Worst case behaves + like a cold build. +- **Compiler / flag upgrades** — bump clang version or change `-O` + level and the cache misses on everything (correct behavior; the + hash includes compiler binary + flags). diff --git a/.incredibuild/README.md b/.incredibuild/README.md new file mode 100644 index 00000000000..b43ea539658 --- /dev/null +++ b/.incredibuild/README.md @@ -0,0 +1,131 @@ +# Incredibuild integration for Bun + +Wraps the Ninja-driven Bun build with [`ib_console`][ib_linux] so the +~1,500 C/C++ TUs (Bun core + boringssl + lsquic + libdeflate + …) and the +105-crate Rust workspace go through a **content-addressed compile cache**. +Re-runs on the same SHA, PR re-pushes, and small refactors get the big +wins; cold builds get ~1×. + +The deployment mode on vnext-processing-engine runners is `--standalone` +(cache only, no remote helpers) — see +[`ACCELERATION.md`](./ACCELERATION.md) for the full speedup curve. + +[ib_linux]: https://github.com/Incredibuild-RND/ib_linux + +## Layout + +``` +.incredibuild/ +├── ib_profile.xml IB profile: which procs intercept, distribute, stay local +├── ib_profile.xsd Vendored copy of ib_linux/data/ib_profile.xsd for offline validation +├── build.sh Wrapper: `ib_console -- bun scripts/build.ts …` +├── validate.sh Validate the profile against the XSD via xmllint +└── README.md This file +``` + +The build.ts side has an opt-in patch: when `BUN_USE_INCREDIBUILD=1`, the +inner `ninja` spawn is wrapped with `ib_console`. When `build.sh` is used, +it sets `BUN_INCREDIBUILD_WRAPPED=1` so we don't nest ib_console twice. + +## Profile in one paragraph + +`clang` / `clang++` / `gcc` / `g++` are listed as `local_only` with +`exclude_args="-c:-S:-E"` and `cached="true"`. The exclusion means the +rule doesn't fire when a compile flag is present, so `clang -c foo.cpp` +falls through to the cache path while `clang foo.o bar.o -o out` (link, +no `-c`) stays pinned and uncached. `ninja`, `cmake`, `cargo`, and `bun` +are `intercepted recursive` so the LD_PRELOAD interceptor propagates +through the whole child tree. `rustc` is `allow_remote` with +`build_script_build` excluded (build.rs scripts inspect the host and +must run locally) — in `--standalone` mode `allow_remote` is moot but +still enables caching for matched processes. + +## Local use + +```bash +# Debug build +.incredibuild/build.sh + +# CI C++ phase (matches the Buildkite cpp-only step) +.incredibuild/build.sh --profile=ci-cpp-only + +# Full release with aggressive remote distribution +IB_FORCE_REMOTE=1 IB_MAX_LOCAL=4 .incredibuild/build.sh --profile=release +``` + +Knobs (all env-var): + +| Var | Default | Purpose | +| --- | --- | --- | +| `IB_CONSOLE` | `ib_console` | path to the binary | +| `IB_PROFILE` | `.incredibuild/ib_profile.xml` | profile XML | +| `IB_FORCE_REMOTE` | unset | `1` adds `--force-remote` | +| `IB_MAX_LOCAL` | unset | int → `--max-local-cores N` | +| `IB_CAPTION` | `bun-` | name in the IB monitor | +| `IB_EXTRA_ARGS` | unset | appended verbatim to ib_console | + +## Use from `bun scripts/build.ts` directly + +```bash +BUN_USE_INCREDIBUILD=1 IB_FORCE_REMOTE=1 \ + bun scripts/build.ts --profile=ci-cpp-only +``` + +This is for **off-grid** use (workstation with ib_linux installed, +traditional self-hosted setups). On a vnext runner you don't need it — +see below. + +## On a vnext-processing-engine IB runner + +`Incredibuild-RND/vnext-processing-engine`'s runner image ships an +[ib-accel](https://github.com/Incredibuild-RND/vnext-processing-engine/tree/main/src/runner_engine/build/ib-accel) +PATH shim that intercepts `ninja`/`cmake`/`cargo`/etc. and wraps them +with `ib_console --standalone --build-cache-local-shared +--build-cache-force --build-cache-basedir=$PWD` transparently. So on +`runs-on: incredibuild-runner`, the job only needs to: + +1. Drop our Bun-specific profile at `$PWD/ib_profile.xml` (highest + priority in `ib_console`'s default load order). +2. Run `bun scripts/build.ts --profile=…` normally. + +That's exactly what `.github/workflows/incredibuild-linux.yml` does. +**Do not** also set `BUN_USE_INCREDIBUILD=1` on a vnext runner — the +shim already wraps; double-wrapping breaks. Set `IB_ACCEL_DEBUG=1` if +you want to see every shim transformation. + +## Prerequisites on the build machine + +For a **vnext IB runner** (the recommended path): nothing — the image +ships ib_linux, ib_console, the PATH shim, the Bun toolchain, and the +coordinator/helper grid wiring out of the box. + +For **off-grid** use: +1. ib_linux installed → `ib_console` on `PATH` (usually at + `/opt/incredibuild/bin/ib_console`). +2. Machine registered with an Incredibuild coordinator (initiator role). +3. A pool of helpers on the same network — these execute the distributed + `clang -c`, `rustc`, parser, and assembler jobs. +4. Bun toolchain: `clang`, `clang++`, `lld`, `ninja`, `cmake`, `cargo`, + `rustc`, `nasm`, `python3`, plus `bun` itself. + +## How to verify it's actually distributing + +```bash +.incredibuild/build.sh --profile=ci-cpp-only 2>&1 | tee /tmp/ib.log +grep -E "remote|helper|task" /tmp/ib.log | head +``` + +ib_console prints per-task placement (local vs helper N) when +`monitor_enable="true"` in the profile (which is the default here). + +## What's distributable vs not + +| Phase | TUs (≈) | Distributable | Notes | +| --- | --- | --- | --- | +| BoringSSL | 600 | yes | mostly `clang -c` | +| WebKit (local build) | thousands | yes | giant win when not using prebuilt | +| libuv / c-ares / libarchive / zstd / zlib / libdeflate | ~250 total | yes | plain C `clang -c` | +| Bun C++ core (`src/**/*.cpp`) | 554 | yes | uses PCH; PCH itself stays local | +| Rust crates | 107 | partial | rustc is `allow_remote`; build.rs is local | +| Final link | 1 | no | always local; LTO needs all object files on box | +| Codegen scripts (JS) | many | no (intercepted, runs local) | `bun` runs them | diff --git a/.incredibuild/build.sh b/.incredibuild/build.sh new file mode 100755 index 00000000000..f5d10138027 --- /dev/null +++ b/.incredibuild/build.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Wrapper: run a Bun build through Incredibuild (ib_console). +# +# Usage: +# .incredibuild/build.sh # debug profile +# .incredibuild/build.sh --profile=ci-cpp-only # CI C++ phase +# .incredibuild/build.sh --profile=release --build-dir=build/release +# +# Env knobs: +# IB_CONSOLE path to ib_console (default: ib_console on PATH) +# IB_PROFILE path to profile XML (default: .incredibuild/ib_profile.xml) +# IB_FORCE_REMOTE "1" to pass --force-remote (push allow_remote off-box) +# IB_MAX_LOCAL integer; pass --max-local-cores N +# IB_CAPTION build name for the IB monitor (default: bun-) +# IB_EXTRA_ARGS extra args appended verbatim to ib_console + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +IB_CONSOLE="${IB_CONSOLE:-ib_console}" +IB_PROFILE="${IB_PROFILE:-$repo_root/.incredibuild/ib_profile.xml}" + +if ! command -v "$IB_CONSOLE" >/dev/null 2>&1; then + echo "error: '$IB_CONSOLE' not on PATH." >&2 + echo " install ib_linux from https://github.com/Incredibuild-RND/ib_linux" >&2 + echo " or set IB_CONSOLE=/opt/incredibuild/bin/ib_console" >&2 + exit 127 +fi + +if [[ ! -f "$IB_PROFILE" ]]; then + echo "error: IB profile not found: $IB_PROFILE" >&2 + exit 1 +fi + +# Default to debug if no --profile passed. (Bun build profile, not IB profile.) +bun_profile="debug" +for arg in "$@"; do + case "$arg" in + --profile=*) bun_profile="${arg#--profile=}" ;; + esac +done + +ib_args=( + --profile "$IB_PROFILE" + --caption "${IB_CAPTION:-bun-$bun_profile}" +) +[[ "${IB_FORCE_REMOTE:-0}" == "1" ]] && ib_args+=(--force-remote) +[[ -n "${IB_MAX_LOCAL:-}" ]] && ib_args+=(--max-local-cores "$IB_MAX_LOCAL") +[[ -n "${IB_EXTRA_ARGS:-}" ]] && read -r -a extra <<<"$IB_EXTRA_ARGS" && ib_args+=("${extra[@]}") + +# Tell scripts/build.ts not to re-wrap ninja under another ib_console — +# the outer ib_console here already intercepts the whole subtree. +export BUN_INCREDIBUILD_WRAPPED=1 + +exec "$IB_CONSOLE" "${ib_args[@]}" -- bun scripts/build.ts "$@" diff --git a/.incredibuild/ib_profile.xml b/.incredibuild/ib_profile.xml new file mode 100644 index 00000000000..bed0eeb5992 --- /dev/null +++ b/.incredibuild/ib_profile.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.incredibuild/ib_profile.xsd b/.incredibuild/ib_profile.xsd new file mode 100644 index 00000000000..47c00e8b3d3 --- /dev/null +++ b/.incredibuild/ib_profile.xsd @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.incredibuild/validate.sh b/.incredibuild/validate.sh new file mode 100755 index 00000000000..c52eb6be624 --- /dev/null +++ b/.incredibuild/validate.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Validate .incredibuild/ib_profile.xml against ib_linux's official XSD. +# This is the same XSD the real ib_console's ProfileConfig parser uses. +# +# ./.incredibuild/validate.sh # auto-locates the XSD +# IB_XSD=/path/to/ib_profile.xsd ./.incredibuild/validate.sh +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +profile="$repo_root/.incredibuild/ib_profile.xml" + +# Locate the XSD. Order: $IB_XSD, vendored copy, installed ib_linux, +# sibling ib_linux checkout. Vendored is the default so CI works without +# cross-repo access to the internal Incredibuild-RND/ib_linux repo. +xsd="" +for candidate in \ + "${IB_XSD:-}" \ + "$repo_root/.incredibuild/ib_profile.xsd" \ + /opt/incredibuild/data/ib_profile.xsd \ + "$repo_root/../ib_linux/data/ib_profile.xsd" \ + /tmp/ib_linux/data/ib_profile.xsd +do + [[ -n "$candidate" && -f "$candidate" ]] && { xsd="$candidate"; break; } +done + +if [[ -z "$xsd" ]]; then + echo "error: ib_profile.xsd not found." >&2 + echo " Set IB_XSD=/path/to/ib_profile.xsd, install ib_linux to /opt/incredibuild," >&2 + echo " or clone https://github.com/Incredibuild-RND/ib_linux next to this repo." >&2 + exit 1 +fi + +if ! command -v xmllint >/dev/null 2>&1; then + echo "error: xmllint not on PATH (apt install libxml2-utils / brew install libxml2)" >&2 + exit 1 +fi + +echo "profile: $profile" +echo "xsd: $xsd" +xmllint --noout --schema "$xsd" "$profile" diff --git a/scripts/build.ts b/scripts/build.ts index e92aa652db3..e5f3d0364f4 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -97,6 +97,27 @@ async function main(): Promise { const ninjaArgv = (cfg: { buildDir: string }) => ["-C", cfg.buildDir, ...args.ninjaArgs, ...args.ninjaTargets]; const ninjaEnv = (env: Record) => ({ ...process.env, ...env }); + // Incredibuild integration. Opt-in via BUN_USE_INCREDIBUILD=1. Skipped when + // BUN_INCREDIBUILD_WRAPPED=1 (set by .incredibuild/build.sh, which already + // wraps the whole bun-build subtree under ib_console). + const useIB = process.env.BUN_USE_INCREDIBUILD === "1" && process.env.BUN_INCREDIBUILD_WRAPPED !== "1"; + const ibConsole = process.env.IB_CONSOLE || "ib_console"; + const ibProfile = process.env.IB_PROFILE || join(process.cwd(), ".incredibuild", "ib_profile.xml"); + const ibArgs = useIB + ? [ + "--profile", + ibProfile, + "--caption", + `bun-${args.profile ?? "build"}`, + ...(process.env.IB_FORCE_REMOTE === "1" ? ["--force-remote"] : []), + ...(process.env.IB_MAX_LOCAL ? ["--max-local-cores", process.env.IB_MAX_LOCAL] : []), + "--", + ] + : []; + const ninjaCmd = useIB ? ibConsole : "ninja"; + const ninjaFullArgv = (cfg: { buildDir: string }) => + useIB ? [...ibArgs, "ninja", ...ninjaArgv(cfg)] : ninjaArgv(cfg); + if (isCI) { // CI: machine/env dump + collapsible groups + annotation-on-failure. printEnvironment(); @@ -109,7 +130,7 @@ async function main(): Promise { } await startGroup("Build", () => - spawnWithAnnotations("ninja", ninjaArgv(result.cfg), { label: "ninja", env: ninjaEnv(result.env) }), + spawnWithAnnotations(ninjaCmd, ninjaFullArgv(result.cfg), { label: "ninja", env: ninjaEnv(result.env) }), ); // cpp-only/rust-only: upload build outputs for downstream link-only. @@ -180,7 +201,7 @@ async function main(): Promise { if (!quiet && interactive) { stdio[STREAM_FD] = 2; } - const ninja = spawnSync("ninja", ninjaArgv(result.cfg), { + const ninja = spawnSync(ninjaCmd, ninjaFullArgv(result.cfg), { stdio, env: ninjaEnv(result.env), // cargo's compile output (now part of the ninja graph via emitRust) can