diff --git a/bench/BENCHMARKING.md b/bench/BENCHMARKING.md new file mode 100644 index 000000000000..da154a355c35 --- /dev/null +++ b/bench/BENCHMARKING.md @@ -0,0 +1,192 @@ +# Benchmarking Playbook (Render Pipeline / Node Streams) + +This is the practical workflow for benchmarking and profiling render pipeline changes in this repo. + +Primary tools: + +- `pnpm bench:render-pipeline` +- `pnpm bench:render-pipeline:analyze` + +## 1. Build-first baseline + +Always rebuild `next` before benchmark runs when framework source changed. + +```bash +pnpm --filter=next build +``` + +## 2. End-to-end benchmark (full app render path) + +This measures the full request path (`renderToHTMLOrFlight`) through `bench/next-minimal-server`. +In `scenario=full` and `scenario=all`, `--capture-cpu` defaults to `true`. + +Node streams only: + +```bash +pnpm bench:render-pipeline \ + --scenario=full \ + --stream-mode=node \ + --build-full=true \ + --json-out=bench/render-pipeline/artifacts//results.json \ + --artifact-dir=bench/render-pipeline/artifacts/ +``` + +Web vs Node comparison: + +```bash +pnpm bench:render-pipeline \ + --scenario=full \ + --stream-mode=both \ + --build-full=true \ + --json-out=bench/render-pipeline/artifacts//results.json \ + --artifact-dir=bench/render-pipeline/artifacts/ +``` + +## 3. Route-focused stress runs + +Use this when targeting streaming-heavy behavior only. + +```bash +pnpm bench:render-pipeline \ + --scenario=full \ + --stream-mode=node \ + --build-full=true \ + --routes=/streaming/heavy,/streaming/chunkstorm,/streaming/wide \ + --warmup-requests=10 \ + --serial-requests=40 \ + --load-requests=400 \ + --load-concurrency=40 \ + --json-out=bench/render-pipeline/artifacts//results.json \ + --artifact-dir=bench/render-pipeline/artifacts/ +``` + +Default stress routes currently include: + +- `/` +- `/streaming/light` +- `/streaming/medium` +- `/streaming/heavy` +- `/streaming/chunkstorm` +- `/streaming/wide` +- `/streaming/bulk` + +## 4. Isolate helper-level costs (micro scenario) + +Use this to quickly test helper-level changes before full runs. + +```bash +pnpm bench:render-pipeline \ + --scenario=micro \ + --iterations=300 \ + --warmup=30 +``` + +Micro benchmark output includes cases for: + +- `teeNodeReadable` +- `createBufferedTransformNode` +- `createInlinedDataNodeStream` +- `continueStaticPrerender` / `continueDynamicPrerender` / `continueDynamicHTMLResume` + +Flight payload mode toggles: + +```bash +# Binary-heavy flight chunks +pnpm bench:render-pipeline --scenario=micro --binary-flight=true + +# UTF-8-heavy flight chunks +pnpm bench:render-pipeline --scenario=micro --binary-flight=false +``` + +Stress payload shape: + +```bash +pnpm bench:render-pipeline \ + --scenario=micro \ + --iterations=300 \ + --warmup=30 \ + --flight-chunks=128 \ + --flight-chunk-bytes=8192 \ + --html-chunks=128 \ + --html-chunk-bytes=32768 +``` + +## 5. Capture CPU profiles and traces + +```bash +pnpm bench:render-pipeline \ + --scenario=full \ + --stream-mode=node \ + --build-full=true \ + --capture-trace=true \ + --capture-next-trace=true \ + --json-out=bench/render-pipeline/artifacts//results.json \ + --artifact-dir=bench/render-pipeline/artifacts/ +``` + +Artifacts are written under: + +- `bench/render-pipeline/artifacts//node/node.cpuprofile` +- `bench/render-pipeline/artifacts//node/node-trace-*.json` +- `bench/render-pipeline/artifacts//node/next-runtime-trace.log` +- `bench/render-pipeline/artifacts//results.json` + +## 6. Analyze hotspots + +```bash +pnpm bench:render-pipeline:analyze \ + --artifact-dir=bench/render-pipeline/artifacts/ \ + --top=20 +``` + +Filter only the Node-stream-relevant hotspots: + +```bash +pnpm bench:render-pipeline:analyze --artifact-dir=bench/render-pipeline/artifacts/ --top=20 > /tmp/analyze.txt +rg "use-flight-response|encodeFlightDataChunkNode|node-stream-tee|flushPending|node-stream-helpers|htmlEscapeJsonString" /tmp/analyze.txt +``` + +## 7. Compare two runs quickly + +```bash +node - <<'NODE' +const fs = require('fs') +const [baseRun, candRun] = process.argv.slice(2) +const load = (name) => + JSON.parse( + fs.readFileSync(`bench/render-pipeline/artifacts/${name}/results.json`, 'utf8') + ).fullResults[0].routeResults + +const base = load(baseRun) +const cand = load(candRun) +for (const b of base) { + const c = cand.find((x) => x.route === b.route && x.phase === b.phase) + if (!c) continue + const throughputDelta = + ((c.throughputRps - b.throughputRps) / b.throughputRps) * 100 + const p95Delta = ((b.latency.p95 - c.latency.p95) / b.latency.p95) * 100 + console.log( + `${b.route} ${b.phase} throughput ${throughputDelta >= 0 ? '+' : ''}${throughputDelta.toFixed(2)}% p95 ${p95Delta >= 0 ? '+' : ''}${p95Delta.toFixed(2)}%` + ) +} +NODE investigation-10-boundary-data investigation-17-profile-current +``` + +## 8. Noise control rules + +Use these rules to keep measurements trustworthy: + +- Build first (`pnpm --filter=next build`) after framework source changes. +- Compare runs with identical route sets and request knobs. +- Repeat suspicious runs at least once (especially if one route regresses while others improve). +- Use dedicated artifact directories per run. +- Prefer relative deltas across multiple runs over one-off absolute numbers. + +## 9. Suggested iteration loop + +1. Change one thing. +2. Build. +3. Run `scenario=micro` for quick signal. +4. Run focused full stress (`heavy/chunkstorm/wide`) with CPU profile. +5. Analyze hotspots and compare deltas. +6. Keep only changes that hold up across repeat runs. diff --git a/bench/basic-app/app/streaming/_shared/client-boundary.js b/bench/basic-app/app/streaming/_shared/client-boundary.js new file mode 100644 index 000000000000..841c5f43ac7f --- /dev/null +++ b/bench/basic-app/app/streaming/_shared/client-boundary.js @@ -0,0 +1,20 @@ +'use client' + +import React from 'react' + +export function StreamingClientBoundary({ + chunkId, + payload, + fragments, + checksum, +}) { + return ( +
+

client-{chunkId}

+

checksum:{checksum}

+

payload-bytes:{payload.length}

+

fragment-count:{fragments.length}

+

{fragments[0] ?? ''}

+
+ ) +} diff --git a/bench/basic-app/app/streaming/_shared/stress-page.js b/bench/basic-app/app/streaming/_shared/stress-page.js new file mode 100644 index 000000000000..4c590abd618e --- /dev/null +++ b/bench/basic-app/app/streaming/_shared/stress-page.js @@ -0,0 +1,105 @@ +import React, { Suspense } from 'react' + +import { StreamingClientBoundary } from './client-boundary' + +function sleep(ms) { + if (ms <= 0) return Promise.resolve() + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function createPayload(title, payloadBytes) { + const prefix = `${title}:` + if (prefix.length >= payloadBytes) return prefix + return `${prefix}${'x'.repeat(payloadBytes - prefix.length)}` +} + +function createClientPayload({ title, id, payloadBytes, fragmentCount }) { + const payload = createPayload(`${title}-client-${id}`, payloadBytes) + const safeFragmentCount = Math.max(1, fragmentCount) + const fragmentSize = Math.max( + 16, + Math.floor(payload.length / safeFragmentCount) + ) + const fragments = Array.from({ length: safeFragmentCount }, (_, index) => { + const start = Math.min(index * fragmentSize, payload.length) + const end = Math.min(start + fragmentSize, payload.length) + return payload.slice(start, end) + }) + + return { + chunkId: id, + payload, + fragments, + checksum: payload.length + id * 31 + safeFragmentCount, + } +} + +async function StreamedChunk({ + title, + id, + delayMs, + payload, + clientPayloadBytes, + clientPayloadFragments, +}) { + await sleep(delayMs) + + const clientPayload = createClientPayload({ + title, + id, + payloadBytes: clientPayloadBytes, + fragmentCount: clientPayloadFragments, + }) + + return ( +
+

chunk-{id}

+

{payload}

+ +
+ ) +} + +export function StreamingStressPage({ + title, + boundaryCount, + payloadBytes, + clientPayloadBytes = Math.max(128, Math.floor(payloadBytes / 2)), + clientPayloadFragments = 4, + maxDelayMs, +}) { + const payload = createPayload(title, payloadBytes) + const boundaries = Array.from({ length: boundaryCount }, (_, index) => index) + + return ( +
+

{title}

+

+ boundaries={boundaryCount} payloadBytes={payloadBytes}{' '} + clientPayloadBytes= + {clientPayloadBytes} clientPayloadFragments={clientPayloadFragments}{' '} + maxDelayMs={maxDelayMs} +

+ + {boundaries.map((id) => { + const delayMs = maxDelayMs === 0 ? 0 : id % (maxDelayMs + 1) + + return ( + loading-{id}} + > + + + ) + })} +
+ ) +} diff --git a/bench/basic-app/app/streaming/bulk/page.js b/bench/basic-app/app/streaming/bulk/page.js new file mode 100644 index 000000000000..ac797a8c66d0 --- /dev/null +++ b/bench/basic-app/app/streaming/bulk/page.js @@ -0,0 +1,21 @@ +import React from 'react' + +export const dynamic = 'force-dynamic' + +const ROWS = 2500 +const PAYLOAD = 'x'.repeat(384) +const DATA = Array.from( + { length: ROWS }, + (_, index) => `row-${index}-${PAYLOAD}` +) + +export default function Page() { + return ( +
+

stream-bulk

+ {DATA.map((line, index) => ( +

{line}

+ ))} +
+ ) +} diff --git a/bench/basic-app/app/streaming/chunkstorm/page.js b/bench/basic-app/app/streaming/chunkstorm/page.js new file mode 100644 index 000000000000..82ac50556539 --- /dev/null +++ b/bench/basic-app/app/streaming/chunkstorm/page.js @@ -0,0 +1,17 @@ +import React from 'react' +import { StreamingStressPage } from '../_shared/stress-page' + +export const dynamic = 'force-dynamic' + +export default function Page() { + return ( + + ) +} diff --git a/bench/basic-app/app/streaming/heavy/page.js b/bench/basic-app/app/streaming/heavy/page.js new file mode 100644 index 000000000000..08b112e8b51b --- /dev/null +++ b/bench/basic-app/app/streaming/heavy/page.js @@ -0,0 +1,17 @@ +import React from 'react' +import { StreamingStressPage } from '../_shared/stress-page' + +export const dynamic = 'force-dynamic' + +export default function Page() { + return ( + + ) +} diff --git a/bench/basic-app/app/streaming/light/page.js b/bench/basic-app/app/streaming/light/page.js new file mode 100644 index 000000000000..b7d96a0c4566 --- /dev/null +++ b/bench/basic-app/app/streaming/light/page.js @@ -0,0 +1,17 @@ +import React from 'react' +import { StreamingStressPage } from '../_shared/stress-page' + +export const dynamic = 'force-dynamic' + +export default function Page() { + return ( + + ) +} diff --git a/bench/basic-app/app/streaming/medium/page.js b/bench/basic-app/app/streaming/medium/page.js new file mode 100644 index 000000000000..a3e5cf1ae426 --- /dev/null +++ b/bench/basic-app/app/streaming/medium/page.js @@ -0,0 +1,17 @@ +import React from 'react' +import { StreamingStressPage } from '../_shared/stress-page' + +export const dynamic = 'force-dynamic' + +export default function Page() { + return ( + + ) +} diff --git a/bench/basic-app/app/streaming/wide/page.js b/bench/basic-app/app/streaming/wide/page.js new file mode 100644 index 000000000000..e66caae9de8b --- /dev/null +++ b/bench/basic-app/app/streaming/wide/page.js @@ -0,0 +1,17 @@ +import React from 'react' +import { StreamingStressPage } from '../_shared/stress-page' + +export const dynamic = 'force-dynamic' + +export default function Page() { + return ( + + ) +} diff --git a/bench/basic-app/benchmark.sh b/bench/basic-app/benchmark.sh new file mode 100755 index 000000000000..1faecb5f482b --- /dev/null +++ b/bench/basic-app/benchmark.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# Benchmark script for comparing web streams vs node streams performance. +# Uses the minimal server (bench/next-minimal-server) for lowest overhead. +# Warms up with 50 requests, then runs two phases: +# Phase 1: 10s at concurrency=1 (single-client latency) +# Phase 2: 10s at concurrency=100 (throughput under load) +# Reports throughput and latency percentiles for each phase. +# +# Usage: +# ./benchmark.sh [duration] [warmup_requests] +# +# Defaults: 10s duration per phase, 50 warmup requests + +set -euo pipefail + +DURATION=${1:-10} +WARMUP_REQS=${2:-50} +PORT=3199 +NEXT_BIN="../../packages/next/dist/bin/next" +MINIMAL_SERVER="../next-minimal-server/bin/minimal-server.js" + +if ! command -v npx &>/dev/null; then + echo "npx is required (for autocannon)" + exit 1 +fi + +cleanup() { + lsof -ti :"$PORT" 2>/dev/null | xargs kill -9 2>/dev/null || true +} +trap cleanup EXIT + +start_server() { + cleanup + sleep 0.5 + PORT=$PORT node "$MINIMAL_SERVER" &>/dev/null & + SERVER_PID=$! + + # Wait for server to be ready + local retries=0 + while ! curl -sf "http://localhost:$PORT" >/dev/null 2>&1; do + retries=$((retries + 1)) + if [ "$retries" -gt 30 ]; then + echo "ERROR: Server failed to start after 15s" + exit 1 + fi + sleep 0.5 + done +} + +stop_server() { + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + cleanup + sleep 1 +} + +warmup() { + echo " Warming up ($WARMUP_REQS requests)..." + for i in $(seq 1 "$WARMUP_REQS"); do + curl -sf "http://localhost:$PORT" >/dev/null 2>&1 || true + done + sleep 0.5 +} + +run_phase() { + local label="$1" + local connections="$2" + + echo "" + echo " --- $label (${DURATION}s, c=$connections) ---" + + local result + result=$(npx autocannon -d "$DURATION" -c "$connections" -j "http://localhost:$PORT" 2>/dev/null) + + node -e " + const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); + const r = d.requests; + const l = d.latency; + console.log(' Throughput:'); + console.log(' avg: ' + r.average + ' req/s'); + console.log(' mean: ' + r.mean + ' req/s'); + console.log(' total: ' + r.total + ' requests in ${DURATION}s'); + console.log(' Latency:'); + console.log(' avg: ' + l.average.toFixed(2) + ' ms'); + console.log(' p50: ' + l.p50.toFixed(2) + ' ms'); + console.log(' p90: ' + l.p90.toFixed(2) + ' ms'); + console.log(' p99: ' + l.p99.toFixed(2) + ' ms'); + console.log(' max: ' + l.max.toFixed(2) + ' ms'); + " <<< "$result" +} + +run_benchmark() { + local mode="$1" + + echo "" + echo "============================================" + echo " $mode" + echo "============================================" + + start_server + warmup + run_phase "Single client" 1 + run_phase "Under load" 100 + stop_server +} + +echo "Benchmark: web streams vs node streams" +echo "=======================================" +echo "Duration: ${DURATION}s per phase | Warmup: ${WARMUP_REQS} reqs" +echo "Server: minimal-server (minimalMode: true)" + +# --- Web Streams (default) --- +cat > next.config.js <<'CONF' +module.exports = {} +CONF + +echo "" +echo "Building (web streams)..." +node "$NEXT_BIN" build &>/dev/null +run_benchmark "Web Streams (default)" + +# --- Node Streams --- +cat > next.config.js <<'CONF' +module.exports = { + experimental: { + useNodeStreams: true, + }, +} +CONF + +echo "" +echo "Building (node streams)..." +node "$NEXT_BIN" build &>/dev/null +run_benchmark "Node Streams (useNodeStreams: true)" + +# Restore config +cat > next.config.js <<'CONF' +module.exports = {} +CONF + +echo "" +echo "Done." diff --git a/bench/basic-app/next.config.js b/bench/basic-app/next.config.js index 0957c472383f..4ba52ba2c8df 100644 --- a/bench/basic-app/next.config.js +++ b/bench/basic-app/next.config.js @@ -1,5 +1 @@ -module.exports = { - experimental: { - serverMinification: true, - }, -} +module.exports = {} diff --git a/bench/next-minimal-server/bin/minimal-server.js b/bench/next-minimal-server/bin/minimal-server.js index 332e53d0b727..ecd6a0e68eab 100755 --- a/bench/next-minimal-server/bin/minimal-server.js +++ b/bench/next-minimal-server/bin/minimal-server.js @@ -26,13 +26,32 @@ const nextServer = new NextServer({ const requestHandler = nextServer.getRequestHandler() -require('http') - .createServer((req, res) => { - console.time('next-request') - return requestHandler(req, res).finally(() => { - console.timeEnd('next-request') - }) - }) - .listen(3000, () => { - console.timeEnd('next-cold-start') +const port = parseInt(process.env.PORT, 10) || 3000 + +const server = require('http').createServer((req, res) => { + return requestHandler(req, res) +}) + +server.listen(port, () => { + console.timeEnd('next-cold-start') + console.log('Listening on port ' + port) +}) + +let shuttingDown = false +function shutdown() { + if (shuttingDown) return + shuttingDown = true + + // Allow Node to exit cleanly so --cpu-prof/--heap-prof outputs are flushed. + server.close(() => { + process.exit(0) }) + + // Fallback in case active keep-alive connections prevent close callback. + setTimeout(() => { + process.exit(1) + }, 5000).unref() +} + +process.on('SIGTERM', shutdown) +process.on('SIGINT', shutdown) diff --git a/bench/render-pipeline/.gitignore b/bench/render-pipeline/.gitignore new file mode 100644 index 000000000000..d4f588edfef5 --- /dev/null +++ b/bench/render-pipeline/.gitignore @@ -0,0 +1 @@ +artifacts/ diff --git a/bench/render-pipeline/README.md b/bench/render-pipeline/README.md new file mode 100644 index 000000000000..da861658a620 --- /dev/null +++ b/bench/render-pipeline/README.md @@ -0,0 +1,105 @@ +# Render Pipeline Benchmark + +This benchmark targets the full App Router render path (`renderToHTMLOrFlight`) via real HTTP requests through `bench/next-minimal-server`. + +It supports: +- `web` vs `node` streams mode comparison +- route-based stress suites for streaming SSR +- CPU/heap profiling for the server process +- Node trace events and Next internal trace artifact capture + +## Quick start + +Run end-to-end benchmark (default stress routes): + +```bash +pnpm bench:render-pipeline --scenario=full --stream-mode=both +``` + +For `scenario=full` and `scenario=all`, CPU profiles are captured by default. +Disable with `--capture-cpu=false` if you want lower-overhead runs. + +Skip rebuild for faster iteration (after you already built once): + +```bash +pnpm bench:render-pipeline --scenario=full --stream-mode=node --build-full=false +``` + +When `--stream-mode=both`, the runner forces `--build-full=true` so web/node +comparisons do not accidentally reuse stale build output. + +Output JSON report: + +```bash +pnpm bench:render-pipeline --scenario=full --stream-mode=both --json-out=/tmp/render-pipeline.json +``` + +## Profiling and traces + +Capture CPU profiles + Node trace events + Next trace logs: + +```bash +pnpm bench:render-pipeline \ + --scenario=full \ + --stream-mode=both \ + --capture-trace=true \ + --capture-next-trace=true +``` + +Artifacts are written to: + +```text +bench/render-pipeline/artifacts// +``` + +Per mode (`web` and `node`) this includes: +- `.cpuprofile` (if `--capture-cpu=true`) +- `.heapprofile` (if `--capture-heap=true`) +- `-trace-*.json` (if `--capture-trace=true`) +- `next-trace-build.log` and `next-runtime-trace.log` (if `--capture-next-trace=true`) + +Open `.cpuprofile` files in Chrome DevTools Performance panel. + +Analyze results and CPU hotspots from artifacts: + +```bash +pnpm bench:render-pipeline:analyze --artifact-dir=bench/render-pipeline/artifacts/ +``` + +Omit `--artifact-dir` to analyze the latest run automatically. + +## Stress routes + +Default routes: +- `/` +- `/streaming/light` +- `/streaming/medium` +- `/streaming/heavy` +- `/streaming/chunkstorm` +- `/streaming/wide` +- `/streaming/bulk` + +The `streaming/*` pages now include a client boundary per Suspense chunk, so benchmark runs also stress Server-to-Client payload serialization in Flight data. + +Override with: + +```bash +pnpm bench:render-pipeline --scenario=full --routes=/,/streaming/heavy +``` + +## Common tuning flags + +- `--warmup-requests=30` +- `--serial-requests=120` +- `--load-requests=1200` +- `--load-concurrency=80` +- `--timeout-ms=30000` +- `--port=3199` + +## Optional micro benchmarks + +The runner also supports helper-only micro benchmarks: + +```bash +pnpm bench:render-pipeline --scenario=micro +``` diff --git a/bench/render-pipeline/analyze-profiles.ts b/bench/render-pipeline/analyze-profiles.ts new file mode 100644 index 000000000000..05cf255d2cfe --- /dev/null +++ b/bench/render-pipeline/analyze-profiles.ts @@ -0,0 +1,399 @@ +// This script must be run with tsx + +import { constants } from 'node:fs' +import { access, readdir, readFile, stat } from 'node:fs/promises' +import { SourceMap } from 'node:module' +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const REPO_ROOT = fileURLToPath(new URL('../..', import.meta.url)) +const DEFAULT_ARTIFACTS_ROOT = resolve( + REPO_ROOT, + 'bench/render-pipeline/artifacts' +) + +type FullRoutePhaseResult = { + mode: 'web' | 'node' + route: string + phase: 'single-client' | 'under-load' + requests: number + concurrency: number + throughputRps: number + latency: { + min: number + median: number + mean: number + p95: number + max: number + } +} + +type BenchmarkJson = { + fullResults?: Array<{ + mode: 'web' | 'node' + routeResults: FullRoutePhaseResult[] + }> +} + +type ProfileAnalysis = { + totalUs: number + runtimeUs: number + runtimeFile: string | null + topModules: Array<{ name: string; us: number }> + topRuntimeSources: Array<{ name: string; us: number }> + topRuntimeSymbols: Array<{ name: string; us: number }> +} + +function usage() { + console.log(`Usage: pnpm bench:render-pipeline:analyze [options] + +Options: + --artifact-dir= Artifact run directory, or parent artifacts directory. + Default: latest run under bench/render-pipeline/artifacts + --top= Number of top hotspots to show per section (default: 15) +`) +} + +function parseArgs() { + const rawArgs = process.argv.slice(2) + if (rawArgs.includes('--help')) { + usage() + process.exit(0) + } + + const args = new Map() + for (const rawArg of rawArgs) { + if (!rawArg.startsWith('--')) continue + const [rawKey, rawValue] = rawArg.slice(2).split('=') + args.set(rawKey, rawValue ?? 'true') + } + + const topRaw = args.get('top') + const top = topRaw ? Number(topRaw) : 15 + if (!Number.isFinite(top) || top < 1) { + throw new Error(`Invalid --top value: ${topRaw}`) + } + + return { + artifactDirArg: args.get('artifact-dir'), + top: Math.floor(top), + } +} + +async function exists(path: string): Promise { + try { + await access(path, constants.F_OK) + return true + } catch { + return false + } +} + +async function resolveArtifactRunDir(artifactDirArg?: string): Promise { + const requested = resolve(REPO_ROOT, artifactDirArg ?? DEFAULT_ARTIFACTS_ROOT) + const requestedResults = resolve(requested, 'results.json') + if (await exists(requestedResults)) { + return requested + } + + const entries = await readdir(requested, { withFileTypes: true }) + const dirs = entries.filter((entry) => entry.isDirectory()) + const runs: Array<{ dir: string; mtimeMs: number }> = [] + + for (const dirent of dirs) { + const dir = resolve(requested, dirent.name) + const resultsPath = resolve(dir, 'results.json') + if (!(await exists(resultsPath))) continue + const stats = await stat(resultsPath) + runs.push({ dir, mtimeMs: stats.mtimeMs }) + } + + if (runs.length === 0) { + throw new Error( + `No artifact run found in ${requested}. Expected a results.json file.` + ) + } + + runs.sort((a, b) => b.mtimeMs - a.mtimeMs) + return runs[0].dir +} + +function toPercent(part: number, total: number): string { + if (total <= 0) return '0.00%' + return `${((part / total) * 100).toFixed(2)}%` +} + +function toMs(us: number): string { + return `${(us / 1000).toFixed(1)}ms` +} + +function sortTop( + entries: Iterable<[string, number]>, + limit: number +): Array<{ name: string; us: number }> { + return [...entries] + .sort((a, b) => b[1] - a[1]) + .slice(0, limit) + .map(([name, us]) => ({ name, us })) +} + +function mapModuleFromUrl(url: string): string { + if (!url || url === '(no-url)') return '(no-url)' + if (url.startsWith('node:')) return url + const appPageMatch = url.match(/app-page-turbo[\w-]*\.runtime\.prod\.js/) + if (appPageMatch) return appPageMatch[0] + if (url.includes('/.next/server/chunks/')) return '.next/server/chunks/*' + if (url.includes('/next/dist/')) return 'next/dist/*' + if (url.includes('/node_modules/')) return 'node_modules/*' + return url +} + +function detectRuntimeFile( + urlsByUs: Array<{ url: string; us: number }> +): string | null { + for (const entry of urlsByUs) { + const match = entry.url.match(/app-page-turbo[\w-]*\.runtime\.prod\.js/) + if (match) return match[0] + } + return null +} + +async function analyzeProfile( + profilePath: string, + top: number +): Promise { + if (!(await exists(profilePath))) return null + + const rawProfile = await readFile(profilePath, 'utf8') + const profile = JSON.parse(rawProfile) as { + nodes: Array<{ + id: number + callFrame: { + functionName: string + url: string + lineNumber: number + columnNumber: number + } + }> + samples: number[] + timeDeltas: number[] + } + + const idToNode = new Map(profile.nodes.map((node) => [node.id, node])) + const urlTotals = new Map() + const moduleTotals = new Map() + let totalUs = 0 + + for (let i = 0; i < profile.samples.length; i++) { + const sampleId = profile.samples[i] + const deltaUs = profile.timeDeltas[i] ?? 0 + totalUs += deltaUs + + const node = idToNode.get(sampleId) + if (!node) continue + const url = node.callFrame.url || '(no-url)' + urlTotals.set(url, (urlTotals.get(url) ?? 0) + deltaUs) + + const moduleName = mapModuleFromUrl(url) + moduleTotals.set(moduleName, (moduleTotals.get(moduleName) ?? 0) + deltaUs) + } + + const topUrls = sortTop(urlTotals.entries(), 30).map((entry) => ({ + url: entry.name, + us: entry.us, + })) + const runtimeFile = detectRuntimeFile(topUrls) + + let runtimeUs = 0 + const runtimeSources = new Map() + const runtimeSymbols = new Map() + let sourceMap: SourceMap | null = null + + if (runtimeFile) { + const mapPath = resolve( + REPO_ROOT, + `packages/next/dist/compiled/next-server/${runtimeFile}.map` + ) + if (await exists(mapPath)) { + sourceMap = new SourceMap(JSON.parse(await readFile(mapPath, 'utf8'))) + } + } + + if (runtimeFile) { + for (let i = 0; i < profile.samples.length; i++) { + const sampleId = profile.samples[i] + const deltaUs = profile.timeDeltas[i] ?? 0 + const node = idToNode.get(sampleId) + if (!node) continue + + const { callFrame } = node + if (!callFrame.url.includes(runtimeFile)) continue + runtimeUs += deltaUs + + const generatedLine = callFrame.lineNumber ?? 0 + const generatedColumn = callFrame.columnNumber ?? 0 + + let sourceName = callFrame.url + let symbolName = callFrame.functionName || '(anonymous)' + let sourceLine = generatedLine + let sourceColumn = generatedColumn + + if (sourceMap) { + const entry = sourceMap.findEntry(generatedLine, generatedColumn) as { + originalSource?: string + originalLine?: number + originalColumn?: number + name?: string + } + if (entry.originalSource) sourceName = entry.originalSource + if (entry.name) symbolName = entry.name + if (entry.originalLine !== undefined) sourceLine = entry.originalLine + if (entry.originalColumn !== undefined) + sourceColumn = entry.originalColumn + } + + runtimeSources.set( + sourceName, + (runtimeSources.get(sourceName) ?? 0) + deltaUs + ) + const symbolKey = `${symbolName} @ ${sourceName}:${sourceLine}:${sourceColumn}` + runtimeSymbols.set( + symbolKey, + (runtimeSymbols.get(symbolKey) ?? 0) + deltaUs + ) + } + } + + return { + totalUs, + runtimeUs, + runtimeFile, + topModules: sortTop(moduleTotals.entries(), top), + topRuntimeSources: sortTop(runtimeSources.entries(), top), + topRuntimeSymbols: sortTop(runtimeSymbols.entries(), top), + } +} + +function printProfileAnalysis( + mode: 'web' | 'node', + analysis: ProfileAnalysis, + top: number +) { + console.log(`\n[${mode}]`) + console.log(` sampled: ${toMs(analysis.totalUs)}`) + if (analysis.runtimeFile) { + console.log( + ` runtime: ${analysis.runtimeFile} (${toMs(analysis.runtimeUs)}, ${toPercent(analysis.runtimeUs, analysis.totalUs)})` + ) + } else { + console.log(' runtime: not detected') + } + + console.log(` top ${top} modules:`) + for (const entry of analysis.topModules) { + console.log( + ` ${toPercent(entry.us, analysis.totalUs).padStart(7)} ${toMs(entry.us).padStart(9)} ${entry.name}` + ) + } + + if (analysis.topRuntimeSources.length > 0) { + console.log(` top ${top} runtime sources:`) + for (const entry of analysis.topRuntimeSources) { + console.log( + ` ${toPercent(entry.us, analysis.runtimeUs).padStart(7)} ${toMs(entry.us).padStart(9)} ${entry.name}` + ) + } + } + + if (analysis.topRuntimeSymbols.length > 0) { + console.log(` top ${top} runtime symbols:`) + for (const entry of analysis.topRuntimeSymbols) { + console.log( + ` ${toPercent(entry.us, analysis.runtimeUs).padStart(7)} ${toMs(entry.us).padStart(9)} ${entry.name}` + ) + } + } +} + +function printComparison(results: BenchmarkJson) { + const fullResults = results.fullResults + if (!fullResults || fullResults.length < 2) return + + const web = fullResults.find((entry) => entry.mode === 'web') + const node = fullResults.find((entry) => entry.mode === 'node') + if (!web || !node) return + + const webByKey = new Map( + web.routeResults.map((item) => [`${item.route}|${item.phase}`, item]) + ) + + console.log('\n[comparison node vs web]') + console.log( + ' route'.padEnd(20) + + 'phase'.padEnd(16) + + 'RPS delta'.padEnd(14) + + 'P95 delta' + ) + + for (const nodeEntry of node.routeResults) { + const key = `${nodeEntry.route}|${nodeEntry.phase}` + const webEntry = webByKey.get(key) + if (!webEntry) continue + const rpsDelta = + ((nodeEntry.throughputRps - webEntry.throughputRps) / + webEntry.throughputRps) * + 100 + const p95Delta = + ((webEntry.latency.p95 - nodeEntry.latency.p95) / webEntry.latency.p95) * + 100 + + const line = + ` ${nodeEntry.route}`.padEnd(20) + + `${nodeEntry.phase}`.padEnd(16) + + `${rpsDelta >= 0 ? '+' : ''}${rpsDelta.toFixed(2)}%`.padEnd(14) + + `${p95Delta >= 0 ? '+' : ''}${p95Delta.toFixed(2)}%` + console.log(line) + } +} + +async function main() { + const { artifactDirArg, top } = parseArgs() + const runDir = await resolveArtifactRunDir(artifactDirArg) + + console.log(`Analyzing render pipeline artifacts:`) + console.log(` ${runDir}`) + + const resultsPath = resolve(runDir, 'results.json') + const resultsRaw = await readFile(resultsPath, 'utf8') + const resultsJson = JSON.parse(resultsRaw) as BenchmarkJson + printComparison(resultsJson) + + const webProfile = resolve(runDir, 'web/web.cpuprofile') + const nodeProfile = resolve(runDir, 'node/node.cpuprofile') + + const [webAnalysis, nodeAnalysis] = await Promise.all([ + analyzeProfile(webProfile, top), + analyzeProfile(nodeProfile, top), + ]) + + if (!webAnalysis && !nodeAnalysis) { + console.log('\nNo CPU profiles found in this artifact run.') + console.log( + 'This analyzer reads only /.cpuprofile artifacts (not trace-event JSON or next-runtime-trace.log).' + ) + console.log( + 'Run benchmark with --capture-cpu=true, e.g. pnpm bench:render-pipeline --scenario=full --stream-mode=node --capture-cpu=true' + ) + return + } + + if (webAnalysis) printProfileAnalysis('web', webAnalysis, top) + if (nodeAnalysis) printProfileAnalysis('node', nodeAnalysis, top) + + console.log('\nDone.') +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/bench/render-pipeline/benchmark.ts b/bench/render-pipeline/benchmark.ts new file mode 100644 index 000000000000..79169d7ccdd7 --- /dev/null +++ b/bench/render-pipeline/benchmark.ts @@ -0,0 +1,1127 @@ +// This script must be run with tsx + +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { access, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises' +import { resolve } from 'node:path' +import { performance } from 'node:perf_hooks' +import { Readable } from 'node:stream' +import { setTimeout as sleep } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' +import { teeNodeReadable } from '../../packages/next/src/server/app-render/node-stream-tee' +import { + createInlinedDataNodeStream, + createInlinedDataReadableStream, +} from '../../packages/next/src/server/app-render/use-flight-response' +import { + chainNodeTransforms, + continueDynamicHTMLResumeNode, + continueDynamicPrerenderNode, + continueStaticPrerenderNode, + createBufferedTransformNode, +} from '../../packages/next/src/server/stream-utils/node-stream-helpers' +import { + continueDynamicHTMLResume, + continueDynamicPrerender, + continueStaticPrerender, +} from '../../packages/next/src/server/stream-utils/node-web-streams-helper' + +const REPO_ROOT = fileURLToPath(new URL('../..', import.meta.url)) +const NEXT_BIN = resolve(REPO_ROOT, 'packages/next/dist/bin/next') +const MINIMAL_SERVER = resolve( + REPO_ROOT, + 'bench/next-minimal-server/bin/minimal-server.js' +) + +type Scenario = 'full' | 'micro' | 'all' +type StreamMode = 'web' | 'node' | 'both' + +type CliOptions = { + scenario: Scenario + jsonOut?: string + + appDir: string + routes: string[] + streamMode: StreamMode + buildFull: boolean + warmupRequests: number + serialRequests: number + loadRequests: number + loadConcurrency: number + timeoutMs: number + port: number + + captureCpu: boolean + captureHeap: boolean + captureTrace: boolean + captureNextTrace: boolean + traceCategories: string + artifactDir: string + + iterations: number + warmup: number + htmlChunks: number + htmlChunkBytes: number + flightChunks: number + flightChunkBytes: number + binaryFlight: boolean +} + +type BenchStats = { + min: number + median: number + mean: number + p95: number + max: number +} + +type BenchResult = { + name: string + group: 'unit' | 'integration' + stats: BenchStats +} + +type BenchCase = { + name: string + group: 'unit' | 'integration' + run: () => Promise +} + +type FullRoutePhaseResult = { + mode: 'web' | 'node' + route: string + phase: 'single-client' | 'under-load' + requests: number + concurrency: number + throughputRps: number + latency: BenchStats +} + +type FullRunResult = { + mode: 'web' | 'node' + routeResults: FullRoutePhaseResult[] +} + +function parseBoolean(value: string): boolean { + return value === '1' || value === 'true' || value === 'yes' +} + +function parseNumberArg( + args: Map, + key: string, + fallback: number +): number { + const value = args.get(key) + if (value === undefined) return fallback + const parsed = Number(value) + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid numeric value for --${key}: ${value}`) + } + return parsed +} + +function parseRoutes(rawRoutes: string | undefined): string[] { + if (!rawRoutes) { + return [ + '/', + '/streaming/light', + '/streaming/medium', + '/streaming/heavy', + '/streaming/chunkstorm', + '/streaming/wide', + '/streaming/bulk', + ] + } + + const routes = rawRoutes + .split(',') + .map((route) => route.trim()) + .filter(Boolean) + + if (routes.length === 0) { + throw new Error('--routes cannot be empty') + } + + for (const route of routes) { + if (!route.startsWith('/')) { + throw new Error(`Each route must start with '/': ${route}`) + } + } + + return routes +} + +function usage() { + console.log(`Usage: pnpm bench:render-pipeline [options] + +Defaults to FULL end-to-end app-render benchmark. + +Options: + --scenario=full|micro|all (default: full) + --json-out= + +Full benchmark options: + --app-dir= (default: bench/basic-app) + --routes=/,/streaming/light,... (default: built-in stress suite) + --stream-mode=web|node|both (default: both) + --build-full=true|false (default: true) + When stream-mode=both, build-full is forced to true. + --warmup-requests= (default: 30) + --serial-requests= (default: 120) + --load-requests= (default: 1200) + --load-concurrency= (default: 80) + --port= (default: 3199) + --timeout-ms= (default: 30000) + +Profiling and trace options: + --capture-cpu=true|false (default: true for scenario=full|all, false for scenario=micro) + --capture-heap=true|false (default: false) + --capture-trace=true|false (default: false) + --capture-next-trace=true|false (default: true) + --trace-categories= (default: node,node.async_hooks,v8) + --artifact-dir= (default: bench/render-pipeline/artifacts/) + +Micro benchmark options: + --iterations= (default: 10) + --warmup= (default: 2) + --html-chunks= (default: 64) + --html-chunk-bytes= (default: 16384) + --flight-chunks= (default: 64) + --flight-chunk-bytes= (default: 4096) + --binary-flight=true|false (default: true) +`) +} + +function parseCli(): CliOptions { + const rawArgs = process.argv.slice(2) + if (rawArgs.includes('--help')) { + usage() + process.exit(0) + } + + const args = new Map() + for (const rawArg of rawArgs) { + if (!rawArg.startsWith('--')) continue + const [rawKey, rawValue] = rawArg.slice(2).split('=') + args.set(rawKey, rawValue ?? 'true') + } + + const scenarioRaw = args.get('scenario') ?? 'full' + if ( + scenarioRaw !== 'full' && + scenarioRaw !== 'micro' && + scenarioRaw !== 'all' + ) { + throw new Error( + `Invalid --scenario value: ${scenarioRaw}. Use full|micro|all` + ) + } + + const streamModeRaw = args.get('stream-mode') ?? 'both' + if ( + streamModeRaw !== 'web' && + streamModeRaw !== 'node' && + streamModeRaw !== 'both' + ) { + throw new Error( + `Invalid --stream-mode value: ${streamModeRaw}. Use web|node|both` + ) + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-') + const artifactDir = resolve( + REPO_ROOT, + args.get('artifact-dir') ?? `bench/render-pipeline/artifacts/${timestamp}` + ) + + const htmlChunkBytes = parseNumberArg(args, 'html-chunk-bytes', 16 * 1024) + const flightChunkBytes = parseNumberArg(args, 'flight-chunk-bytes', 4 * 1024) + const iterations = parseNumberArg(args, 'iterations', 10) + const warmup = parseNumberArg(args, 'warmup', 2) + + if (htmlChunkBytes < 64) throw new Error('--html-chunk-bytes must be >= 64') + if (flightChunkBytes < 64) + throw new Error('--flight-chunk-bytes must be >= 64') + if (iterations < 1) throw new Error('--iterations must be >= 1') + if (warmup < 0) throw new Error('--warmup must be >= 0') + + const routes = parseRoutes(args.get('routes')) + const buildFull = parseBoolean(args.get('build-full') ?? 'true') + const defaultCaptureCpu = + scenarioRaw === 'full' || scenarioRaw === 'all' ? 'true' : 'false' + const shouldForceBuildFull = + (scenarioRaw === 'full' || scenarioRaw === 'all') && + streamModeRaw === 'both' && + !buildFull + + if (shouldForceBuildFull) { + console.warn( + '[bench/render-pipeline] forcing --build-full=true for stream-mode=both to avoid comparing stale build output.' + ) + } + + return { + scenario: scenarioRaw, + jsonOut: args.get('json-out'), + + appDir: resolve(REPO_ROOT, args.get('app-dir') ?? 'bench/basic-app'), + routes, + streamMode: streamModeRaw, + buildFull: buildFull || shouldForceBuildFull, + warmupRequests: parseNumberArg(args, 'warmup-requests', 30), + serialRequests: parseNumberArg(args, 'serial-requests', 120), + loadRequests: parseNumberArg(args, 'load-requests', 1200), + loadConcurrency: parseNumberArg(args, 'load-concurrency', 80), + timeoutMs: parseNumberArg(args, 'timeout-ms', 30_000), + port: parseNumberArg(args, 'port', 3199), + + captureCpu: parseBoolean(args.get('capture-cpu') ?? defaultCaptureCpu), + captureHeap: parseBoolean(args.get('capture-heap') ?? 'false'), + captureTrace: parseBoolean(args.get('capture-trace') ?? 'false'), + captureNextTrace: parseBoolean(args.get('capture-next-trace') ?? 'true'), + traceCategories: args.get('trace-categories') ?? 'node,node.async_hooks,v8', + artifactDir, + + iterations, + warmup, + htmlChunks: parseNumberArg(args, 'html-chunks', 64), + htmlChunkBytes, + flightChunks: parseNumberArg(args, 'flight-chunks', 64), + flightChunkBytes, + binaryFlight: parseBoolean(args.get('binary-flight') ?? 'true'), + } +} + +function fixedSizeChunkWithPrefix(prefix: Buffer, size: number, fill: number) { + if (prefix.byteLength >= size) { + return prefix.subarray(0, size) + } + return Buffer.concat([prefix, Buffer.alloc(size - prefix.byteLength, fill)]) +} + +function fixedSizeChunkWithSuffix(suffix: Buffer, size: number, fill: number) { + if (suffix.byteLength >= size) { + return suffix.subarray(suffix.byteLength - size) + } + return Buffer.concat([Buffer.alloc(size - suffix.byteLength, fill), suffix]) +} + +function makeHtmlChunks(chunkCount: number, chunkBytes: number): Buffer[] { + const chunks: Buffer[] = [] + const prefix = Buffer.from('') + const suffix = Buffer.from('') + + if (chunkCount < 2) { + throw new Error('--html-chunks must be >= 2') + } + + chunks.push(fixedSizeChunkWithPrefix(prefix, chunkBytes, 97)) + + for (let i = 1; i < chunkCount - 1; i++) { + chunks.push(Buffer.alloc(chunkBytes, 97 + (i % 26))) + } + + chunks.push(fixedSizeChunkWithSuffix(suffix, chunkBytes, 122)) + return chunks +} + +function makeFlightChunks( + chunkCount: number, + chunkBytes: number, + binary: boolean +): Buffer[] { + const chunks: Buffer[] = [] + for (let i = 0; i < chunkCount; i++) { + const chunk = Buffer.alloc(chunkBytes) + if (binary) { + for (let j = 0; j < chunkBytes; j++) { + chunk[j] = (i * 17 + j * 31) % 256 + } + } else { + chunk.fill(97 + (i % 26)) + } + chunks.push(chunk) + } + return chunks +} + +function createWebStream( + chunks: readonly Uint8Array[] +): ReadableStream { + let index = 0 + return new ReadableStream({ + pull(controller) { + if (index >= chunks.length) { + controller.close() + return + } + controller.enqueue(chunks[index++]) + }, + }) +} + +async function consumeNodeReadable(stream: Readable): Promise { + let totalBytes = 0 + for await (const chunk of stream) { + if (typeof chunk === 'string') { + totalBytes += Buffer.byteLength(chunk) + } else { + totalBytes += (chunk as Uint8Array).byteLength + } + } + return totalBytes +} + +async function consumeWebReadable( + stream: ReadableStream +): Promise { + let totalBytes = 0 + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) break + totalBytes += value.byteLength + } + return totalBytes +} + +function computeStats(samples: number[]): BenchStats { + const sorted = [...samples].sort((a, b) => a - b) + const min = sorted[0] + const max = sorted[sorted.length - 1] + const median = sorted[Math.floor(sorted.length / 2)] + const mean = samples.reduce((sum, value) => sum + value, 0) / samples.length + const p95 = sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] + return { min, median, mean, p95, max } +} + +async function runBenchCase( + bench: BenchCase, + iterations: number, + warmup: number +): Promise { + for (let i = 0; i < warmup; i++) { + await bench.run() + } + + const samples: number[] = [] + for (let i = 0; i < iterations; i++) { + const start = performance.now() + await bench.run() + samples.push(performance.now() - start) + } + + return { + name: bench.name, + group: bench.group, + stats: computeStats(samples), + } +} + +function printMicroResults(results: BenchResult[]) { + const groups: Array<'unit' | 'integration'> = ['unit', 'integration'] + for (const group of groups) { + const groupResults = results.filter((result) => result.group === group) + if (groupResults.length === 0) continue + console.log(`\n${group.toUpperCase()} BENCHMARKS`) + console.log( + 'name'.padEnd(42), + 'median'.padStart(10), + 'p95'.padStart(10), + 'mean'.padStart(10), + 'min'.padStart(10), + 'max'.padStart(10) + ) + for (const result of groupResults) { + const { stats } = result + console.log( + result.name.padEnd(42), + `${stats.median.toFixed(2)}ms`.padStart(10), + `${stats.p95.toFixed(2)}ms`.padStart(10), + `${stats.mean.toFixed(2)}ms`.padStart(10), + `${stats.min.toFixed(2)}ms`.padStart(10), + `${stats.max.toFixed(2)}ms`.padStart(10) + ) + } + } +} + +function buildMicroBenchCases( + htmlChunks: Buffer[], + flightChunks: Buffer[], + secondaryFlightChunks: Buffer[], + secondaryFlightLabel: string +): BenchCase[] { + const webHtmlChunks = htmlChunks.map( + (chunk) => new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength) + ) + const webFlightChunks = flightChunks.map( + (chunk) => new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength) + ) + const webSecondaryFlightChunks = secondaryFlightChunks.map( + (chunk) => new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength) + ) + + return [ + { + name: 'teeNodeReadable (drain both branches)', + group: 'unit', + run: async () => { + const source = Readable.from(htmlChunks) + const [left, right] = teeNodeReadable(source) + const [leftBytes, rightBytes] = await Promise.all([ + consumeNodeReadable(left as Readable), + consumeNodeReadable(right as Readable), + ]) + return leftBytes + rightBytes + }, + }, + { + name: 'createBufferedTransformNode only', + group: 'unit', + run: async () => { + const source = Readable.from(htmlChunks) + const transformed = chainNodeTransforms(source, [ + createBufferedTransformNode(), + ]) + return consumeNodeReadable(transformed) + }, + }, + { + name: 'createInlinedDataNodeStream only', + group: 'unit', + run: async () => { + const source = Readable.from(flightChunks) + const transformed = chainNodeTransforms(source, [ + createInlinedDataNodeStream(undefined, null), + ]) + return consumeNodeReadable(transformed) + }, + }, + { + name: `createInlinedDataNodeStream only (${secondaryFlightLabel})`, + group: 'unit', + run: async () => { + const source = Readable.from(secondaryFlightChunks) + const transformed = chainNodeTransforms(source, [ + createInlinedDataNodeStream(undefined, null), + ]) + return consumeNodeReadable(transformed) + }, + }, + { + name: 'Node continueStaticPrerender', + group: 'integration', + run: async () => { + const renderStream = Readable.from(htmlChunks) + const inlinedDataStream = chainNodeTransforms( + Readable.from(flightChunks), + [createInlinedDataNodeStream(undefined, null)] + ) + const stream = await continueStaticPrerenderNode(renderStream, { + inlinedDataStream, + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeNodeReadable(stream) + }, + }, + { + name: 'Node continueDynamicPrerender', + group: 'integration', + run: async () => { + const renderStream = Readable.from(htmlChunks) + const stream = await continueDynamicPrerenderNode(renderStream, { + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeNodeReadable(stream) + }, + }, + { + name: 'Node continueDynamicHTMLResume', + group: 'integration', + run: async () => { + const renderStream = Readable.from(htmlChunks) + const inlinedDataStream = chainNodeTransforms( + Readable.from(flightChunks), + [createInlinedDataNodeStream(undefined, null)] + ) + const stream = await continueDynamicHTMLResumeNode(renderStream, { + inlinedDataStream, + delayDataUntilFirstHtmlChunk: false, + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeNodeReadable(stream) + }, + }, + { + name: `Node continueDynamicHTMLResume (${secondaryFlightLabel})`, + group: 'integration', + run: async () => { + const renderStream = Readable.from(htmlChunks) + const inlinedDataStream = chainNodeTransforms( + Readable.from(secondaryFlightChunks), + [createInlinedDataNodeStream(undefined, null)] + ) + const stream = await continueDynamicHTMLResumeNode(renderStream, { + inlinedDataStream, + delayDataUntilFirstHtmlChunk: false, + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeNodeReadable(stream) + }, + }, + { + name: 'Web continueStaticPrerender', + group: 'integration', + run: async () => { + const renderStream = createWebStream(webHtmlChunks) + const inlinedDataStream = createInlinedDataReadableStream( + createWebStream(webFlightChunks), + undefined, + null + ) + const stream = await continueStaticPrerender(renderStream, { + inlinedDataStream, + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeWebReadable(stream) + }, + }, + { + name: 'Web continueDynamicPrerender', + group: 'integration', + run: async () => { + const renderStream = createWebStream(webHtmlChunks) + const stream = await continueDynamicPrerender(renderStream, { + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeWebReadable(stream) + }, + }, + { + name: 'Web continueDynamicHTMLResume', + group: 'integration', + run: async () => { + const renderStream = createWebStream(webHtmlChunks) + const inlinedDataStream = createInlinedDataReadableStream( + createWebStream(webFlightChunks), + undefined, + null + ) + const stream = await continueDynamicHTMLResume(renderStream, { + inlinedDataStream, + delayDataUntilFirstHtmlChunk: false, + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeWebReadable(stream) + }, + }, + { + name: `Web continueDynamicHTMLResume (${secondaryFlightLabel})`, + group: 'integration', + run: async () => { + const renderStream = createWebStream(webHtmlChunks) + const inlinedDataStream = createInlinedDataReadableStream( + createWebStream(webSecondaryFlightChunks), + undefined, + null + ) + const stream = await continueDynamicHTMLResume(renderStream, { + inlinedDataStream, + delayDataUntilFirstHtmlChunk: false, + getServerInsertedHTML: async () => '', + getServerInsertedMetadata: async () => '', + deploymentId: undefined, + }) + return consumeWebReadable(stream) + }, + }, + ] +} + +async function runMicroBenchmarks(options: CliOptions): Promise { + const prevRuntime = process.env.NEXT_RUNTIME + const prevUseNodeStreams = process.env.__NEXT_USE_NODE_STREAMS + process.env.NEXT_RUNTIME = 'nodejs' + process.env.__NEXT_USE_NODE_STREAMS = 'true' + + try { + const htmlChunks = makeHtmlChunks( + options.htmlChunks, + options.htmlChunkBytes + ) + const flightChunks = makeFlightChunks( + options.flightChunks, + options.flightChunkBytes, + options.binaryFlight + ) + const secondaryFlightChunks = makeFlightChunks( + options.flightChunks, + options.flightChunkBytes, + !options.binaryFlight + ) + const secondaryFlightLabel = options.binaryFlight + ? 'utf8 flight' + : 'binary flight' + + const cases = buildMicroBenchCases( + htmlChunks, + flightChunks, + secondaryFlightChunks, + secondaryFlightLabel + ) + const results: BenchResult[] = [] + for (const benchCase of cases) { + const result = await runBenchCase( + benchCase, + options.iterations, + options.warmup + ) + results.push(result) + } + return results + } finally { + if (prevRuntime === undefined) { + delete process.env.NEXT_RUNTIME + } else { + process.env.NEXT_RUNTIME = prevRuntime + } + + if (prevUseNodeStreams === undefined) { + delete process.env.__NEXT_USE_NODE_STREAMS + } else { + process.env.__NEXT_USE_NODE_STREAMS = prevUseNodeStreams + } + } +} + +async function runCommand( + command: string, + args: string[], + cwd: string, + env: NodeJS.ProcessEnv = process.env +): Promise { + const child = spawn(command, args, { + cwd, + env, + stdio: 'inherit', + }) + const [code] = (await once(child, 'exit')) as [number | null] + if (code !== 0) { + throw new Error( + `Command failed: ${command} ${args.join(' ')} (exit ${code})` + ) + } +} + +async function ensureNextBuilt() { + try { + await access(NEXT_BIN) + } catch { + throw new Error( + `Missing ${NEXT_BIN}. Build Next.js first (pnpm --filter=next build).` + ) + } +} + +function configForMode(mode: 'web' | 'node'): string { + if (mode === 'web') { + return 'module.exports = {}\n' + } + return `module.exports = { + experimental: { + useNodeStreams: true, + }, +}\n` +} + +async function waitForServerReady( + url: string, + timeoutMs: number +): Promise { + const start = performance.now() + while (performance.now() - start < timeoutMs) { + try { + const response = await fetch(url, { cache: 'no-store' }) + await response.arrayBuffer() + if (response.ok) return + } catch { + // server not ready yet + } + await sleep(200) + } + throw new Error(`Server did not become ready within ${timeoutMs}ms`) +} + +async function requestLatencyMs( + url: string, + timeoutMs: number +): Promise { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), timeoutMs) + + try { + const start = performance.now() + const response = await fetch(url, { + cache: 'no-store', + signal: controller.signal, + }) + await response.arrayBuffer() + if (!response.ok) { + throw new Error(`Request failed (${response.status}) for ${url}`) + } + return performance.now() - start + } finally { + clearTimeout(timeout) + } +} + +async function runSerialRequests( + url: string, + count: number, + timeoutMs: number +): Promise { + const latencies: number[] = [] + for (let i = 0; i < count; i++) { + latencies.push(await requestLatencyMs(url, timeoutMs)) + } + return latencies +} + +async function runConcurrentRequests( + url: string, + totalRequests: number, + concurrency: number, + timeoutMs: number +): Promise { + const latencies = new Array(totalRequests) + let index = 0 + + const workers = Array.from({ length: Math.max(1, concurrency) }, async () => { + while (true) { + const current = index + index++ + if (current >= totalRequests) return + latencies[current] = await requestLatencyMs(url, timeoutMs) + } + }) + + await Promise.all(workers) + return latencies +} + +async function copyIfExists(fromPath: string, toPath: string) { + try { + await access(fromPath) + await copyFile(fromPath, toPath) + } catch { + // Ignore missing optional traces. + } +} + +function printFullResults(results: FullRunResult[]) { + console.log('\nFULL APP-RENDER BENCHMARKS (end-to-end request path)') + + for (const result of results) { + console.log(`\nMode: ${result.mode}`) + + for (const route of new Set( + result.routeResults.map((entry) => entry.route) + )) { + console.log(` Route: ${route}`) + const routeEntries = result.routeResults.filter( + (entry) => entry.route === route + ) + for (const entry of routeEntries) { + console.log( + ` ${entry.phase} requests=${entry.requests} concurrency=${entry.concurrency}` + ) + console.log( + ` throughput=${entry.throughputRps.toFixed(2)} req/s median=${entry.latency.median.toFixed(2)}ms p95=${entry.latency.p95.toFixed(2)}ms` + ) + } + } + } + + if (results.length === 2) { + const web = results.find((result) => result.mode === 'web') + const node = results.find((result) => result.mode === 'node') + if (web && node) { + console.log('\nComparison (node vs web)') + + const joinKeys = new Set( + web.routeResults.map((entry) => `${entry.route}|${entry.phase}`) + ) + + for (const key of joinKeys) { + const [route, phase] = key.split('|') as [ + string, + 'single-client' | 'under-load', + ] + const webEntry = web.routeResults.find( + (entry) => entry.route === route && entry.phase === phase + ) + const nodeEntry = node.routeResults.find( + (entry) => entry.route === route && entry.phase === phase + ) + + if (!webEntry || !nodeEntry) continue + + const throughputDelta = + ((nodeEntry.throughputRps - webEntry.throughputRps) / + webEntry.throughputRps) * + 100 + const p95Delta = + ((webEntry.latency.p95 - nodeEntry.latency.p95) / + webEntry.latency.p95) * + 100 + + console.log(` ${route} (${phase})`) + console.log( + ` throughput delta: ${throughputDelta >= 0 ? '+' : ''}${throughputDelta.toFixed(2)}%` + ) + console.log( + ` p95 latency delta: ${p95Delta >= 0 ? '+' : ''}${p95Delta.toFixed(2)}% (positive is better)` + ) + } + } + } +} + +async function runFullModeBenchmark( + options: CliOptions, + mode: 'web' | 'node' +): Promise { + const nextConfigPath = resolve(options.appDir, 'next.config.js') + const originalConfig = await readFile(nextConfigPath, 'utf8') + + let server: ReturnType | null = null + const routeResults: FullRoutePhaseResult[] = [] + const modeArtifactDir = resolve(options.artifactDir, mode) + + await mkdir(modeArtifactDir, { recursive: true }) + + try { + await writeFile(nextConfigPath, configForMode(mode)) + + if (options.buildFull) { + console.log(`\n[full/${mode}] building app fixture...`) + await runCommand('node', [NEXT_BIN, 'build'], options.appDir, { + ...process.env, + NEXT_TELEMETRY_DISABLED: '1', + }) + if (options.captureNextTrace) { + await copyIfExists( + resolve(options.appDir, '.next/trace-build'), + resolve(modeArtifactDir, 'next-trace-build.log') + ) + } + } + + console.log(`[full/${mode}] starting minimal server...`) + + const serverArgs: string[] = [] + if (options.captureCpu) { + serverArgs.push( + '--cpu-prof', + `--cpu-prof-dir=${modeArtifactDir}`, + `--cpu-prof-name=${mode}.cpuprofile` + ) + } + if (options.captureHeap) { + serverArgs.push( + '--heap-prof', + `--heap-prof-dir=${modeArtifactDir}`, + `--heap-prof-name=${mode}.heapprofile` + ) + } + if (options.captureTrace) { + serverArgs.push( + '--trace-events-enabled', + `--trace-event-categories=${options.traceCategories}`, + `--trace-event-file-pattern=${resolve(modeArtifactDir, `${mode}-trace-\${pid}.json`)}` + ) + } + serverArgs.push(MINIMAL_SERVER) + + server = spawn('node', serverArgs, { + cwd: options.appDir, + env: { + ...process.env, + NODE_ENV: 'production', + NEXT_TELEMETRY_DISABLED: '1', + PORT: String(options.port), + }, + stdio: 'ignore', + }) + + await waitForServerReady( + `http://127.0.0.1:${options.port}${options.routes[0]}`, + options.timeoutMs + ) + + for (const route of options.routes) { + const url = `http://127.0.0.1:${options.port}${route}` + + console.log( + `[full/${mode}] route ${route}: warmup ${options.warmupRequests}` + ) + await runSerialRequests(url, options.warmupRequests, options.timeoutMs) + + console.log(`[full/${mode}] route ${route}: single-client phase`) + const serialStart = performance.now() + const serialLatencies = await runSerialRequests( + url, + options.serialRequests, + options.timeoutMs + ) + const serialDurationMs = performance.now() - serialStart + routeResults.push({ + mode, + route, + phase: 'single-client', + requests: options.serialRequests, + concurrency: 1, + throughputRps: options.serialRequests / (serialDurationMs / 1000), + latency: computeStats(serialLatencies), + }) + + console.log(`[full/${mode}] route ${route}: under-load phase`) + const loadStart = performance.now() + const loadLatencies = await runConcurrentRequests( + url, + options.loadRequests, + options.loadConcurrency, + options.timeoutMs + ) + const loadDurationMs = performance.now() - loadStart + routeResults.push({ + mode, + route, + phase: 'under-load', + requests: options.loadRequests, + concurrency: options.loadConcurrency, + throughputRps: options.loadRequests / (loadDurationMs / 1000), + latency: computeStats(loadLatencies), + }) + } + + return { mode, routeResults } + } finally { + if (server) { + const tryKill = async (signal: NodeJS.Signals, timeoutMs: number) => { + server!.kill(signal) + const didExit = await Promise.race([ + once(server!, 'exit') + .then(() => true) + .catch(() => true), + sleep(timeoutMs).then(() => false), + ]) + return didExit + } + + if (!(await tryKill('SIGINT', 3000))) { + if (!(await tryKill('SIGTERM', 3000))) { + server.kill('SIGKILL') + await once(server, 'exit').catch(() => undefined) + } + } + } + + if (options.captureNextTrace) { + await copyIfExists( + resolve(options.appDir, '.next/trace'), + resolve(modeArtifactDir, 'next-runtime-trace.log') + ) + } + + await writeFile(nextConfigPath, originalConfig) + } +} + +async function runFullBenchmarks( + options: CliOptions +): Promise { + await ensureNextBuilt() + await mkdir(options.artifactDir, { recursive: true }) + + const modes: Array<'web' | 'node'> = + options.streamMode === 'both' ? ['web', 'node'] : [options.streamMode] + + const results: FullRunResult[] = [] + for (const mode of modes) { + results.push(await runFullModeBenchmark(options, mode)) + } + return results +} + +async function main() { + const options = parseCli() + + console.log('Render pipeline benchmark') + console.log(`scenario=${options.scenario}`) + + let microResults: BenchResult[] | undefined + let fullResults: FullRunResult[] | undefined + + if (options.scenario === 'micro' || options.scenario === 'all') { + console.log( + `\nRunning micro benchmarks: iterations=${options.iterations} warmup=${options.warmup}` + ) + console.log( + `html=${options.htmlChunks}x${options.htmlChunkBytes} flight=${options.flightChunks}x${options.flightChunkBytes} binaryFlight=${options.binaryFlight}` + ) + + microResults = await runMicroBenchmarks(options) + printMicroResults(microResults) + } + + if (options.scenario === 'full' || options.scenario === 'all') { + console.log( + `\nRunning full benchmark: appDir=${options.appDir} streamMode=${options.streamMode}` + ) + console.log(`routes=${options.routes.join(', ')}`) + console.log(`artifacts=${options.artifactDir}`) + + fullResults = await runFullBenchmarks(options) + printFullResults(fullResults) + } + + if (options.jsonOut) { + const outputPath = resolve(process.cwd(), options.jsonOut) + await writeFile( + outputPath, + JSON.stringify( + { + options, + microResults, + fullResults, + generatedAt: new Date().toISOString(), + node: process.version, + }, + null, + 2 + ) + ) + console.log(`\nWrote JSON report: ${outputPath}`) + } +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/eslint.cli.config.mjs b/eslint.cli.config.mjs index 6f7319621cd5..85729ea54796 100644 --- a/eslint.cli.config.mjs +++ b/eslint.cli.config.mjs @@ -10,7 +10,13 @@ export default defineConfig([ // This override adds type-checked rules. // Linting with type-checked rules is very slow and needs a lot of memory, // so we exclude non-essential files. - ignores: ['examples/**/*', 'test/**/*', '**/*.d.ts', 'turbopack/**/*'], + ignores: [ + 'bench/**/*', + 'examples/**/*', + 'test/**/*', + '**/*.d.ts', + 'turbopack/**/*', + ], languageOptions: { parserOptions: { project: true, diff --git a/package.json b/package.json index 3d8f0c921f1f..cb708e6d3887 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "build": "turbo run build --remote-cache-timeout 60 --summarize true", "lerna": "lerna", "dev": "turbo run dev --parallel --filter=\"!@next/bundle-analyzer-ui\"", + "bench:render-pipeline": "tsx bench/render-pipeline/benchmark.ts", + "bench:render-pipeline:analyze": "tsx bench/render-pipeline/analyze-profiles.ts", "pack-next": "tsx scripts/pack-next.ts", "test-types": "tsc", "test-unit": "jest test/unit/ packages/next/ packages/font",