diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 01c897981..209a7c603 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -122,9 +122,14 @@ export { type ExtractedFrames, type ExtractionOptions, type ExtractionResult, + type ExtractionPhaseBreakdown, } from "./services/videoFrameExtractor.js"; -export { createVideoFrameInjector } from "./services/videoFrameInjector.js"; +export { + createVideoFrameInjector, + createEmptyInjectorCacheStats, + type InjectorCacheStats, +} from "./services/videoFrameInjector.js"; export { parseAudioElements, processCompositionAudio } from "./services/audioMixer.js"; export type { AudioElement, AudioTrack, MixResult } from "./services/audioMixer.types.js"; diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index d41ac47fa..5db766126 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -198,6 +198,46 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { // (116/300); on the actual reporter's ScreenCaptureKit clip, 18–44% across // segments. <10% threshold leaves margin across ffmpeg versions without // letting a regression slip through. + it("populates phaseBreakdown with timings for resolve, probe, VFR preflight, and extract", async () => { + const outputDir = join(FIXTURE_DIR, "out-phase-breakdown"); + mkdirSync(outputDir, { recursive: true }); + + const video: VideoElement = { + id: "vbreak", + src: VFR_FIXTURE, + start: 0, + end: 2, + mediaStart: 0, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir, + }); + + expect(result.errors).toEqual([]); + const pb = result.phaseBreakdown; + // Each phase ran; non-negative is the only universal invariant since + // resolveMs can round to 0 on fast local paths. + expect(pb.resolveMs).toBeGreaterThanOrEqual(0); + expect(pb.probeMs).toBeGreaterThanOrEqual(0); + expect(pb.hdrPreflightMs).toBeGreaterThanOrEqual(0); + expect(pb.vfrPreflightMs).toBeGreaterThanOrEqual(0); + expect(pb.extractMs).toBeGreaterThan(0); + // The VFR fixture is synthesized with irregular timestamps, so the VFR + // preflight must have actually run and been counted. + expect(pb.vfrPreflightCount).toBe(1); + expect(pb.vfrPreflightMs).toBeGreaterThan(0); + // No HDR source, so the HDR preflight is skipped entirely. + expect(pb.hdrPreflightCount).toBe(0); + expect(pb.hdrPreflightMs).toBe(0); + // Phases are bounded by total wall time (allow 50ms slack for timer + // resolution + overhead between the Date.now() samples). + const phaseSum = pb.resolveMs + pb.probeMs + pb.vfrPreflightMs + pb.extractMs; + expect(phaseSum).toBeLessThanOrEqual(result.durationMs + 50); + }, 60_000); + it("produces the full frame count and no duplicate-frame runs on the full VFR file", async () => { const outputDir = join(FIXTURE_DIR, "out-full"); mkdirSync(outputDir, { recursive: true }); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index a78ac6408..0b4f91f8c 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -42,12 +42,29 @@ export interface ExtractionOptions { format?: "jpg" | "png"; } +export interface ExtractionPhaseBreakdown { + /** Resolve relative paths + download remote inputs. */ + resolveMs: number; + /** ffprobe passes across all inputs (color-space probe + VFR metadata). */ + probeMs: number; + /** Sum of per-input `convertSdrToHdr` re-encodes. */ + hdrPreflightMs: number; + /** Sum of per-input `convertVfrToCfr` re-encodes. */ + vfrPreflightMs: number; + /** Phase 3 — parallel frame extraction (wall time, not summed). */ + extractMs: number; + /** Counts of inputs hitting each preflight, for ratio analysis. */ + hdrPreflightCount: number; + vfrPreflightCount: number; +} + export interface ExtractionResult { success: boolean; extracted: ExtractedFrames[]; errors: Array<{ videoId: string; error: string }>; totalFramesExtracted: number; durationMs: number; + phaseBreakdown: ExtractionPhaseBreakdown; } export function parseVideoElements(html: string): VideoElement[] { @@ -364,8 +381,18 @@ export async function extractAllVideoFrames( const extracted: ExtractedFrames[] = []; const errors: Array<{ videoId: string; error: string }> = []; let totalFramesExtracted = 0; + const phaseBreakdown: ExtractionPhaseBreakdown = { + resolveMs: 0, + probeMs: 0, + hdrPreflightMs: 0, + vfrPreflightMs: 0, + extractMs: 0, + hdrPreflightCount: 0, + vfrPreflightCount: 0, + }; // Phase 1: Resolve paths and download remote videos + const resolveStart = Date.now(); const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = []; for (const video of videos) { if (signal?.aborted) break; @@ -392,14 +419,17 @@ export async function extractAllVideoFrames( errors.push({ videoId: video.id, error: err instanceof Error ? err.message : String(err) }); } } + phaseBreakdown.resolveMs = Date.now() - resolveStart; // Phase 2: Probe color spaces and normalize if mixed HDR/SDR + const probeStart = Date.now(); const videoColorSpaces = await Promise.all( resolvedVideos.map(async ({ videoPath }) => { const metadata = await extractVideoMetadata(videoPath); return metadata.colorSpace; }), ); + phaseBreakdown.probeMs += Date.now() - probeStart; const hasAnyHdr = videoColorSpaces.some(isHdrColorSpaceUtil); if (hasAnyHdr) { @@ -414,14 +444,18 @@ export async function extractAllVideoFrames( const entry = resolvedVideos[i]; if (!entry) continue; const convertedPath = join(convertDir, `${entry.video.id}_hdr.mp4`); + const hdrStart = Date.now(); try { await convertSdrToHdr(entry.videoPath, convertedPath, signal, config); entry.videoPath = convertedPath; + phaseBreakdown.hdrPreflightCount += 1; } catch (err) { errors.push({ videoId: entry.video.id, error: `SDR→HDR conversion failed: ${err instanceof Error ? err.message : String(err)}`, }); + } finally { + phaseBreakdown.hdrPreflightMs += Date.now() - hdrStart; } } } @@ -434,7 +468,9 @@ export async function extractAllVideoFrames( if (signal?.aborted) break; const entry = resolvedVideos[i]; if (!entry) continue; + const vfrProbeStart = Date.now(); const metadata = await extractVideoMetadata(entry.videoPath); + phaseBreakdown.probeMs += Date.now() - vfrProbeStart; if (!metadata.isVFR) continue; let segDuration = entry.video.end - entry.video.start; @@ -445,6 +481,7 @@ export async function extractAllVideoFrames( mkdirSync(vfrNormDir, { recursive: true }); const normalizedPath = join(vfrNormDir, `${entry.video.id}_cfr.mp4`); + const vfrStart = Date.now(); try { await convertVfrToCfr( entry.videoPath, @@ -460,15 +497,19 @@ export async function extractAllVideoFrames( // extraction must seek from 0, not the original mediaStart. Shallow-copy // to avoid mutating the caller's VideoElement. entry.video = { ...entry.video, mediaStart: 0 }; + phaseBreakdown.vfrPreflightCount += 1; } catch (err) { errors.push({ videoId: entry.video.id, error: err instanceof Error ? err.message : String(err), }); + } finally { + phaseBreakdown.vfrPreflightMs += Date.now() - vfrStart; } } // Phase 3: Extract frames (parallel) + const extractStart = Date.now(); const results = await Promise.all( resolvedVideos.map(async ({ video, videoPath }) => { if (signal?.aborted) { @@ -508,6 +549,8 @@ export async function extractAllVideoFrames( }), ); + phaseBreakdown.extractMs = Date.now() - extractStart; + // Collect results and errors for (const item of results) { if ("error" in item && item.error) { @@ -524,6 +567,7 @@ export async function extractAllVideoFrames( errors, totalFramesExtracted, durationMs: Date.now() - startTime, + phaseBreakdown, }; } diff --git a/packages/engine/src/services/videoFrameInjector.test.ts b/packages/engine/src/services/videoFrameInjector.test.ts new file mode 100644 index 000000000..6b9dabf6d --- /dev/null +++ b/packages/engine/src/services/videoFrameInjector.test.ts @@ -0,0 +1,97 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + __createFrameDataUriCacheForTests as createCache, + createEmptyInjectorCacheStats, +} from "./videoFrameInjector.js"; + +// The injector's frame-dataURI LRU is what the performance review flags as +// bottleneck #8 (disk reads per frame). These tests validate the cumulative +// counters that the producer exposes via `RenderPerfSummary.injectorStats` +// so future PRs — the extraction-cache work in particular — have a trustable +// hit-rate signal to compare against. +describe("InjectorCacheStats via frame-dataURI LRU", () => { + const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-injector-stats-")); + const FRAME_A = join(FIXTURE_DIR, "a.jpg"); + const FRAME_B = join(FIXTURE_DIR, "b.jpg"); + const FRAME_C = join(FIXTURE_DIR, "c.jpg"); + + beforeAll(() => { + // Content is irrelevant — the cache keys on path and base64-encodes the bytes. + writeFileSync(FRAME_A, Buffer.from([0xff, 0xd8, 0xff, 0xe0])); + writeFileSync(FRAME_B, Buffer.from([0xff, 0xd8, 0xff, 0xe1])); + writeFileSync(FRAME_C, Buffer.from([0xff, 0xd8, 0xff, 0xe2])); + }); + + afterAll(() => { + rmSync(FIXTURE_DIR, { recursive: true, force: true }); + }); + + it("counts a miss on first read and a hit on re-read", async () => { + const stats = createEmptyInjectorCacheStats(); + const cache = createCache(32, stats); + + await cache.get(FRAME_A); + await cache.get(FRAME_A); + + expect(stats.misses).toBe(1); + expect(stats.hits).toBe(1); + expect(stats.inFlightCoalesced).toBe(0); + expect(stats.peakEntries).toBe(1); + }); + + it("coalesces concurrent reads of the same path into one miss", async () => { + const stats = createEmptyInjectorCacheStats(); + const cache = createCache(32, stats); + + const [a, b, c] = await Promise.all([ + cache.get(FRAME_B), + cache.get(FRAME_B), + cache.get(FRAME_B), + ]); + + expect(a).toBe(b); + expect(b).toBe(c); + // Exactly one disk read is issued, and the other two concurrent requests + // are tallied as coalesced. + expect(stats.misses).toBe(1); + expect(stats.inFlightCoalesced).toBe(2); + expect(stats.hits).toBe(0); + }); + + it("tracks peakEntries as the LRU fills", async () => { + const stats = createEmptyInjectorCacheStats(); + const cache = createCache(32, stats); + + await cache.get(FRAME_A); + await cache.get(FRAME_B); + await cache.get(FRAME_C); + + expect(stats.peakEntries).toBe(3); + }); + + it("evicts under pressure without inflating peakEntries past the limit", async () => { + const stats = createEmptyInjectorCacheStats(); + // Tiny limit forces eviction after 2 inserts. + const cache = createCache(2, stats); + + await cache.get(FRAME_A); + await cache.get(FRAME_B); + await cache.get(FRAME_C); + + expect(stats.peakEntries).toBe(2); + // FRAME_A was evicted when FRAME_C was inserted — re-reading it is a miss. + await cache.get(FRAME_A); + expect(stats.misses).toBe(4); + }); + + it("leaves counters untouched when no stats object is provided", async () => { + // Regression guard: the sentinel check in the cache (stats != null) is the + // only thing keeping existing callers' performance unchanged. This test + // simply exercises the path and confirms nothing throws. + const cache = createCache(32); + await expect(cache.get(FRAME_A)).resolves.toMatch(/^data:image\/jpeg;base64,/); + }); +}); diff --git a/packages/engine/src/services/videoFrameInjector.ts b/packages/engine/src/services/videoFrameInjector.ts index 97df1638e..a23e913c5 100644 --- a/packages/engine/src/services/videoFrameInjector.ts +++ b/packages/engine/src/services/videoFrameInjector.ts @@ -14,7 +14,37 @@ import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotSe import { type BeforeCaptureHook } from "./frameCapture.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; -function createFrameDataUriCache(cacheLimit: number) { +/** + * Per-render metrics for the frame-dataURI LRU. Populated by the injector + * when the caller passes a stats object to `createVideoFrameInjector`. + * + * Designed to be read once at end-of-render; values are cumulative counters + * plus a peak watermark. + */ +export interface InjectorCacheStats { + /** Cache returned a previously-loaded data URI without touching disk. */ + hits: number; + /** Cache miss — a new disk read was issued. */ + misses: number; + /** Concurrent request for an in-flight read was coalesced onto the pending promise. */ + inFlightCoalesced: number; + /** Max size the LRU reached during the render (counts of data-URI entries). */ + peakEntries: number; +} + +export function createEmptyInjectorCacheStats(): InjectorCacheStats { + return { hits: 0, misses: 0, inFlightCoalesced: 0, peakEntries: 0 }; +} + +/** + * Exported for unit tests only — not part of the package's public API. + * Used to validate the LRU / stats behavior without spinning up a Chrome page. + */ +export function __createFrameDataUriCacheForTests(cacheLimit: number, stats?: InjectorCacheStats) { + return createFrameDataUriCache(cacheLimit, stats); +} + +function createFrameDataUriCache(cacheLimit: number, stats?: InjectorCacheStats) { const cache = new Map(); const inFlight = new Map>(); @@ -29,21 +59,27 @@ function createFrameDataUriCache(cacheLimit: number) { cache.delete(oldestKey); } } + if (stats && cache.size > stats.peakEntries) { + stats.peakEntries = cache.size; + } return dataUri; } async function get(framePath: string): Promise { const cached = cache.get(framePath); if (cached) { + if (stats) stats.hits += 1; remember(framePath, cached); return cached; } const existing = inFlight.get(framePath); if (existing) { + if (stats) stats.inFlightCoalesced += 1; return existing; } + if (stats) stats.misses += 1; const pending = fs .readFile(framePath) .then((frameData) => { @@ -68,6 +104,7 @@ function createFrameDataUriCache(cacheLimit: number) { export function createVideoFrameInjector( frameLookup: FrameLookupTable | null, config?: Partial>, + stats?: InjectorCacheStats, ): BeforeCaptureHook | null { if (!frameLookup) return null; @@ -75,7 +112,7 @@ export function createVideoFrameInjector( 32, config?.frameDataUriCacheLimit ?? DEFAULT_CONFIG.frameDataUriCacheLimit, ); - const frameCache = createFrameDataUriCache(cacheLimit); + const frameCache = createFrameDataUriCache(cacheLimit, stats); const lastInjectedFrameByVideo = new Map(); return async (page: Page, time: number) => { diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 3a886d38a..003df336f 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -22,6 +22,7 @@ import { writeFileSync, copyFileSync, appendFileSync, + statSync, } from "fs"; import { parseHTML } from "linkedom"; import { @@ -43,6 +44,9 @@ import { type CaptureOptions, type CaptureSession, createVideoFrameInjector, + createEmptyInjectorCacheStats, + type InjectorCacheStats, + type ExtractionPhaseBreakdown, encodeFramesFromDir, encodeFramesChunkedConcat, muxVideoWithAudio, @@ -116,6 +120,40 @@ async function safeCleanup( } } +/** + * Recursively sum file sizes under `root`. Used at end-of-render to report a + * lower bound on peak `workDir` usage (the extracted-frame dir, captured-frame + * dir, and intermediate audio/video artifacts all coexist at this point, just + * before cleanup). Silently tolerates missing/unreadable paths — the caller + * only uses this for telemetry, not correctness. + */ +function sumDirBytes(root: string): number { + let total = 0; + const walk = (dir: string): void => { + let entries: string[]; + try { + entries = readdirSync(dir); + } catch { + return; + } + for (const name of entries) { + const full = join(dir, name); + try { + const st = statSync(full); + if (st.isDirectory()) { + walk(full); + } else if (st.isFile()) { + total += st.size; + } + } catch { + // Missing between readdir and stat — skip. + } + } + }; + walk(root); + return total; +} + /** * Cache of the maximum 1-based frame index present in each pre-extracted frame * directory (e.g. `frame_0001.png … frame_0150.png` → 150). The directory is @@ -211,6 +249,20 @@ export interface RenderPerfSummary { captureAvgMs?: number; capturePeakMs?: number; hdrDiagnostics?: HdrDiagnostics; + /** + * Phase-level split of the extractor wall time (resolve / probe / HDR + * preflight / VFR preflight / Phase 3 extract). Absent when the + * composition had no video inputs. + */ + videoExtractBreakdown?: ExtractionPhaseBreakdown; + /** + * Frame-data-URI LRU counters for the sequential-capture path. Parallel + * workers run in separate processes and don't contribute here; absent + * when parallel was the only path taken. + */ + injectorStats?: InjectorCacheStats; + /** Peak bytes under `workDir` (extracted frames + captured frames + transient dirs). */ + tmpPeakBytes?: number; } export interface HdrDiagnostics { @@ -993,6 +1045,12 @@ export async function executeRenderJob( let frameLookup: FrameLookupTable | null = null; const compiledDir = join(workDir, "compiled"); let extractionResult: Awaited> | null = null; + // Shared LRU stats for sequential-capture injector instances. Parallel + // workers live in separate processes and cannot mutate this object, so + // only sequential renders populate it (and only those render types + // contribute to `perfSummary.injectorStats`). + const injectorCacheStats: InjectorCacheStats = createEmptyInjectorCacheStats(); + let sequentialInjectorUsed = false; // Probe ORIGINAL color spaces before extraction (which may convert SDR→HDR). // This is needed to identify which videos are natively HDR vs converted-SDR @@ -1239,11 +1297,13 @@ export async function executeRenderJob( // readiness wait for these IDs; otherwise the render hangs 45s and // throws "video metadata not ready" even though we never asked the // browser to decode the video. + const domInjector = createVideoFrameInjector(frameLookup, cfg, injectorCacheStats); + if (domInjector) sequentialInjectorUsed = true; const domSession = await createCaptureSession( fileServer.url, framesDir, { ...captureOptions, skipReadinessVideoIds: Array.from(nativeHdrVideoIds) }, - createVideoFrameInjector(frameLookup), + domInjector, cfg, ); await initializeSession(domSession); @@ -2023,7 +2083,8 @@ export async function executeRenderJob( } else { // Sequential capture → streaming encode - const videoInjector = createVideoFrameInjector(frameLookup); + const videoInjector = createVideoFrameInjector(frameLookup, cfg, injectorCacheStats); + if (videoInjector) sequentialInjectorUsed = true; const session = probeSession ?? (await createCaptureSession( @@ -2125,7 +2186,8 @@ export async function executeRenderJob( } else { // Sequential capture - const videoInjector = createVideoFrameInjector(frameLookup); + const videoInjector = createVideoFrameInjector(frameLookup, cfg, injectorCacheStats); + if (videoInjector) sequentialInjectorUsed = true; const session = probeSession ?? (await createCaptureSession( @@ -2253,6 +2315,13 @@ export async function executeRenderJob( perfStages.assembleMs = Date.now() - stage6Start; + // Sample workDir size just before cleanup — by this point both the + // extracted-frame dir and captured-frame dir (plus intermediate audio + + // video-only artifacts) are all on disk, so this is close to the peak + // usage footprint for the render. The doc's validation plan lists this + // as one of the three reference metrics. + const tmpPeakBytes = sumDirBytes(workDir); + // ── Complete ───────────────────────────────────────────────────────── job.outputPath = outputPath; updateJobStatus(job, "complete", "Render complete", 100, onProgress); @@ -2279,6 +2348,9 @@ export async function executeRenderJob( : undefined, captureAvgMs: totalFrames > 0 ? Math.round((perfStages.captureMs ?? 0) / totalFrames) : undefined, + videoExtractBreakdown: extractionResult?.phaseBreakdown, + injectorStats: sequentialInjectorUsed ? { ...injectorCacheStats } : undefined, + tmpPeakBytes, }; job.perfSummary = perfSummary; if (job.config.debug) { diff --git a/packages/producer/src/services/videoFrameInjector.ts b/packages/producer/src/services/videoFrameInjector.ts index df9c36786..8249a9c6c 100644 --- a/packages/producer/src/services/videoFrameInjector.ts +++ b/packages/producer/src/services/videoFrameInjector.ts @@ -2,4 +2,8 @@ * Re-exported from @hyperframes/engine. * @see engine/src/services/videoFrameInjector.ts for implementation. */ -export { createVideoFrameInjector } from "@hyperframes/engine"; +export { + createVideoFrameInjector, + createEmptyInjectorCacheStats, + type InjectorCacheStats, +} from "@hyperframes/engine";