perf(engine): webp extraction format + injector MIME routing

packages/engine/src/services/extractionCache.test.ts

@@ -36,7 +36,9 @@ describe("computeExtractionCacheKey", () => {
 
   it("prefixes the key with the schema version so a future on-disk format change cannot collide", () => {
     const key = computeExtractionCacheKey(base);
-    expect(key.startsWith("v1-")).toBe(true);
+    // Shape: `v<N>-<hex>` — we don't pin N here so schema bumps (e.g. v1→v2
+    // for the WebP change) stay local to the extractionCache module.
+    expect(key).toMatch(/^v\d+-[0-9a-f]{32}$/);
   });
 
   it.each([

packages/engine/src/services/extractionCache.ts

@@ -36,11 +36,14 @@ import { join } from "path";
 
 /**
  * Schema version embedded in every cache key. Bump whenever the on-disk
- * format of extracted frames changes in a way that breaks older entries
- * (e.g. the upcoming WebP-unified-format PR changes the frame extension
- * and MIME handling, so it will bump this to 2).
+ * format of extracted frames changes in a way that breaks older entries.
+ *
+ * - v1: jpg / png output from the pre-WebP extractor.
+ * - v2: adds webp as an extraction format. A v2 key for webp content can
+ *   never collide with a v1 jpg/png key, so both generations can coexist
+ *   under the same cache root during migration without cross-serving.
  */
-const CACHE_SCHEMA_VERSION = 1;
+const CACHE_SCHEMA_VERSION = 2;
 
 /**
  * Sentinel filename written inside each completed cache entry directory.
@@ -62,7 +65,7 @@ export interface ExtractionCacheKeyInputs {
   /** Target fps for extracted frames. */
   fps: number;
   /** Extracted frame format ("jpg" or "png"). */
-  format: "jpg" | "png";
+  format: "jpg" | "png" | "webp";
 }
 
 /**
@@ -143,7 +146,7 @@ export interface CacheHit {
 export function lookupCacheEntry(
   cacheRoot: string,
   key: string,
-  format: "jpg" | "png",
+  format: "jpg" | "png" | "webp",
 ): CacheHit | null {
   const { dir, sentinel } = resolveCacheEntryPaths(cacheRoot, key);
   if (!existsSync(sentinel)) return null;

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -495,6 +495,91 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames with extraction cache", () =
     expect(result2.extracted[0]?.totalFrames).toBe(result1.extracted[0]?.totalFrames);
   }, 60_000);
 
+  it("writes webp frames and reuses them on a second call", async () => {
+    const cacheRoot = join(FIXTURE_DIR, "cache-webp");
+    mkdirSync(cacheRoot, { recursive: true });
+
+    const video: VideoElement = {
+      id: "vid",
+      src: SOURCE,
+      start: 0,
+      end: 1,
+      mediaStart: 0,
+      hasAudio: false,
+    };
+
+    const out1 = join(FIXTURE_DIR, "webp-1");
+    mkdirSync(out1, { recursive: true });
+    const r1 = await extractAllVideoFrames(
+      [video],
+      FIXTURE_DIR,
+      { fps: 30, outputDir: out1, format: "webp" },
+      undefined,
+      { extractCacheDir: cacheRoot },
+    );
+    expect(r1.errors).toEqual([]);
+    expect(r1.phaseBreakdown.cacheMisses).toBe(1);
+    // Cache hit on second call confirms the on-disk files are valid webp
+    // (lookupCacheEntry filters by extension); assert the file list directly
+    // too so a broken libwebp encode would surface as an empty dir.
+    const cacheDir = r1.extracted[0]?.outputDir;
+    expect(cacheDir).toBeDefined();
+    const frames = readdirSync(cacheDir!).filter((f) => f.endsWith(".webp"));
+    expect(frames.length).toBeGreaterThan(0);
+
+    const out2 = join(FIXTURE_DIR, "webp-2");
+    mkdirSync(out2, { recursive: true });
+    const r2 = await extractAllVideoFrames(
+      [video],
+      FIXTURE_DIR,
+      { fps: 30, outputDir: out2, format: "webp" },
+      undefined,
+      { extractCacheDir: cacheRoot },
+    );
+    expect(r2.phaseBreakdown.cacheHits).toBe(1);
+    expect(r2.phaseBreakdown.cacheMisses).toBe(0);
+    expect(r2.extracted[0]?.totalFrames).toBe(r1.extracted[0]?.totalFrames);
+  }, 60_000);
+
+  it("does not cross-serve a jpg cache entry as webp (format is part of the key)", async () => {
+    const cacheRoot = join(FIXTURE_DIR, "cache-cross-format");
+    mkdirSync(cacheRoot, { recursive: true });
+
+    const video: VideoElement = {
+      id: "vid",
+      src: SOURCE,
+      start: 0,
+      end: 1,
+      mediaStart: 0,
+      hasAudio: false,
+    };
+
+    // Populate the cache with a jpg entry first.
+    const outJpg = join(FIXTURE_DIR, "cross-jpg");
+    mkdirSync(outJpg, { recursive: true });
+    const rJpg = await extractAllVideoFrames(
+      [video],
+      FIXTURE_DIR,
+      { fps: 30, outputDir: outJpg, format: "jpg" },
+      undefined,
+      { extractCacheDir: cacheRoot },
+    );
+    expect(rJpg.phaseBreakdown.cacheMisses).toBe(1);
+
+    // Same inputs but format=webp — must be a fresh miss (different key).
+    const outWebp = join(FIXTURE_DIR, "cross-webp");
+    mkdirSync(outWebp, { recursive: true });
+    const rWebp = await extractAllVideoFrames(
+      [video],
+      FIXTURE_DIR,
+      { fps: 30, outputDir: outWebp, format: "webp" },
+      undefined,
+      { extractCacheDir: cacheRoot },
+    );
+    expect(rWebp.phaseBreakdown.cacheHits).toBe(0);
+    expect(rWebp.phaseBreakdown.cacheMisses).toBe(1);
+  }, 60_000);
+
   it("misses again when fps changes (keyed on fps)", async () => {
     const cacheRoot = join(FIXTURE_DIR, "cache-fps");
     mkdirSync(cacheRoot, { recursive: true });

packages/engine/src/services/videoFrameExtractor.ts

@@ -53,7 +53,17 @@ export interface ExtractionOptions {
   fps: number;
   outputDir: string;
   quality?: number;
-  format?: "jpg" | "png";
+  /**
+   * On-disk frame format.
+   *
+   * - `"webp"` (recommended) — smaller files than jpg at equivalent quality,
+   *   handles alpha natively, decoded natively by Chrome. Default for the
+   *   producer's SDR extraction path.
+   * - `"jpg"` (legacy default) — opaque only, smallest for no-alpha content.
+   * - `"png"` — lossless, retained for external callers and alpha paths
+   *   that specifically want PNG semantics.
+   */
+  format?: "jpg" | "png" | "webp";
 }
 
 export interface ExtractionPhaseBreakdown {
@@ -220,8 +230,15 @@ export async function extractVideoFramesRange(
   vfFilters.push(`fps=${fps}`);
   args.push("-vf", vfFilters.join(","));
 
-  args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
-  if (format === "png") args.push("-compression_level", "6");
+  if (format === "webp") {
+    // libwebp: `-quality` is 0-100, higher = better (inverse of JPEG's -q:v).
+    // Lossy mode by default — near-lossless at quality=95 but ~5-10x smaller
+    // than PNG and typically smaller than a visually-equivalent JPEG.
+    args.push("-c:v", "libwebp", "-quality", String(quality), "-lossless", "0");
+  } else {
+    args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
+    if (format === "png") args.push("-compression_level", "6");
+  }
   args.push("-y", outputPattern);
 
   return new Promise((resolve, reject) => {

packages/engine/src/services/videoFrameInjector.test.ts

@@ -95,3 +95,37 @@ describe("InjectorCacheStats via frame-dataURI LRU", () => {
     await expect(cache.get(FRAME_A)).resolves.toMatch(/^data:image\/jpeg;base64,/);
   });
 });
+
+// The injector's MIME detection drives what Chrome does with the data URI —
+// a mis-tagged WebP frame decodes as a broken image. This block verifies the
+// mapping through the LRU's public surface (the only way MIME-tagged URIs
+// leave the module).
+describe("frame path → MIME type tagging", () => {
+  const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-injector-mime-"));
+  const WEBP = join(FIXTURE_DIR, "frame_00001.webp");
+  const PNG = join(FIXTURE_DIR, "frame_00001.png");
+  const JPG = join(FIXTURE_DIR, "frame_00001.jpg");
+  const UNKNOWN = join(FIXTURE_DIR, "frame_00001.xyz");
+
+  beforeAll(() => {
+    // Content isn't validated, only the extension drives MIME selection.
+    writeFileSync(WEBP, Buffer.from("WEBP"));
+    writeFileSync(PNG, Buffer.from([0x89, 0x50, 0x4e, 0x47]));
+    writeFileSync(JPG, Buffer.from([0xff, 0xd8, 0xff, 0xe0]));
+    writeFileSync(UNKNOWN, Buffer.from("xx"));
+  });
+
+  afterAll(() => {
+    rmSync(FIXTURE_DIR, { recursive: true, force: true });
+  });
+
+  it.each([
+    [".webp path", WEBP, /^data:image\/webp;base64,/],
+    [".png path", PNG, /^data:image\/png;base64,/],
+    [".jpg path", JPG, /^data:image\/jpeg;base64,/],
+    ["unknown extension falls back to jpeg", UNKNOWN, /^data:image\/jpeg;base64,/],
+  ])("tags %s correctly", async (_label, path, mimePattern) => {
+    const cache = createCache(32);
+    await expect(cache.get(path)).resolves.toMatch(mimePattern);
+  });
+});

packages/engine/src/services/videoFrameInjector.ts

@@ -36,6 +36,18 @@ export function createEmptyInjectorCacheStats(): InjectorCacheStats {
   return { hits: 0, misses: 0, inFlightCoalesced: 0, peakEntries: 0 };
 }
 
+/**
+ * Map a frame file path to the MIME type that should accompany its data URI.
+ * Chrome decodes webp, png, and jpeg natively inside `<img src="data:…">`.
+ * Unknown extensions default to JPEG — the legacy pre-WebP behavior — so
+ * callers that produce non-standard filenames don't regress.
+ */
+function mimeTypeForFramePath(framePath: string): string {
+  if (framePath.endsWith(".webp")) return "image/webp";
+  if (framePath.endsWith(".png")) return "image/png";
+  return "image/jpeg";
+}
+
 /**
  * Exported for unit tests only — not part of the package's public API.
  * Used to validate the LRU / stats behavior without spinning up a Chrome page.
@@ -83,7 +95,7 @@ function createFrameDataUriCache(cacheLimit: number, stats?: InjectorCacheStats)
     const pending = fs
       .readFile(framePath)
       .then((frameData) => {
-        const mimeType = framePath.endsWith(".png") ? "image/png" : "image/jpeg";
+        const mimeType = mimeTypeForFramePath(framePath);
         const dataUri = `data:${mimeType};base64,${frameData.toString("base64")}`;
         return remember(framePath, dataUri);
       })

packages/producer/src/services/renderOrchestrator.ts

@@ -1112,7 +1112,17 @@ export async function executeRenderJob(
       extractionResult = await extractAllVideoFrames(
         composition.videos,
         projectDir,
-        { fps: job.config.fps, outputDir: join(workDir, "video-frames") },
+        {
+          fps: job.config.fps,
+          outputDir: join(workDir, "video-frames"),
+          // WebP gives smaller files than JPEG at equivalent visual quality
+          // AND handles alpha natively — used by the producer as the single
+          // extraction format so we don't have a jpg/png split. HDR pixel
+          // paths still produce PNG on their own out-of-band codepath (see
+          // the HDR pre-extract block below); this setting only affects
+          // SDR frames that flow through the `<img>` injector.
+          format: "webp",
+        },
         abortSignal,
         // Forward extractCacheDir (when configured) so repeat renders of
         // the same source+window+fps+format pair skip Phase 3 entirely.