diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 01c897981..041418527 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -152,6 +152,7 @@ export { export { quantizeTimeToFrame, MEDIA_VISUAL_STYLE_PROPERTIES } from "@hyperframes/core"; export { + extractMediaMetadata, extractVideoMetadata, extractAudioMetadata, analyzeKeyframeIntervals, diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 3980e3a1c..480238880 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -9,7 +9,7 @@ import { spawn } from "child_process"; import { existsSync, mkdirSync, readdirSync, rmSync } from "fs"; import { join } from "path"; import { parseHTML } from "linkedom"; -import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js"; +import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, @@ -158,7 +158,7 @@ export async function extractVideoFramesRange( const videoOutputDir = join(outputDir, videoId); if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true }); - const metadata = await extractVideoMetadata(videoPath); + const metadata = await extractMediaMetadata(videoPath); const framePattern = `frame_%05d.${format}`; const outputPattern = join(videoOutputDir, framePattern); @@ -407,7 +407,7 @@ export async function extractAllVideoFrames( // Phase 2: Probe color spaces and normalize if mixed HDR/SDR const videoColorSpaces = await Promise.all( resolvedVideos.map(async ({ videoPath }) => { - const metadata = await extractVideoMetadata(videoPath); + const metadata = await extractMediaMetadata(videoPath); return metadata.colorSpace; }), ); @@ -453,7 +453,7 @@ export async function extractAllVideoFrames( if (signal?.aborted) break; const entry = resolvedVideos[i]; if (!entry) continue; - const metadata = await extractVideoMetadata(entry.videoPath); + const metadata = await extractMediaMetadata(entry.videoPath); if (!metadata.isVFR) continue; let segDuration = entry.video.end - entry.video.start; @@ -499,7 +499,7 @@ export async function extractAllVideoFrames( // Fallback: if no data-duration/data-end was specified (end is Infinity or 0), // probe the actual video file to get its natural duration. if (!Number.isFinite(videoDuration) || videoDuration <= 0) { - const metadata = await extractVideoMetadata(videoPath); + const metadata = await extractMediaMetadata(videoPath); const sourceDuration = metadata.durationSeconds - video.mediaStart; videoDuration = sourceDuration > 0 ? sourceDuration : metadata.durationSeconds; video.end = video.start + videoDuration; diff --git a/packages/engine/src/utils/ffprobe.test.ts b/packages/engine/src/utils/ffprobe.test.ts index 20ebd5887..7d3a66781 100644 --- a/packages/engine/src/utils/ffprobe.test.ts +++ b/packages/engine/src/utils/ffprobe.test.ts @@ -1,7 +1,7 @@ import { readFileSync } from "fs"; import { resolve } from "path"; import { describe, expect, it } from "vitest"; -import { extractPngMetadataFromBuffer, extractVideoMetadata } from "./ffprobe.js"; +import { extractMediaMetadata, extractPngMetadataFromBuffer } from "./ffprobe.js"; function crc32(buf: Buffer): number { let crc = 0xffffffff; @@ -51,14 +51,14 @@ function buildMinimalPng(options?: { : buildPngWithChunks([ihdr, cicp, idat, iend]); } -describe("extractVideoMetadata", () => { +describe("extractMediaMetadata", () => { it("reads HDR PNG cICP metadata when ffprobe color fields are absent", async () => { const fixturePath = resolve( __dirname, "../../../producer/tests/hdr-regression/src/hdr-photo-pq.png", ); - const metadata = await extractVideoMetadata(fixturePath); + const metadata = await extractMediaMetadata(fixturePath); expect(metadata.colorSpace).toEqual({ colorPrimaries: "bt2020", diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index d72befc35..929405b52 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -210,7 +210,14 @@ function parseFrameRate(frameRateStr: string | undefined): number { return parseFloat(frameRateStr) || 0; } -export async function extractVideoMetadata(filePath: string): Promise { +/** + * Probe a media file (video, image, or container) and return normalized metadata. + * + * Despite the legacy name `extractVideoMetadata` (still exported as a + * deprecated alias below), this also handles still images such as PNG so it + * can be used uniformly for any visual asset the HDR pipeline encounters. + */ +export async function extractMediaMetadata(filePath: string): Promise { const cached = videoMetadataCache.get(filePath); if (cached) return cached; @@ -286,6 +293,14 @@ export async function extractVideoMetadata(filePath: string): Promise { const cached = audioMetadataCache.get(filePath); if (cached) return cached; diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index 96662df72..c7fdc7e44 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -17,7 +17,7 @@ import process from "node:process"; import { createRenderJob, executeRenderJob } from "./services/renderOrchestrator.js"; import { compileForRender } from "./services/htmlCompiler.js"; import { validateCompilation } from "./services/compilationTester.js"; -import { extractVideoMetadata } from "./utils/ffprobe.js"; +import { extractMediaMetadata } from "./utils/ffprobe.js"; import { buildRmsEnvelope, compareAudioEnvelopes } from "./utils/audioRegression.js"; // ── Types ──────────────────────────────────────────────────────────────────── @@ -625,7 +625,7 @@ async function runTestSuite( // Visual comparison (100 frames, 1 per 1% of video duration) logPretty("Comparing visual quality (100 checkpoints)...", "🔍"); - const videoMetadata = await extractVideoMetadata(renderedOutputPath); + const videoMetadata = await extractMediaMetadata(renderedOutputPath); const videoDuration = videoMetadata.durationSeconds; const visualCheckpoints: Array<{ time: number; psnr: number; passed: boolean }> = []; diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index 892c292de..5e1418135 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -23,7 +23,7 @@ import { rewriteAssetPaths, rewriteCssAssetUrls, } from "@hyperframes/core"; -import { extractVideoMetadata, extractAudioMetadata } from "../utils/ffprobe.js"; +import { extractMediaMetadata, extractAudioMetadata } from "../utils/ffprobe.js"; import { isPathInside, toExternalAssetKey } from "../utils/paths.js"; import { parseVideoElements, @@ -152,7 +152,7 @@ async function resolveMediaDuration( const metadata = tagName === "video" - ? await extractVideoMetadata(filePath) + ? await extractMediaMetadata(filePath) : await extractAudioMetadata(filePath); const fileDuration = metadata.durationSeconds; @@ -1033,7 +1033,7 @@ export async function compileForRender( if (isHttpUrl(video.src)) continue; const videoPath = resolve(projectDir, video.src); const reencode = `ffmpeg -i "${video.src}" -c:v libx264 -r 30 -g 30 -keyint_min 30 -movflags +faststart -c:a copy output.mp4`; - Promise.all([analyzeKeyframeIntervals(videoPath), extractVideoMetadata(videoPath)]) + Promise.all([analyzeKeyframeIntervals(videoPath), extractMediaMetadata(videoPath)]) .then(([analysis, metadata]) => { if (analysis.isProblematic) { console.warn( diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index c372539a0..54fcea821 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -42,6 +42,7 @@ import { prepareCaptureSessionForReuse, type CaptureOptions, type CaptureSession, + type BeforeCaptureHook, createVideoFrameInjector, encodeFramesFromDir, encodeFramesChunkedConcat, @@ -61,7 +62,7 @@ import { analyzeCompositionHdr, isHdrColorSpace, runFfmpeg, - extractVideoMetadata, + extractMediaMetadata, type VideoColorSpace, initTransparentBackground, captureAlphaPng, @@ -149,6 +150,34 @@ function getMaxFrameIndex(frameDir: string): number { return max; } +// Diagnostic helpers used by the HDR layered compositor when KEEP_TEMP=1 +// is set. They are pure (capture no state), so we keep them at module scope +// to avoid re-creating closures per frame and to make them callable from +// any future composite path that needs to log non-zero pixel counts. +function countNonZeroAlpha(rgba: Uint8Array): number { + let n = 0; + for (let p = 3; p < rgba.length; p += 4) { + if (rgba[p] !== 0) n++; + } + return n; +} + +function countNonZeroRgb48(buf: Uint8Array): number { + let n = 0; + for (let p = 0; p < buf.length; p += 6) { + if ( + buf[p] !== 0 || + buf[p + 1] !== 0 || + buf[p + 2] !== 0 || + buf[p + 3] !== 0 || + buf[p + 4] !== 0 || + buf[p + 5] !== 0 + ) + n++; + } + return n; +} + /** * Metadata for a shader transition between two scenes, extracted from * `window.__hf.transitions`. Re-exported from the engine so the producer @@ -572,6 +601,269 @@ function blitHdrImageLayer( } } +/** + * Dependencies passed to `compositeHdrFrame`. + * + * Every field except the per-frame arguments is captured once when the HDR + * render path opens its `try { ... }` block and reused across every frame — + * extracting them into an explicit struct lets the helper live at module + * scope (no closure-over-renderJob) and keeps the per-call signature small. + */ +interface HdrCompositeContext { + log: ProducerLogger; + domSession: CaptureSession; + beforeCaptureHook: BeforeCaptureHook | null; + width: number; + height: number; + fps: number; + effectiveHdr: { transfer: HdrTransfer }; + nativeHdrImageIds: Set; + hdrImageBuffers: Map; + hdrFrameDirs: Map; + hdrVideoStartTimes: Map; + imageTransfers: Map; + videoTransfers: Map; + debugDumpEnabled: boolean; + debugDumpDir: string | null; +} + +/** + * Composite a single HDR frame into a pre-allocated `rgb48le` canvas. + * + * Bottom-to-top z-order: HDR layers are blitted directly from cached image + * buffers / extracted video frames; DOM layers are screenshotted with a + * mass-hide mask (so each layer paints only its own elements) and then + * blended into the canvas via `blitRgba8OverRgb48le` in the active HDR + * transfer space. + * + * The `elementFilter` parameter exists so the transition path can composite + * each scene independently; pass `undefined` for whole-stack rendering. + * + * @param ctx - Long-lived dependencies (logger, browser session, dimensions, + * HDR layer maps). Captured once per render — see + * {@link HdrCompositeContext}. + * @param canvas - Pre-allocated `width * height * 6` byte buffer. Caller must + * zero-fill before every frame (this helper does not). + * @param time - Seek time in seconds. + * @param fullStacking - Stacking info for ALL elements at this time. Even when + * filtering, every other element id is needed to build + * the DOM-layer hide-list. + * @param elementFilter - When set, only elements whose id is in the set are + * composited. + * @param debugFrameIndex - Frame index used to label per-layer diagnostic + * dumps. Pass `-1` to disable per-layer dumps even + * when `KEEP_TEMP=1` (e.g. for warmup frames). + */ +async function compositeHdrFrame( + ctx: HdrCompositeContext, + canvas: Buffer, + time: number, + fullStacking: ElementStackingInfo[], + elementFilter?: Set, + debugFrameIndex: number = -1, +): Promise { + const { + log, + domSession, + beforeCaptureHook, + width, + height, + fps, + effectiveHdr, + nativeHdrImageIds, + hdrImageBuffers, + hdrFrameDirs, + hdrVideoStartTimes, + imageTransfers, + videoTransfers, + debugDumpEnabled, + debugDumpDir, + } = ctx; + + const filteredStacking = elementFilter + ? fullStacking.filter((e) => elementFilter.has(e.id)) + : fullStacking; + + const layers = groupIntoLayers(filteredStacking); + + const shouldLog = debugDumpEnabled && debugFrameIndex >= 0; + if (shouldLog) { + log.info("[diag] compositeToBuffer plan", { + frame: debugFrameIndex, + time: time.toFixed(3), + filterSize: elementFilter?.size, + fullStackingCount: fullStacking.length, + filteredCount: filteredStacking.length, + layerCount: layers.length, + layers: layers.map((l) => + l.type === "hdr" + ? { + type: "hdr", + id: l.element.id, + z: l.element.zIndex, + visible: l.element.visible, + opacity: l.element.opacity, + bounds: `${Math.round(l.element.x)},${Math.round(l.element.y)} ${Math.round(l.element.width)}x${Math.round(l.element.height)}`, + } + : { type: "dom", ids: l.elementIds }, + ), + }); + } + + for (const [layerIdx, layer] of layers.entries()) { + if (layer.type === "hdr") { + const before = shouldLog ? countNonZeroRgb48(canvas) : 0; + const isHdrImage = nativeHdrImageIds.has(layer.element.id); + if (isHdrImage) { + blitHdrImageLayer( + canvas, + layer.element, + hdrImageBuffers, + width, + height, + log, + imageTransfers.get(layer.element.id), + effectiveHdr.transfer, + ); + } else { + blitHdrVideoLayer( + canvas, + layer.element, + time, + fps, + hdrFrameDirs, + hdrVideoStartTimes, + width, + height, + log, + videoTransfers.get(layer.element.id), + effectiveHdr.transfer, + ); + } + if (shouldLog) { + const after = countNonZeroRgb48(canvas); + if (isHdrImage) { + const buf = hdrImageBuffers.get(layer.element.id); + log.info("[diag] hdr layer blit", { + frame: debugFrameIndex, + layerIdx, + id: layer.element.id, + kind: "image", + pixelsAdded: after - before, + totalNonZero: after, + bufferDecoded: !!buf, + bufferDims: buf ? `${buf.width}x${buf.height}` : null, + }); + } else { + const frameDir = hdrFrameDirs.get(layer.element.id); + const startTime = hdrVideoStartTimes.get(layer.element.id) ?? 0; + const localTime = time - startTime; + const frameNum = Math.floor(localTime * fps) + 1; + const expectedFrame = frameDir + ? join(frameDir, `frame_${String(frameNum).padStart(4, "0")}.png`) + : null; + log.info("[diag] hdr layer blit", { + frame: debugFrameIndex, + layerIdx, + id: layer.element.id, + kind: "video", + pixelsAdded: after - before, + totalNonZero: after, + startTime, + localTime: localTime.toFixed(3), + hdrFrameNum: frameNum, + expectedFrame, + expectedFrameExists: expectedFrame ? existsSync(expectedFrame) : false, + }); + } + } + } else { + // DOM layer: capture only elements in this layer. + // + // Each layer gets a fresh seek + inject cycle to guarantee correct + // visibility state — avoids fragile interactions between the frame + // injector, applyDomLayerMask, removeDomLayerMask, and GSAP re-seek. + // + // The mask: + // - mass-hides every body descendant via stylesheet + // - re-shows the layer's elements (and their descendants and + // their injected `__render_frame_*` siblings) so deep-nested + // content stays visible even though intermediate ancestors + // are hidden + // - inline-hides every other data-start element so they don't + // paint when they happen to be descendants of a layer element + // (most importantly: HDR videos and other-layer SDR videos + // that live inside `#root` when capturing the root DOM layer) + // + // Without the mask, every DOM screenshot captures the full page + // (root background, sibling scenes' static content, the painted + // border/box-shadow of cards, etc.) and the resulting opaque + // pixels overwrite previously composited HDR content beneath. + const allElementIds = fullStacking.map((e) => e.id); + const layerIds = new Set(layer.elementIds); + const hideIds = allElementIds.filter((id) => !layerIds.has(id)); + + // 1. Seek GSAP to restore all animated properties from clean state + await domSession.page.evaluate((t: number) => { + if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t); + }, time); + + // 2. Run frame injector to set correct SDR video visibility + if (beforeCaptureHook) { + await beforeCaptureHook(domSession.page, time); + } + + // 3. Install the mask (mass-hide stylesheet + inline-hide non-layer ids) + await applyDomLayerMask(domSession.page, layer.elementIds, hideIds); + + // 4. Screenshot + const domPng = await captureAlphaPng(domSession.page, width, height); + + // 5. Tear down the mask + await removeDomLayerMask(domSession.page, hideIds); + + try { + const { data: domRgba } = decodePng(domPng); + const before = shouldLog ? countNonZeroRgb48(canvas) : 0; + const alphaPixels = shouldLog ? countNonZeroAlpha(domRgba) : 0; + blitRgba8OverRgb48le(domRgba, canvas, width, height, effectiveHdr.transfer); + if (shouldLog && debugDumpDir) { + const after = countNonZeroRgb48(canvas); + const dumpName = `frame_${String(debugFrameIndex).padStart(4, "0")}_layer_${String(layerIdx).padStart(2, "0")}_dom.png`; + const dumpPath = join(debugDumpDir, dumpName); + writeFileSync(dumpPath, domPng); + log.info("[diag] dom layer blit", { + frame: debugFrameIndex, + layerIdx, + layerIds: layer.elementIds, + hideCount: hideIds.length, + pngBytes: domPng.length, + alphaPixels, + pixelsAdded: after - before, + totalNonZero: after, + dumpPath, + }); + } + } catch (err) { + log.warn("DOM layer decode/blit failed; skipping overlay", { + layerIds: layer.elementIds, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + + if (shouldLog && debugDumpDir) { + const finalNonZero = countNonZeroRgb48(canvas); + log.info("[diag] compositeToBuffer end", { + frame: debugFrameIndex, + finalNonZeroPixels: finalNonZero, + totalPixels: width * height, + coverage: ((finalNonZero / (width * height)) * 100).toFixed(1) + "%", + }); + } +} + export function createRenderJob(config: RenderConfig): RenderJob { return { id: randomUUID(), @@ -1011,7 +1303,7 @@ export async function executeRenderJob( videoPath = fromCompiled; } if (!existsSync(videoPath)) return; - const meta = await extractVideoMetadata(videoPath); + const meta = await extractMediaMetadata(videoPath); if (isHdrColorSpace(meta.colorSpace)) { nativeHdrVideoIds.add(v.id); videoTransfers.set(v.id, detectTransfer(meta.colorSpace)); @@ -1038,7 +1330,7 @@ export async function executeRenderJob( imgPath = fromCompiled; } if (!existsSync(imgPath)) return null; - const meta = await extractVideoMetadata(imgPath); + const meta = await extractMediaMetadata(imgPath); if (isHdrColorSpace(meta.colorSpace)) { nativeHdrImageIds.add(img.id); imageTransfers.set(img.id, detectTransfer(meta.colorSpace)); @@ -1172,6 +1464,19 @@ export async function executeRenderJob( quality: needsAlpha ? undefined : job.config.quality === "draft" ? 80 : 95, }; + // Native HDR videos (e.g. HEVC) may be undecodable by Chrome on the current + // platform — Linux headless-shell ships without HEVC support. Their pixels + // come from out-of-band ffmpeg extraction, so the DOM `