diff --git a/docs/guides/hdr.mdx b/docs/guides/hdr.mdx index 9446d4d6..5774cfae 100644 --- a/docs/guides/hdr.mdx +++ b/docs/guides/hdr.mdx @@ -163,6 +163,8 @@ The container runs the same probe → composite → encode pipeline as the local - **MP4 only** — `--hdr` with `--format mov` or `--format webm` falls back to SDR - **HDR images: 16-bit PNG only** — other formats (JPEG, WebP, AVIF, APNG) are not decoded as HDR and fall through the SDR DOM path +- **H.265 only — H.264 is stripped** — calling the encoder with `codec: "h264"` and `hdr: { transfer }` is rejected; the encoder logs a warning, drops `hdr`, and tags the output as SDR/BT.709. `libx264` cannot encode HDR, so the alternative would be a "half-HDR" file (BT.2020 container tags but a BT.709 VUI block in the bitstream) which confuses HDR-aware players. +- **GPU H.265 emits color tags but no static mastering metadata** — `useGpu: true` with HDR (nvenc, videotoolbox, qsv, vaapi) tags the stream with BT.2020 + the correct transfer (smpte2084 / arib-std-b67) but does **not** embed `master-display` or `max-cll` SEI. ffmpeg does not let those flags pass through hardware encoders. The output is suitable for previews and authoring but not for HDR10-aware delivery (Apple TV, YouTube, Netflix). For spec-compliant HDR10 production output, leave `useGpu: false` so the SW `libx265` path embeds the mastering metadata. - **Player support** — the [``](/packages/player) web component plays back the encoded MP4 in the browser and inherits whatever HDR support the host browser provides; it does not implement its own HDR pipeline - **Headed Chrome HDR DOM capture** — the engine ships a separate WebGPU-based capture path for rendering CSS-animated DOM directly into HDR (`initHdrReadback`, `launchHdrBrowser`). It requires headed Chrome with `--enable-unsafe-webgpu` and is not used by the default render pipeline. See [Engine: HDR](/packages/engine#hdr-apis) if you are building a custom integration. diff --git a/packages/engine/src/config.ts b/packages/engine/src/config.ts index b0ea98d9..ea8055c4 100644 --- a/packages/engine/src/config.ts +++ b/packages/engine/src/config.ts @@ -182,7 +182,7 @@ export function resolveConfig(overrides?: Partial): EngineConfig { hdr: (() => { const raw = env("PRODUCER_HDR_TRANSFER"); if (raw === "hlg" || raw === "pq") return { transfer: raw }; - return undefined; + return false; })(), hdrAutoDetect: envBool("PRODUCER_HDR_AUTO_DETECT", DEFAULT_CONFIG.hdrAutoDetect), diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts index cf914959..98e104c8 100644 --- a/packages/engine/src/services/chunkEncoder.test.ts +++ b/packages/engine/src/services/chunkEncoder.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import { ENCODER_PRESETS, getEncoderPreset, buildEncoderArgs } from "./chunkEncoder.js"; describe("ENCODER_PRESETS", () => { @@ -285,35 +285,101 @@ describe("buildEncoderArgs HDR color space", () => { const baseOptions = { fps: 30, width: 1920, height: 1080 }; const inputArgs = ["-framerate", "30", "-i", "frames/%04d.png"]; - it("keeps bt709 color tags when HDR flag is set but frames are still Chrome sRGB captures", () => { - // HDR flag gives H.265 + 10-bit encoding but pixels are still sRGB/bt709. - // Tagging as bt2020 causes orange shift — so we tag truthfully as bt709. + it("emits BT.2020 + arib-std-b67 tags for HDR HLG (h265 SW)", () => { + // When options.hdr is set, the caller asserts the input pixels are + // already in the BT.2020 color space — tag the output truthfully so + // HDR-aware players apply the right transform. const args = buildEncoderArgs( { ...baseOptions, codec: "h265", preset: "medium", quality: 23, hdr: { transfer: "hlg" } }, inputArgs, "out.mp4", ); + expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt2020nc"); + expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt2020"); + expect(args[args.indexOf("-color_trc:v") + 1]).toBe("arib-std-b67"); + const paramIdx = args.indexOf("-x265-params"); + expect(args[paramIdx + 1]).toContain("colorprim=bt2020"); + expect(args[paramIdx + 1]).toContain("transfer=arib-std-b67"); + expect(args[paramIdx + 1]).toContain("colormatrix=bt2020nc"); + }); + + it("emits BT.2020 + smpte2084 tags for HDR PQ (h265 SW)", () => { + const args = buildEncoderArgs( + { ...baseOptions, codec: "h265", preset: "medium", quality: 23, hdr: { transfer: "pq" } }, + inputArgs, + "out.mp4", + ); + expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt2020nc"); + expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt2020"); + expect(args[args.indexOf("-color_trc:v") + 1]).toBe("smpte2084"); + const paramIdx = args.indexOf("-x265-params"); + expect(args[paramIdx + 1]).toContain("colorprim=bt2020"); + expect(args[paramIdx + 1]).toContain("transfer=smpte2084"); + expect(args[paramIdx + 1]).toContain("colormatrix=bt2020nc"); + }); + + it("embeds HDR static mastering metadata in x265-params when HDR is set", () => { + // master-display + max-cll SEI messages are required so HDR-aware + // players (Apple QuickTime, YouTube, HDR TVs) treat the stream as + // HDR10 instead of falling back to SDR BT.2020 tone-mapping. + const args = buildEncoderArgs( + { ...baseOptions, codec: "h265", preset: "medium", quality: 23, hdr: { transfer: "pq" } }, + inputArgs, + "out.mp4", + ); + const paramIdx = args.indexOf("-x265-params"); + expect(args[paramIdx + 1]).toContain("master-display="); + expect(args[paramIdx + 1]).toContain("max-cll="); + }); + + it("uses bt709 when HDR is not set (SDR Chrome captures)", () => { + const args = buildEncoderArgs( + { ...baseOptions, codec: "h265", preset: "medium", quality: 23 }, + inputArgs, + "out.mp4", + ); expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt709"); - expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt709"); expect(args[args.indexOf("-color_trc:v") + 1]).toBe("bt709"); const paramIdx = args.indexOf("-x265-params"); expect(args[paramIdx + 1]).toContain("colorprim=bt709"); - expect(args[paramIdx + 1]).toContain("transfer=bt709"); + expect(args[paramIdx + 1]).not.toContain("master-display"); }); - it("uses bt709 when HDR is not set", () => { + it("does not embed HDR mastering metadata when HDR is not set", () => { const args = buildEncoderArgs( { ...baseOptions, codec: "h265", preset: "medium", quality: 23 }, inputArgs, "out.mp4", ); + const paramIdx = args.indexOf("-x265-params"); + expect(args[paramIdx + 1]).not.toContain("master-display"); + expect(args[paramIdx + 1]).not.toContain("max-cll"); + }); + + it("strips HDR and tags as SDR/BT.709 when codec=h264 (libx264 has no HDR support)", () => { + // libx264 cannot encode HDR. Rather than emit a "half-HDR" file (BT.2020 + // container tags + BT.709 VUI inside the bitstream — confusing to HDR-aware + // players), we strip hdr and tag the whole output as SDR/BT.709. The caller + // gets a warning telling them to use codec=h265 for real HDR output. + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const args = buildEncoderArgs( + { ...baseOptions, codec: "h264", preset: "medium", quality: 23, hdr: { transfer: "pq" } }, + inputArgs, + "out.mp4", + ); + const paramIdx = args.indexOf("-x264-params"); + expect(args[paramIdx + 1]).toContain("colorprim=bt709"); + expect(args[paramIdx + 1]).not.toContain("master-display"); expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt709"); + expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt709"); expect(args[args.indexOf("-color_trc:v") + 1]).toBe("bt709"); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("HDR is not supported with codec=h264"), + ); + warnSpy.mockRestore(); }); - it("uses range conversion (not colorspace) for HDR CPU encoding", () => { - // Chrome screenshots are sRGB — we don't convert primaries (causes color shifts). - // Just range-convert and let the bt2020 container metadata + 10-bit handle the rest. + it("uses range conversion for HDR CPU encoding", () => { const args = buildEncoderArgs( { ...baseOptions, codec: "h265", preset: "medium", quality: 23, hdr: { transfer: "hlg" } }, inputArgs, @@ -322,7 +388,6 @@ describe("buildEncoderArgs HDR color space", () => { const vfIdx = args.indexOf("-vf"); expect(vfIdx).toBeGreaterThan(-1); expect(args[vfIdx + 1]).toContain("scale=in_range=pc:out_range=tv"); - expect(args[vfIdx + 1]).not.toContain("colorspace"); }); it("uses same range conversion for SDR CPU encoding", () => { @@ -334,4 +399,29 @@ describe("buildEncoderArgs HDR color space", () => { const vfIdx = args.indexOf("-vf"); expect(args[vfIdx + 1]).toContain("scale=in_range=pc:out_range=tv"); }); + + it("tags BT.2020 + transfer for HDR GPU H.265 (no mastering metadata via -x265-params)", () => { + // GPU encoders (nvenc, videotoolbox, qsv, vaapi) still emit the BT.2020 + // color tags via the codec-level -colorspace/-color_primaries/-color_trc + // flags, but cannot accept x265-params, so HDR static mastering metadata + // (master-display, max-cll) is not embedded. Acceptable for previews, + // not for HDR-aware delivery. + const args = buildEncoderArgs( + { + ...baseOptions, + codec: "h265", + preset: "medium", + quality: 23, + useGpu: true, + hdr: { transfer: "pq" }, + }, + inputArgs, + "out.mp4", + "nvenc", + ); + expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt2020nc"); + expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt2020"); + expect(args[args.indexOf("-color_trc:v") + 1]).toBe("smpte2084"); + expect(args.indexOf("-x265-params")).toBe(-1); + }); }); diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts index b7658d36..f44361ea 100644 --- a/packages/engine/src/services/chunkEncoder.ts +++ b/packages/engine/src/services/chunkEncoder.ts @@ -10,7 +10,7 @@ import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSy import { join, dirname } from "path"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { type GpuEncoder, getCachedGpuEncoder, getGpuEncoderName } from "../utils/gpuEncoder.js"; -import { type HdrTransfer } from "../utils/hdr.js"; +import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js"; @@ -88,6 +88,18 @@ export function buildEncoderArgs( useGpu = false, } = options; + // libx264 cannot encode HDR. If a caller passes hdr with codec=h264 we'd + // produce a "half-HDR" file (BT.2020 container tags but a BT.709 VUI block + // inside the bitstream) which confuses HDR-aware players. Strip hdr and + // log a warning so the caller picks h265 (the SDR-tagged output is honest). + if (options.hdr && codec === "h264") { + console.warn( + "[chunkEncoder] HDR is not supported with codec=h264 (libx264 has no HDR support). " + + "Stripping HDR metadata and tagging output as SDR/BT.709. Use codec=h265 for HDR output.", + ); + options = { ...options, hdr: undefined }; + } + const args: string[] = [...inputArgs, "-r", String(fps)]; const shouldUseGpu = useGpu && gpuEncoder !== null; @@ -130,8 +142,14 @@ export function buildEncoderArgs( // Encoder-specific params: anti-banding + color space tagging. // aq-mode=3 redistributes bits to dark flat areas (gradients). + // For HDR x265 paths we additionally embed BT.2020 + transfer + HDR static + // mastering metadata via x265-params; libx264 only carries BT.709 tags + // since HDR through H.264 is not supported by this encoder path. const xParamsFlag = codec === "h264" ? "-x264-params" : "-x265-params"; - const colorParams = "colorprim=bt709:transfer=bt709:colormatrix=bt709"; + const colorParams = + codec === "h265" && options.hdr + ? getHdrEncoderColorParams(options.hdr.transfer).x265ColorParams + : "colorprim=bt709:transfer=bt709:colormatrix=bt709"; if (preset === "ultrafast") { args.push(xParamsFlag, `aq-mode=3:${colorParams}`); } else { @@ -157,22 +175,44 @@ export function buildEncoderArgs( } // Color space metadata — tags the output so players interpret colors correctly. - // Chrome screenshots are always sRGB/bt709 pixels regardless of --hdr flag. - // We tag truthfully as bt709 even for HDR output — the --hdr flag gives - // H.265 + 10-bit encoding (better quality/compression) without lying about - // the color space. Tagging as bt2020 when pixels are bt709 causes browsers - // to apply the wrong color transform, producing visible orange/warm shifts. + // + // Default (no options.hdr): Chrome screenshots are sRGB/bt709 pixels and + // we tag them truthfully as bt709. Tagging as bt2020 when pixels are bt709 + // causes browsers to apply the wrong color transform, producing visible + // orange/warm shifts. + // + // HDR (options.hdr provided): the caller asserts the input pixels are + // already in the BT.2020 color space (e.g. extracted HDR video frames or a + // pre-tagged source). We tag the output as BT.2020 + the corresponding + // transfer (smpte2084 for PQ, arib-std-b67 for HLG). HDR static mastering + // metadata (master-display, max-cll) is embedded only in the SW libx265 + // path above; GPU H.265 + HDR carries the color tags but not the static + // metadata, which is acceptable for previews but not for HDR-aware delivery. if (codec === "h264" || codec === "h265") { - args.push( - "-colorspace:v", - "bt709", - "-color_primaries:v", - "bt709", - "-color_trc:v", - "bt709", - "-color_range", - "tv", - ); + if (options.hdr) { + const transferTag = options.hdr.transfer === "pq" ? "smpte2084" : "arib-std-b67"; + args.push( + "-colorspace:v", + "bt2020nc", + "-color_primaries:v", + "bt2020", + "-color_trc:v", + transferTag, + "-color_range", + "tv", + ); + } else { + args.push( + "-colorspace:v", + "bt709", + "-color_primaries:v", + "bt709", + "-color_trc:v", + "bt709", + "-color_range", + "tv", + ); + } // Range conversion: Chrome's full-range RGB → limited/TV range. if (gpuEncoder === "vaapi") { diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index a78ac640..3980e3a1 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -10,7 +10,11 @@ import { existsSync, mkdirSync, readdirSync, rmSync } from "fs"; import { join } from "path"; import { parseHTML } from "linkedom"; import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js"; -import { isHdrColorSpace as isHdrColorSpaceUtil } from "../utils/hdr.js"; +import { + analyzeCompositionHdr, + isHdrColorSpace as isHdrColorSpaceUtil, + type HdrTransfer, +} from "../utils/hdr.js"; import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; @@ -250,21 +254,28 @@ export async function extractVideoFramesRange( } /** - * Convert an SDR video to HDR color space (HLG / BT.2020) so it can be - * composited alongside HDR content without looking washed out. + * Convert an SDR (BT.709) video to BT.2020 wide-gamut so it can be composited + * alongside HDR content without looking washed out. * - * Uses zscale for color space conversion with a nominal peak luminance of - * 600 nits — high enough that SDR content doesn't appear too dark next to - * HDR, matching the approach used by HeyGen's Rio pipeline. + * Uses FFmpeg's `colorspace` filter to remap BT.709 → BT.2020 (no real tone + * mapping — just a primaries swap so the input fits inside the wider HDR + * gamut), then re-tags the stream with the caller's target HDR transfer + * function (PQ for HDR10, HLG for broadcast HDR). The output transfer must + * match the dominant transfer of the surrounding HDR content; otherwise the + * downstream encoder will tag the final video with the wrong curve. */ async function convertSdrToHdr( inputPath: string, outputPath: string, + targetTransfer: HdrTransfer, signal?: AbortSignal, config?: Partial>, ): Promise { const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; + // smpte2084 = PQ (HDR10), arib-std-b67 = HLG. + const colorTrc = targetTransfer === "pq" ? "smpte2084" : "arib-std-b67"; + const args = [ "-i", inputPath, @@ -273,7 +284,7 @@ async function convertSdrToHdr( "-color_primaries", "bt2020", "-color_trc", - "arib-std-b67", + colorTrc, "-colorspace", "bt2020nc", "-c:v", @@ -401,8 +412,15 @@ export async function extractAllVideoFrames( }), ); - const hasAnyHdr = videoColorSpaces.some(isHdrColorSpaceUtil); - if (hasAnyHdr) { + const hdrInfo = analyzeCompositionHdr(videoColorSpaces); + if (hdrInfo.hasHdr && hdrInfo.dominantTransfer) { + // dominantTransfer is "majority wins" — if a composition mixes PQ and HLG + // sources (rare but legal), the minority transfer's videos get converted + // with the wrong curve. We treat this as caller-error: a single composition + // should not mix PQ and HLG sources, the orchestrator picks one transfer + // for the whole render, and any source not on that curve is normalized to + // it. If you need both transfers, render two separate compositions. + const targetTransfer = hdrInfo.dominantTransfer; const convertDir = join(options.outputDir, "_hdr_normalized"); mkdirSync(convertDir, { recursive: true }); @@ -410,12 +428,13 @@ export async function extractAllVideoFrames( if (signal?.aborted) break; const cs = videoColorSpaces[i] ?? null; if (!isHdrColorSpaceUtil(cs)) { - // SDR video in a mixed timeline — convert to HDR color space + // SDR video in a mixed timeline — convert to the dominant HDR transfer + // so the encoder tags the final video correctly (PQ vs HLG). const entry = resolvedVideos[i]; if (!entry) continue; const convertedPath = join(convertDir, `${entry.video.id}_hdr.mp4`); try { - await convertSdrToHdr(entry.videoPath, convertedPath, signal, config); + await convertSdrToHdr(entry.videoPath, convertedPath, targetTransfer, signal, config); entry.videoPath = convertedPath; } catch (err) { errors.push({ diff --git a/packages/producer/tests/hdr-regression/src/index.html b/packages/producer/tests/hdr-regression/src/index.html index 824f3713..48a7f837 100644 --- a/packages/producer/tests/hdr-regression/src/index.html +++ b/packages/producer/tests/hdr-regression/src/index.html @@ -241,4 +241,4 @@ window.__timelines["hdr-regression"] = tl; - + \ No newline at end of file