diff --git a/packages/core/src/runtime/state.ts b/packages/core/src/runtime/state.ts index da80473d..a0527911 100644 --- a/packages/core/src/runtime/state.ts +++ b/packages/core/src/runtime/state.ts @@ -30,6 +30,27 @@ export type RuntimeState = { bridgeLastPostedAt: number; bridgeLastPostedPlaying: boolean; bridgeLastPostedMuted: boolean; + /** + * Max interval (ms) between outbound timeline samples on the parent-frame + * control bridge. The bridge posts on every changed frame, but also at + * least once per this interval so a paused/idle timeline still confirms + * its position to any listener. + * + * **Cross-reference (do not change in isolation)**: the parent-frame + * audio-mirror loop in `` waits for + * `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES` consecutive over-threshold + * samples before issuing a `currentTime` correction. The product of + * those two constants is the worst-case A/V re-sync latency: + * + * worst_case_correction_latency_ms + * ≈ MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES × bridgeMaxPostIntervalMs + * + * Today: `2 × 80 ms = 160 ms`, which sits comfortably under the + * perceptual A/V re-sync tolerance. If you raise this interval, audit + * `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES` in + * `packages/player/src/hyperframes-player.ts` — leaving it at `2` will + * silently push correction latency past the tolerance budget. + */ bridgeMaxPostIntervalMs: number; timelinePollIntervalId: ReturnType | null; controlBridgeHandler: ((event: MessageEvent) => void) | null; diff --git a/packages/player/src/hyperframes-player.test.ts b/packages/player/src/hyperframes-player.test.ts index c55e3f7d..6f95e47f 100644 --- a/packages/player/src/hyperframes-player.test.ts +++ b/packages/player/src/hyperframes-player.test.ts @@ -440,3 +440,203 @@ describe("HyperframesPlayer media MutationObserver scoping", () => { expect(observeSpy.mock.calls[0]?.[0]).toBe(fakeDoc.body); }); }); + +// ── Parent-proxy time-mirror coalescing ── +// +// `_mirrorParentMediaTime` is the steady-state correction loop that nudges +// every parent-frame audio/video proxy back onto the iframe's timeline. The +// post-`P1-4` contract: a single over-threshold sample (one slow bridge tick, +// one tab-throttled rAF, one GC pause) is absorbed by a per-proxy counter and +// does NOT cost a `currentTime` write. Only a *trending* drift — two +// consecutive samples above the 50 ms threshold — triggers a seek. Forced +// callers (audio-ownership promotion, brand-new proxy initialization) bypass +// the gate so the listener never hears a misaligned sample on cut-over. + +describe("HyperframesPlayer parent-proxy time-mirror coalescing", () => { + type DriftEntry = { + el: { currentTime: number; src: string; pause: () => void }; + start: number; + duration: number; + driftSamples: number; + }; + type PlayerInternal = HTMLElement & { + _parentMedia: DriftEntry[]; + _mirrorParentMediaTime: (timelineSeconds: number, options?: { force?: boolean }) => void; + _promoteToParentProxy?: () => void; + }; + + let player: PlayerInternal; + + beforeEach(async () => { + await import("./hyperframes-player.js"); + player = document.createElement("hyperframes-player") as PlayerInternal; + document.body.appendChild(player); + // No audio-src was set, so `_parentMedia` is empty. Tests push synthetic + // POJO entries — `_mirrorParentMediaTime` only reads/writes + // `el.currentTime`, so a plain object stands in fine for HTMLMediaElement. + }); + + afterEach(() => { + player.remove(); + vi.restoreAllMocks(); + }); + + function makeEntry( + opts: { + currentTime?: number; + start?: number; + duration?: number; + driftSamples?: number; + } = {}, + ): DriftEntry { + // Include `pause`/`src` so `disconnectedCallback`'s teardown loop + // (`m.el.pause(); m.el.src = ""`) doesn't blow up when the player is + // removed at the end of the test — `_mirrorParentMediaTime` itself only + // touches `currentTime`. + const entry: DriftEntry = { + el: { + currentTime: opts.currentTime ?? 0, + src: "", + pause: vi.fn(), + }, + start: opts.start ?? 0, + duration: opts.duration ?? 100, + driftSamples: opts.driftSamples ?? 0, + }; + player._parentMedia.push(entry); + return entry; + } + + it("initializes new parent-media entries with driftSamples=0", () => { + // Mock Audio just for this test so the audio-src bootstrap path produces + // a real entry rather than throwing on construction. + const mockAudio = { + src: "", + preload: "", + muted: false, + playbackRate: 1, + currentTime: 0, + paused: true, + play: vi.fn().mockResolvedValue(undefined), + pause: vi.fn(), + load: vi.fn(), + }; + vi.spyOn(globalThis, "Audio").mockImplementation( + () => mockAudio as unknown as HTMLAudioElement, + ); + + const fresh = document.createElement("hyperframes-player") as PlayerInternal; + fresh.setAttribute("audio-src", "https://cdn.example.com/narration.mp3"); + document.body.appendChild(fresh); + + expect(fresh._parentMedia).toHaveLength(1); + expect(fresh._parentMedia[0]?.driftSamples).toBe(0); + fresh.remove(); + }); + + it("does nothing when drift is within the 50 ms threshold", () => { + const m = makeEntry({ currentTime: 5 }); + player._mirrorParentMediaTime(5.04); + expect(m.el.currentTime).toBe(5); + expect(m.driftSamples).toBe(0); + }); + + it("absorbs a single over-threshold spike without writing currentTime", () => { + const m = makeEntry({ currentTime: 5 }); + player._mirrorParentMediaTime(5.5); + expect(m.el.currentTime).toBe(5); + expect(m.driftSamples).toBe(1); + }); + + it("issues a seek on the second consecutive over-threshold sample", () => { + const m = makeEntry({ currentTime: 5 }); + player._mirrorParentMediaTime(5.5); + expect(m.el.currentTime).toBe(5); + expect(m.driftSamples).toBe(1); + // Second sample with the same drift: the gate trips, the write fires, + // and the counter resets so the proxy doesn't re-seek every later tick. + player._mirrorParentMediaTime(5.5); + expect(m.el.currentTime).toBe(5.5); + expect(m.driftSamples).toBe(0); + }); + + it("resets the counter when a sample comes back within threshold", () => { + const m = makeEntry({ currentTime: 5 }); + player._mirrorParentMediaTime(5.5); + expect(m.driftSamples).toBe(1); + // Recovery — counter must clear so a later isolated spike doesn't + // accidentally satisfy the 2-sample gate by piggy-backing on stale state. + player._mirrorParentMediaTime(5.02); + expect(m.driftSamples).toBe(0); + expect(m.el.currentTime).toBe(5); + player._mirrorParentMediaTime(5.5); + expect(m.driftSamples).toBe(1); + expect(m.el.currentTime).toBe(5); + }); + + it("force: true writes immediately on the first over-threshold sample", () => { + const m = makeEntry({ currentTime: 5 }); + player._mirrorParentMediaTime(5.5, { force: true }); + expect(m.el.currentTime).toBe(5.5); + expect(m.driftSamples).toBe(0); + }); + + it("force: true clears any pre-existing drift counter", () => { + const m = makeEntry({ currentTime: 5, driftSamples: 1 }); + player._mirrorParentMediaTime(5.5, { force: true }); + expect(m.el.currentTime).toBe(5.5); + expect(m.driftSamples).toBe(0); + }); + + it("does not seek out-of-range entries and resets their counters", () => { + // Active window [10, 15). currentTime=99 is a sentinel — if the function + // ever writes inside an out-of-range branch the test catches it because + // relTime would be 5 (or 15), not 99. + const m = makeEntry({ + currentTime: 99, + start: 10, + duration: 5, + driftSamples: 5, + }); + player._mirrorParentMediaTime(5); + expect(m.el.currentTime).toBe(99); + expect(m.driftSamples).toBe(0); + // Boundary: relTime === duration → still out of range (the loop uses `>=`). + m.driftSamples = 7; + player._mirrorParentMediaTime(15); + expect(m.el.currentTime).toBe(99); + expect(m.driftSamples).toBe(0); + }); + + it("tracks drift independently across multiple proxies", () => { + // a is drifted; b is aligned. A single tick must increment a's counter + // and reset b's — proving the per-entry state is genuinely per-entry. + const a = makeEntry({ currentTime: 5 }); + const b = makeEntry({ currentTime: 7.01, driftSamples: 1 }); + player._mirrorParentMediaTime(7); + expect(a.el.currentTime).toBe(5); + expect(a.driftSamples).toBe(1); + expect(b.el.currentTime).toBe(7.01); + expect(b.driftSamples).toBe(0); + }); + + it("force: true bypasses the gate for every proxy in a single sweep", () => { + const a = makeEntry({ currentTime: 5 }); + const b = makeEntry({ currentTime: 8 }); + player._mirrorParentMediaTime(7, { force: true }); + expect(a.el.currentTime).toBe(7); + expect(b.el.currentTime).toBe(7); + expect(a.driftSamples).toBe(0); + expect(b.driftSamples).toBe(0); + }); + + it("_promoteToParentProxy invokes _mirrorParentMediaTime with force: true", () => { + // Integration check of the promotion call site — we cannot tolerate even + // ~80 ms of audible drift across an ownership flip, so the call site + // must opt out of the jitter gate. + const spy = vi.spyOn(player, "_mirrorParentMediaTime"); + player._promoteToParentProxy?.(); + const forcedCall = spy.mock.calls.find(([, opts]) => opts?.force === true); + expect(forcedCall).toBeDefined(); + }); +}); diff --git a/packages/player/src/hyperframes-player.ts b/packages/player/src/hyperframes-player.ts index dee14601..66c7d820 100644 --- a/packages/player/src/hyperframes-player.ts +++ b/packages/player/src/hyperframes-player.ts @@ -60,6 +60,16 @@ class HyperframesPlayer extends HTMLElement { el: HTMLMediaElement; start: number; duration: number; + /** + * Count of consecutive steady-state samples in which the proxy's + * `currentTime` was found drifted beyond `MIRROR_DRIFT_THRESHOLD_SECONDS`. + * Reset on every in-threshold sample. `_mirrorParentMediaTime` only + * issues a write once this passes `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES`, + * which absorbs single-sample jitter (e.g. one slow bridge tick) without + * thrashing the media element with seeks. Forced calls (promotion, + * media-added) bypass the gate and reset the counter. + */ + driftSamples: number; }> = []; /** @@ -631,12 +641,62 @@ class HyperframesPlayer extends HTMLElement { */ private static readonly MIRROR_DRIFT_THRESHOLD_SECONDS = 0.05; - private _mirrorParentMediaTime(timelineSeconds: number) { + /** + * How many *consecutive* over-threshold steady-state samples we wait for + * before issuing a `currentTime` write. A value of 2 means a single + * spike (one slow bridge tick, one tab-throttled rAF batch, one GC pause) + * is absorbed without a seek; sustained drift still corrects on the very + * next tick after the threshold is crossed twice in a row. + * + * **Coupling with the timeline-control bridge** — read before changing: + * worst_case_correction_latency_ms + * ≈ MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES × bridgeMaxPostIntervalMs + * + * `bridgeMaxPostIntervalMs` (currently `80`) lives at + * `packages/core/src/runtime/state.ts` (field on `RuntimeState`). At + * today's values, worst-case is `2 × 80 ms = 160 ms` — still well under + * the human shot-change tolerance for A/V re-sync. If you bump bridge + * cadence (raising `bridgeMaxPostIntervalMs`) you may need to drop this + * constant to `1` to keep the product under ~150 ms; if you tighten + * cadence you can raise this to absorb more jitter without perceptual + * cost. There is a back-reference in `state.ts` next to + * `bridgeMaxPostIntervalMs` so a change to either side surfaces the + * coupling. + */ + private static readonly MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES = 2; + + /** + * Mirror parent-proxy `currentTime` to the iframe timeline. Defaults to + * the *coalesced* path: a single over-threshold sample is treated as + * jitter and merely increments a per-proxy counter; the actual seek only + * fires once `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES` consecutive + * samples agree. Pass `{ force: true }` for one-shot alignment moments + * (audio-ownership promotion, brand-new proxy initialization) where we + * cannot tolerate even ~80 ms of misaligned audible playback. + * + * The counter is also reset on any in-threshold sample and on any + * out-of-range timeline position, so a proxy that drops back into a + * scene later starts fresh rather than carrying stale samples from the + * last time it was active. + */ + private _mirrorParentMediaTime(timelineSeconds: number, options?: { force?: boolean }) { + const force = options?.force === true; + const requiredSamples = HyperframesPlayer.MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES; + const threshold = HyperframesPlayer.MIRROR_DRIFT_THRESHOLD_SECONDS; for (const m of this._parentMedia) { const relTime = timelineSeconds - m.start; - if (relTime < 0 || relTime >= m.duration) continue; - if (Math.abs(m.el.currentTime - relTime) > HyperframesPlayer.MIRROR_DRIFT_THRESHOLD_SECONDS) { - m.el.currentTime = relTime; + if (relTime < 0 || relTime >= m.duration) { + m.driftSamples = 0; + continue; + } + if (Math.abs(m.el.currentTime - relTime) > threshold) { + m.driftSamples += 1; + if (force || m.driftSamples >= requiredSamples) { + m.el.currentTime = relTime; + m.driftSamples = 0; + } + } else { + m.driftSamples = 0; } } } @@ -668,7 +728,10 @@ class HyperframesPlayer extends HTMLElement { // precisely because the scenario that triggered promotion is // "autoplay blocked" — the iframe can't make noise on its own. this._sendControl("set-media-output-muted", { muted: true }); - this._mirrorParentMediaTime(this._currentTime); + // One-shot alignment: a brand-new proxy must pick up the iframe's exact + // timeline position immediately to avoid an audible jump. Bypass the + // jitter-coalescing gate. + this._mirrorParentMediaTime(this._currentTime, { force: true }); if (!this._paused) this._playParentMedia(); this.dispatchEvent( new CustomEvent("audioownershipchange", { @@ -688,7 +751,7 @@ class HyperframesPlayer extends HTMLElement { tag: "audio" | "video", start: number, duration: number, - ): { el: HTMLMediaElement; start: number; duration: number } | null { + ): { el: HTMLMediaElement; start: number; duration: number; driftSamples: number } | null { // Deduplicate — browsers normalize URLs so we compare on the element after assignment if (this._parentMedia.some((m) => m.el.src === src)) return null; @@ -699,7 +762,7 @@ class HyperframesPlayer extends HTMLElement { el.muted = this.muted; if (this.playbackRate !== 1) el.playbackRate = this.playbackRate; - const entry = { el, start, duration }; + const entry = { el, start, duration, driftSamples: 0 }; this._parentMedia.push(entry); return entry; } @@ -778,7 +841,10 @@ class HyperframesPlayer extends HTMLElement { // start producing audio right away — otherwise it sits silent through // the next several hundred ms until the next runtime state message. if (created && this._audioOwner === "parent") { - this._mirrorParentMediaTime(this._currentTime); + // One-shot alignment: a freshly-created proxy must catch up to the + // current timeline position on the very first sample, so bypass the + // jitter-coalescing gate. + this._mirrorParentMediaTime(this._currentTime, { force: true }); if (!this._paused && created.el.src) { created.el.play().catch((err: unknown) => this._reportPlaybackError(err)); }