feat(studio): drag assets from the sidebar onto the timeline

packages/core/src/runtime/init.test.ts

@@ -0,0 +1,126 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { initSandboxRuntimeModular } from "./init";
+import type { RuntimeTimelineLike } from "./types";
+
+function createMockTimeline(duration: number): RuntimeTimelineLike {
+  const state = { time: 0, paused: true };
+  return {
+    play: () => {
+      state.paused = false;
+    },
+    pause: () => {
+      state.paused = true;
+    },
+    seek: (time: number) => {
+      state.time = time;
+    },
+    totalTime: (time: number) => {
+      state.time = time;
+    },
+    time: () => state.time,
+    duration: () => duration,
+    add: () => {},
+    paused: (value?: boolean) => {
+      if (typeof value === "boolean") {
+        state.paused = value;
+      }
+      return state.paused;
+    },
+    timeScale: () => {},
+    set: () => {},
+    getChildren: () => [],
+  };
+}
+
+describe("initSandboxRuntimeModular", () => {
+  const originalRequestAnimationFrame = window.requestAnimationFrame;
+  const originalCancelAnimationFrame = window.cancelAnimationFrame;
+
+  beforeEach(() => {
+    document.body.innerHTML = "";
+    (globalThis as typeof globalThis & { CSS?: { escape?: (value: string) => string } }).CSS ??= {};
+    globalThis.CSS.escape ??= (value: string) => value;
+    window.requestAnimationFrame = ((callback: FrameRequestCallback) => {
+      callback(0);
+      return 1;
+    }) as typeof window.requestAnimationFrame;
+    window.cancelAnimationFrame = (() => {}) as typeof window.cancelAnimationFrame;
+  });
+
+  afterEach(() => {
+    (window as Window & { __hfRuntimeTeardown?: (() => void) | null }).__hfRuntimeTeardown?.();
+    document.body.innerHTML = "";
+    delete (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> }).__timelines;
+    delete (window as Window & { __player?: unknown }).__player;
+    delete (window as Window & { __playerReady?: boolean }).__playerReady;
+    delete (window as Window & { __renderReady?: boolean }).__renderReady;
+    window.requestAnimationFrame = originalRequestAnimationFrame;
+    window.cancelAnimationFrame = originalCancelAnimationFrame;
+  });
+
+  it("uses the shorter live child timeline when the authored window is longer", () => {
+    const root = document.createElement("div");
+    root.setAttribute("data-composition-id", "main");
+    root.setAttribute("data-root", "true");
+    root.setAttribute("data-width", "1920");
+    root.setAttribute("data-height", "1080");
+    document.body.appendChild(root);
+
+    const child = document.createElement("div");
+    child.setAttribute("data-composition-id", "slide-1");
+    child.setAttribute("data-start", "0");
+    child.setAttribute("data-hf-authored-duration", "14");
+    root.appendChild(child);
+
+    (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> }).__timelines = {
+      main: createMockTimeline(20),
+      "slide-1": createMockTimeline(8),
+    };
+
+    initSandboxRuntimeModular();
+
+    const player = (
+      window as Window & {
+        __player?: { renderSeek: (timeSeconds: number) => void };
+      }
+    ).__player;
+    expect(player).toBeDefined();
+
+    player?.renderSeek(9);
+
+    expect(child.style.visibility).toBe("hidden");
+  });
+
+  it("uses the shorter authored host window when the child timeline is longer", () => {
+    const root = document.createElement("div");
+    root.setAttribute("data-composition-id", "main");
+    root.setAttribute("data-root", "true");
+    root.setAttribute("data-width", "1920");
+    root.setAttribute("data-height", "1080");
+    document.body.appendChild(root);
+
+    const child = document.createElement("div");
+    child.setAttribute("data-composition-id", "slide-1");
+    child.setAttribute("data-start", "0");
+    child.setAttribute("data-hf-authored-duration", "2");
+    root.appendChild(child);
+
+    (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> }).__timelines = {
+      main: createMockTimeline(20),
+      "slide-1": createMockTimeline(8),
+    };
+
+    initSandboxRuntimeModular();
+
+    const player = (
+      window as Window & {
+        __player?: { renderSeek: (timeSeconds: number) => void };
+      }
+    ).__player;
+    expect(player).toBeDefined();
+
+    player?.renderSeek(3);
+
+    expect(child.style.visibility).toBe("hidden");
+  });
+});

packages/core/src/runtime/init.ts

@@ -368,13 +368,16 @@ export function initSandboxRuntimeModular(): void {
     return resolver.resolveStartForElement(element, fallback);
   };
 
-  const resolveDurationForElement = (element: Element): number | null => {
+  const resolveDurationForElement = (
+    element: Element,
+    opts?: { includeAuthoredTimingAttrs?: boolean },
+  ): number | null => {
     const resolver = createRuntimeStartTimeResolver({
       timelineRegistry: (window.__timelines ?? {}) as Record<
         string,
         RuntimeTimelineLike | undefined
       >,
-      includeAuthoredTimingAttrs: true,
+      includeAuthoredTimingAttrs: opts?.includeAuthoredTimingAttrs ?? true,
     });
     return resolver.resolveDurationForElement(element);
   };
@@ -1234,18 +1237,28 @@ export function initSandboxRuntimeModular(): void {
       }
 
       const start = resolveStartForElement(rawNode, 0);
-      const duration = resolveDurationForElement(rawNode);
-      const end = duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY;
-      // For composition hosts, use the composition timeline's duration to compute end
-      let computedEnd = end;
+      let duration = resolveDurationForElement(rawNode);
       const compId = rawNode.getAttribute("data-composition-id");
-      if (compId && !Number.isFinite(end)) {
+      if (compId) {
         const compTimeline = (window.__timelines ?? {})[compId];
+        let liveDuration: number | null = null;
         if (compTimeline && typeof compTimeline.duration === "function") {
-          const compDur = compTimeline.duration();
-          if (compDur > 0) computedEnd = start + compDur;
+          const compDur = Number(compTimeline.duration());
+          if (Number.isFinite(compDur) && compDur > 0) {
+            liveDuration = compDur;
+          }
+        }
+
+        // Composition hosts must respect both the authored clip window in the parent
+        // composition and the child composition's own live timeline duration.
+        if (duration != null && duration > 0 && liveDuration != null) {
+          duration = Math.min(duration, liveDuration);
+        } else if ((duration == null || duration <= 0) && liveDuration != null) {
+          duration = liveDuration;
         }
       }
+      const computedEnd =
+        duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY;
       const isVisibleNow =
         state.currentTime >= start &&
         (Number.isFinite(computedEnd) ? state.currentTime < computedEnd : true);

packages/core/src/studio-api/helpers/mediaValidation.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, it } from "vitest";
+import { validateUploadedMedia, validateUploadedMediaBuffer } from "./mediaValidation.js";
+
+describe("validateUploadedMedia", () => {
+  it("passes through non-media files", () => {
+    expect(
+      validateUploadedMedia("/tmp/test.svg", () => ({ status: 0, stdout: "", stderr: "" })),
+    ).toEqual({
+      ok: true,
+    });
+  });
+
+  it("accepts video files with a video stream", () => {
+    expect(
+      validateUploadedMedia("/tmp/test.mp4", () => ({
+        status: 0,
+        stdout: JSON.stringify({ streams: [{ codec_type: "video" }] }),
+        stderr: "",
+      })),
+    ).toEqual({ ok: true });
+  });
+
+  it("rejects video files with no supported video stream", () => {
+    expect(
+      validateUploadedMedia("/tmp/test.mp4", () => ({
+        status: 0,
+        stdout: JSON.stringify({ streams: [] }),
+        stderr: "",
+      })),
+    ).toEqual({ ok: false, reason: "no supported video stream found" });
+  });
+
+  it("accepts audio files with an audio stream", () => {
+    expect(
+      validateUploadedMedia("/tmp/test.wav", () => ({
+        status: 0,
+        stdout: JSON.stringify({ streams: [{ codec_type: "audio" }] }),
+        stderr: "",
+      })),
+    ).toEqual({ ok: true });
+  });
+
+  it("does not block upload when ffprobe is unavailable", () => {
+    expect(
+      validateUploadedMedia("/tmp/test.mp4", () => ({
+        status: null,
+        stdout: "",
+        stderr: "",
+        error: { code: "ENOENT" } as NodeJS.ErrnoException,
+      })),
+    ).toEqual({ ok: true });
+  });
+});
+
+describe("validateUploadedMediaBuffer", () => {
+  it("validates media from a temp file that preserves the extension", () => {
+    let inspectedPath = "";
+    expect(
+      validateUploadedMediaBuffer("raycast.mp4", new Uint8Array([0, 1, 2]), (_command, args) => {
+        inspectedPath = args.at(-1) ?? "";
+        return {
+          status: 0,
+          stdout: JSON.stringify({ streams: [{ codec_type: "video" }] }),
+          stderr: "",
+        };
+      }),
+    ).toEqual({ ok: true });
+
+    expect(inspectedPath).toMatch(/raycast\.mp4$/);
+  });
+});

packages/core/src/studio-api/helpers/mediaValidation.ts

@@ -0,0 +1,80 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+
+const VIDEO_EXT = /\.(mp4|webm|mov)$/i;
+const AUDIO_EXT = /\.(mp3|wav|ogg|m4a|aac)$/i;
+
+type FfprobeRunner = (
+  command: string,
+  args: string[],
+) => {
+  status: number | null;
+  stdout: string | Buffer;
+  stderr: string | Buffer;
+  error?: NodeJS.ErrnoException;
+};
+
+export function validateUploadedMedia(
+  filePath: string,
+  runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner,
+): { ok: true } | { ok: false; reason: string } {
+  const isVideo = VIDEO_EXT.test(filePath);
+  const isAudio = AUDIO_EXT.test(filePath);
+  if (!isVideo && !isAudio) {
+    return { ok: true };
+  }
+
+  const result = runner("ffprobe", [
+    "-v",
+    "error",
+    "-show_entries",
+    "stream=codec_type",
+    "-of",
+    "json",
+    filePath,
+  ]);
+
+  if (result.error?.code === "ENOENT") {
+    return { ok: true };
+  }
+  if (result.status !== 0) {
+    return { ok: false, reason: "ffprobe failed to read the media file" };
+  }
+
+  try {
+    const parsed = JSON.parse(String(result.stdout || "{}")) as {
+      streams?: Array<{ codec_type?: string }>;
+    };
+    const streams = parsed.streams ?? [];
+    const hasVideo = streams.some((stream) => stream.codec_type === "video");
+    const hasAudio = streams.some((stream) => stream.codec_type === "audio");
+
+    if (isVideo && !hasVideo) {
+      return { ok: false, reason: "no supported video stream found" };
+    }
+    if (isAudio && !hasAudio) {
+      return { ok: false, reason: "no supported audio stream found" };
+    }
+    return { ok: true };
+  } catch {
+    return { ok: false, reason: "ffprobe returned unreadable media metadata" };
+  }
+}
+
+export function validateUploadedMediaBuffer(
+  fileName: string,
+  buffer: Uint8Array,
+  runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner,
+): { ok: true } | { ok: false; reason: string } {
+  const tempDir = mkdtempSync(join(tmpdir(), "hyperframes-upload-"));
+  const tempPath = join(tempDir, basename(fileName));
+
+  try {
+    writeFileSync(tempPath, buffer);
+    return validateUploadedMedia(tempPath, runner);
+  } finally {
+    rmSync(tempDir, { recursive: true, force: true });
+  }
+}

packages/core/src/studio-api/helpers/sourceMutation.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from "vitest";
+import { removeElementFromHtml } from "./sourceMutation.js";
+
+describe("removeElementFromHtml", () => {
+  it("removes a self-closing element by id", () => {
+    const html = `<!doctype html><html><body><div data-composition-id="main"><img id="photo" src="asset.png" /><div id="rest"></div></div></body></html>`;
+
+    const updated = removeElementFromHtml(html, { id: "photo" });
+
+    expect(updated).not.toContain(`id="photo"`);
+    expect(updated).toContain(`id="rest"`);
+  });
+
+  it("removes a matched composition host by selector", () => {
+    const html = `<!doctype html><html><body><div data-composition-id="main"><div data-composition-id="scene-a"><span>Scene A</span></div><div data-composition-id="scene-b"></div></div></body></html>`;
+
+    const updated = removeElementFromHtml(html, {
+      selector: '[data-composition-id="scene-a"]',
+    });
+
+    expect(updated).not.toContain(`data-composition-id="scene-a"`);
+    expect(updated).toContain(`data-composition-id="scene-b"`);
+  });
+
+  it("supports fragment html by returning updated body markup", () => {
+    const html = `<div id="photo"></div><div id="rest"></div>`;
+
+    expect(removeElementFromHtml(html, { id: "photo" })).toBe(`<div id="rest"></div>`);
+  });
+});