From 4b2c875f21b04b3c2c279c0eca35f5984d5ed0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 23 Apr 2026 18:37:35 -0400 Subject: [PATCH 1/7] feat(studio): drag assets from the sidebar onto the timeline --- packages/studio/src/App.tsx | 197 ++++++++++++++++-- .../studio/src/components/nle/NLELayout.tsx | 6 + .../src/components/sidebar/AssetsTab.tsx | 7 + .../src/player/components/Timeline.test.ts | 52 +++++ .../studio/src/player/components/Timeline.tsx | 108 +++++++++- .../src/utils/timelineAssetDrop.test.ts | 69 ++++++ .../studio/src/utils/timelineAssetDrop.ts | 77 +++++++ 7 files changed, 489 insertions(+), 27 deletions(-) create mode 100644 packages/studio/src/utils/timelineAssetDrop.test.ts create mode 100644 packages/studio/src/utils/timelineAssetDrop.ts diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index 889891204..cb8fe189f 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -13,6 +13,14 @@ import { LintModal } from "./components/LintModal"; import type { LintFinding } from "./components/LintModal"; import { MediaPreview } from "./components/MediaPreview"; import { isMediaFile } from "./utils/mediaTypes"; +import { + buildTimelineAssetId, + buildTimelineAssetInsertHtml, + getTimelineAssetKind, + insertTimelineAssetIntoSource, + resolveTimelineAssetSrc, + type TimelineAssetKind, +} from "./utils/timelineAssetDrop"; import { CaptionOverlay } from "./captions/components/CaptionOverlay"; import { CaptionPropertyPanel } from "./captions/components/CaptionPropertyPanel"; import { CaptionTimeline } from "./captions/components/CaptionTimeline"; @@ -45,6 +53,56 @@ interface AppToast { tone: "error" | "info"; } +const DEFAULT_TIMELINE_ASSET_DURATION: Record = { + image: 3, + video: 5, + audio: 5, +}; + +function collectHtmlIds(source: string): string[] { + return Array.from(source.matchAll(/\bid="([^"]+)"/g), (match) => match[1] ?? ""); +} + +async function resolveDroppedAssetDuration( + projectId: string, + assetPath: string, + kind: TimelineAssetKind, +): Promise { + if (kind === "image") return DEFAULT_TIMELINE_ASSET_DURATION.image; + + const media = document.createElement(kind === "video" ? "video" : "audio"); + media.preload = "metadata"; + media.src = `/api/projects/${projectId}/preview/${assetPath}`; + + const duration = await new Promise((resolve) => { + const timeout = window.setTimeout(() => resolve(DEFAULT_TIMELINE_ASSET_DURATION[kind]), 3000); + const finalize = (value: number) => { + window.clearTimeout(timeout); + resolve(value); + }; + + media.addEventListener( + "loadedmetadata", + () => { + const raw = Number(media.duration); + finalize( + Number.isFinite(raw) && raw > 0 + ? Math.round(raw * 100) / 100 + : DEFAULT_TIMELINE_ASSET_DURATION[kind], + ); + }, + { once: true }, + ); + media.addEventListener("error", () => finalize(DEFAULT_TIMELINE_ASSET_DURATION[kind]), { + once: true, + }); + }); + + media.src = ""; + media.load(); + return duration; +} + // ── Main App ── export function StudioApp() { @@ -717,6 +775,128 @@ export function StudioApp() { [activeCompPath], ); + const showToast = useCallback((message: string, tone: AppToast["tone"] = "error") => { + if (toastTimerRef.current) clearTimeout(toastTimerRef.current); + setAppToast({ message, tone }); + toastTimerRef.current = setTimeout(() => setAppToast(null), 4000); + }, []); + + const handleBlockedTimelineEdit = useCallback( + (_element: TimelineElement) => { + const now = Date.now(); + if (now - lastBlockedTimelineToastAtRef.current < 1500) return; + lastBlockedTimelineToastAtRef.current = now; + showToast("This clip can’t be moved or resized from the timeline yet.", "info"); + }, + [showToast], + ); + + const handleTimelineAssetDrop = useCallback( + async (assetPath: string, placement: Pick) => { + const pid = projectIdRef.current; + if (!pid) throw new Error("No active project"); + + const kind = getTimelineAssetKind(assetPath); + if (!kind) { + showToast("Only image, video, and audio assets can be dropped onto the timeline."); + return; + } + + const targetPath = activeCompPath || "index.html"; + try { + const response = await fetch( + `/api/projects/${pid}/files/${encodeURIComponent(targetPath)}`, + ); + if (!response.ok) { + throw new Error(`Failed to read ${targetPath}`); + } + + const data = (await response.json()) as { content?: string }; + const originalContent = data.content; + if (typeof originalContent !== "string") { + throw new Error(`Missing file contents for ${targetPath}`); + } + + const normalizedStart = Number(formatTimelineAttributeNumber(placement.start)); + const normalizedDuration = Number( + formatTimelineAttributeNumber(await resolveDroppedAssetDuration(pid, assetPath, kind)), + ); + const newId = buildTimelineAssetId(assetPath, collectHtmlIds(originalContent)); + const resolvedAssetSrc = resolveTimelineAssetSrc(targetPath, assetPath); + + const resolvedTargetPath = targetPath || "index.html"; + const relevantElements = timelineElements.filter( + (timelineElement) => + (timelineElement.sourceFile || activeCompPath || "index.html") === resolvedTargetPath, + ); + const trackZIndices = buildTrackZIndexMap([ + ...relevantElements.map((timelineElement) => timelineElement.track), + placement.track, + ]); + + let patchedContent = originalContent; + for (const timelineElement of relevantElements) { + const elementTarget = timelineElement.domId + ? { + id: timelineElement.domId, + selector: timelineElement.selector, + selectorIndex: timelineElement.selectorIndex, + } + : timelineElement.selector + ? { + selector: timelineElement.selector, + selectorIndex: timelineElement.selectorIndex, + } + : null; + if (!elementTarget) continue; + const nextZIndex = trackZIndices.get(timelineElement.track); + if (nextZIndex == null) continue; + patchedContent = applyPatchByTarget(patchedContent, elementTarget, { + type: "inline-style", + property: "z-index", + value: String(nextZIndex), + }); + } + + patchedContent = insertTimelineAssetIntoSource( + patchedContent, + buildTimelineAssetInsertHtml({ + id: newId, + assetPath: resolvedAssetSrc, + kind, + start: normalizedStart, + duration: normalizedDuration, + track: placement.track, + zIndex: trackZIndices.get(placement.track) ?? 1, + }), + ); + + const saveResponse = await fetch( + `/api/projects/${pid}/files/${encodeURIComponent(targetPath)}`, + { + method: "PUT", + headers: { "Content-Type": "text/plain" }, + body: patchedContent, + }, + ); + if (!saveResponse.ok) { + throw new Error(`Failed to save ${targetPath}`); + } + + if (editingPathRef.current === targetPath) { + setEditingFile({ path: targetPath, content: patchedContent }); + } + + setRefreshKey((k) => k + 1); + } catch (error) { + const message = + error instanceof Error ? error.message : "Failed to drop asset onto timeline"; + showToast(message); + } + }, + [activeCompPath, showToast, timelineElements], + ); + // ── File Management Handlers ── const refreshFileTree = useCallback(async () => { @@ -840,22 +1020,6 @@ export function StudioApp() { const handleMoveFile = handleRenameFile; - const showToast = useCallback((message: string, tone: AppToast["tone"] = "error") => { - if (toastTimerRef.current) clearTimeout(toastTimerRef.current); - setAppToast({ message, tone }); - toastTimerRef.current = setTimeout(() => setAppToast(null), 4000); - }, []); - - const handleBlockedTimelineEdit = useCallback( - (_element: TimelineElement) => { - const now = Date.now(); - if (now - lastBlockedTimelineToastAtRef.current < 1500) return; - lastBlockedTimelineToastAtRef.current = now; - showToast("This clip can’t be moved or resized from the timeline yet.", "info"); - }, - [showToast], - ); - const handleImportFiles = useCallback( async (files: FileList, dir?: string) => { const pid = projectIdRef.current; @@ -1151,6 +1315,7 @@ export function StudioApp() { activeCompositionPath={activeCompPath} timelineToolbar={timelineToolbar} renderClipContent={renderClipContent} + onAssetDrop={handleTimelineAssetDrop} onMoveElement={handleTimelineElementMove} onResizeElement={handleTimelineElementResize} onBlockedEditAttempt={handleBlockedTimelineEdit} diff --git a/packages/studio/src/components/nle/NLELayout.tsx b/packages/studio/src/components/nle/NLELayout.tsx index 5c2bc21c1..2ece9ff16 100644 --- a/packages/studio/src/components/nle/NLELayout.tsx +++ b/packages/studio/src/components/nle/NLELayout.tsx @@ -28,6 +28,10 @@ interface NLELayoutProps { element: TimelineElement, style: { clip: string; label: string }, ) => ReactNode; + onAssetDrop?: ( + assetPath: string, + placement: Pick, + ) => Promise | void; /** Persist timeline move actions back into source HTML */ onMoveElement?: ( element: TimelineElement, @@ -61,6 +65,7 @@ export const NLELayout = memo(function NLELayout({ onIframeRef, onCompositionChange, renderClipContent, + onAssetDrop, onMoveElement, onResizeElement, onBlockedEditAttempt, @@ -393,6 +398,7 @@ export const NLELayout = memo(function NLELayout({ onSeek={seek} onDrillDown={handleDrillDown} renderClipContent={renderClipContent} + onAssetDrop={onAssetDrop} onMoveElement={onMoveElement} onResizeElement={onResizeElement} onBlockedEditAttempt={onBlockedEditAttempt} diff --git a/packages/studio/src/components/sidebar/AssetsTab.tsx b/packages/studio/src/components/sidebar/AssetsTab.tsx index 6b8fc16f3..eccbf9d88 100644 --- a/packages/studio/src/components/sidebar/AssetsTab.tsx +++ b/packages/studio/src/components/sidebar/AssetsTab.tsx @@ -1,6 +1,7 @@ import { memo, useState, useCallback, useRef } from "react"; import { VideoFrameThumbnail } from "../ui/VideoFrameThumbnail"; import { MEDIA_EXT, IMAGE_EXT, VIDEO_EXT, AUDIO_EXT } from "../../utils/mediaTypes"; +import { TIMELINE_ASSET_MIME } from "../../utils/timelineAssetDrop"; interface AssetsTabProps { projectId: string; @@ -106,7 +107,13 @@ function AssetCard({ return ( <>
onCopy(asset)} + onDragStart={(e) => { + e.dataTransfer.effectAllowed = "copy"; + e.dataTransfer.setData(TIMELINE_ASSET_MIME, JSON.stringify({ path: asset })); + e.dataTransfer.setData("text/plain", asset); + }} onContextMenu={(e) => { e.preventDefault(); setContextMenu({ x: e.clientX, y: e.clientY }); diff --git a/packages/studio/src/player/components/Timeline.test.ts b/packages/studio/src/player/components/Timeline.test.ts index 22ff783e9..d5eca23de 100644 --- a/packages/studio/src/player/components/Timeline.test.ts +++ b/packages/studio/src/player/components/Timeline.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect } from "vitest"; import { generateTicks, + getDefaultDroppedTrack, getTimelineCanvasHeight, + resolveTimelineAssetDrop, getTimelinePlayheadLeft, getTimelineScrollLeftForZoomTransition, shouldAutoScrollTimeline, @@ -162,3 +164,53 @@ describe("getTimelineCanvasHeight", () => { expect(getTimelineCanvasHeight(0)).toBeGreaterThan(24); }); }); + +describe("getDefaultDroppedTrack", () => { + it("defaults to track 0 when there are no rows yet", () => { + expect(getDefaultDroppedTrack([])).toBe(0); + }); + + it("creates a new bottom track when dropped below existing rows", () => { + expect(getDefaultDroppedTrack([0, 1, 5], 10)).toBe(6); + }); +}); + +describe("resolveTimelineAssetDrop", () => { + it("maps drop coordinates to a start time and visible track", () => { + expect( + resolveTimelineAssetDrop( + { + rectLeft: 100, + rectTop: 200, + scrollLeft: 0, + scrollTop: 0, + pixelsPerSecond: 100, + duration: 10, + trackHeight: 72, + trackOrder: [0, 3, 7], + }, + 432, + 310, + ), + ).toEqual({ start: 3, track: 3 }); + }); + + it("can create a new bottom track when dropped below the last visible row", () => { + expect( + resolveTimelineAssetDrop( + { + rectLeft: 100, + rectTop: 200, + scrollLeft: 0, + scrollTop: 0, + pixelsPerSecond: 100, + duration: 10, + trackHeight: 72, + trackOrder: [0, 3, 7], + }, + 250, + 600, + ), + ).toEqual({ start: 1.18, track: 8 }); + }); +}); diff --git a/packages/studio/src/player/components/Timeline.tsx b/packages/studio/src/player/components/Timeline.tsx index d08ebfc92..225d1689b 100644 --- a/packages/studio/src/player/components/Timeline.tsx +++ b/packages/studio/src/player/components/Timeline.tsx @@ -27,6 +27,7 @@ import { type TimelineTheme, } from "./timelineTheme"; import { getTimelinePixelsPerSecond } from "./timelineZoom"; +import { TIMELINE_ASSET_MIME } from "../../utils/timelineAssetDrop"; /* ── Layout ─────────────────────────────────────────────────────── */ const GUTTER = 32; @@ -140,6 +141,42 @@ export function getTimelineCanvasHeight(trackCount: number): number { return RULER_H + Math.max(0, trackCount) * TRACK_H + TIMELINE_SCROLL_BUFFER; } +export function getDefaultDroppedTrack(trackOrder: number[], rowIndex?: number): number { + if (trackOrder.length === 0) return 0; + if (rowIndex == null || rowIndex < 0) return trackOrder[0]; + if (rowIndex >= trackOrder.length) { + return Math.max(...trackOrder) + 1; + } + return trackOrder[rowIndex] ?? trackOrder[trackOrder.length - 1] ?? 0; +} + +export function resolveTimelineAssetDrop( + input: { + rectLeft: number; + rectTop: number; + scrollLeft: number; + scrollTop: number; + pixelsPerSecond: number; + duration: number; + trackHeight: number; + trackOrder: number[]; + }, + clientX: number, + clientY: number, +): { start: number; track: number } { + const x = clientX - input.rectLeft + input.scrollLeft - GUTTER; + const y = clientY - input.rectTop + input.scrollTop - RULER_H; + const start = Math.max( + 0, + Math.min(input.duration, Math.round((x / Math.max(input.pixelsPerSecond, 1)) * 100) / 100), + ); + const rowIndex = Math.floor(y / Math.max(input.trackHeight, 1)); + return { + start, + track: getDefaultDroppedTrack(input.trackOrder, rowIndex), + }; +} + /* ── Component ──────────────────────────────────────────────────── */ interface TimelineProps { /** Called when user seeks via ruler/track click or playhead drag */ @@ -155,6 +192,11 @@ interface TimelineProps { renderClipOverlay?: (element: import("../store/playerStore").TimelineElement) => ReactNode; /** Called when files are dropped onto the empty timeline */ onFileDrop?: (files: File[]) => void; + /** Called when an existing asset is dropped from the Assets tab */ + onAssetDrop?: ( + assetPath: string, + placement: { start: number; track: number }, + ) => Promise | void; /** Persist a clip move back into source HTML */ onMoveElement?: ( element: import("../store/playerStore").TimelineElement, @@ -213,6 +255,7 @@ export const Timeline = memo(function Timeline({ renderClipContent, renderClipOverlay, onFileDrop, + onAssetDrop, onMoveElement, onResizeElement, onBlockedEditAttempt, @@ -833,6 +876,55 @@ export const Timeline = memo(function Timeline({ ); const [isDragOver, setIsDragOver] = useState(false); + const handleAssetDragOver = useCallback((e: React.DragEvent) => { + const hasFiles = e.dataTransfer.files.length > 0; + const hasAsset = Array.from(e.dataTransfer.types).includes(TIMELINE_ASSET_MIME); + if (!hasFiles && !hasAsset) return; + e.preventDefault(); + if (hasAsset) { + e.dataTransfer.dropEffect = "copy"; + } + setIsDragOver(true); + }, []); + + const handleAssetDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + setIsDragOver(false); + if (onFileDrop && e.dataTransfer.files.length > 0) { + onFileDrop(Array.from(e.dataTransfer.files)); + return; + } + + const assetPayload = e.dataTransfer.getData(TIMELINE_ASSET_MIME); + if (!assetPayload || !onAssetDrop) return; + try { + const parsed = JSON.parse(assetPayload) as { path?: string }; + if (!parsed.path) return; + const scroll = scrollRef.current; + const rect = scroll?.getBoundingClientRect(); + if (!scroll || !rect) return; + const placement = resolveTimelineAssetDrop( + { + rectLeft: rect.left, + rectTop: rect.top, + scrollLeft: scroll.scrollLeft, + scrollTop: scroll.scrollTop, + pixelsPerSecond: ppsRef.current, + duration: durationRef.current, + trackHeight: TRACK_H, + trackOrder: trackOrderRef.current, + }, + e.clientX, + e.clientY, + ); + void onAssetDrop(parsed.path, placement); + } catch { + // ignore malformed drag payloads + } + }, + [onAssetDrop, onFileDrop], + ); if (!timelineReady || elements.length === 0) { return ( @@ -840,18 +932,9 @@ export const Timeline = memo(function Timeline({ className={`h-full border-t bg-[#0a0a0b] flex flex-col select-none transition-colors duration-150 ${ isDragOver ? "border-studio-accent/50 bg-studio-accent/[0.03]" : "border-neutral-800/50" }`} - onDragOver={(e) => { - e.preventDefault(); - setIsDragOver(true); - }} + onDragOver={handleAssetDragOver} onDragLeave={() => setIsDragOver(false)} - onDrop={(e) => { - e.preventDefault(); - setIsDragOver(false); - if (onFileDrop && e.dataTransfer.files.length > 0) { - onFileDrop(Array.from(e.dataTransfer.files)); - } - }} + onDrop={handleAssetDrop} > {/* Ruler */}
setIsDragOver(false)} + onDrop={handleAssetDrop} onPointerDown={handlePointerDown} onPointerMove={handlePointerMove} onPointerUp={handlePointerUp} diff --git a/packages/studio/src/utils/timelineAssetDrop.test.ts b/packages/studio/src/utils/timelineAssetDrop.test.ts new file mode 100644 index 000000000..e045abf71 --- /dev/null +++ b/packages/studio/src/utils/timelineAssetDrop.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { + buildTimelineAssetInsertHtml, + getTimelineAssetKind, + insertTimelineAssetIntoSource, + resolveTimelineAssetSrc, +} from "./timelineAssetDrop"; + +describe("getTimelineAssetKind", () => { + it("detects image, video, and audio assets", () => { + expect(getTimelineAssetKind("assets/photo.png")).toBe("image"); + expect(getTimelineAssetKind("assets/clip.mp4")).toBe("video"); + expect(getTimelineAssetKind("assets/music.wav")).toBe("audio"); + }); +}); + +describe("buildTimelineAssetInsertHtml", () => { + it("builds an image clip with explicit timing and track", () => { + expect( + buildTimelineAssetInsertHtml({ + id: "photo_asset", + assetPath: "assets/photo.png", + kind: "image", + start: 1.25, + duration: 3, + track: 2, + zIndex: 4, + }), + ).toContain('img id="photo_asset"'); + }); + + it("builds an audio clip without visual layout styles", () => { + const html = buildTimelineAssetInsertHtml({ + id: "music_asset", + assetPath: "assets/music.wav", + kind: "audio", + start: 0.5, + duration: 5, + track: 0, + zIndex: 1, + }); + expect(html).toContain(" { + it("keeps project-root asset paths for index.html", () => { + expect(resolveTimelineAssetSrc("index.html", "assets/photo.png")).toBe("assets/photo.png"); + }); + + it("rewrites asset paths relative to sub-compositions", () => { + expect(resolveTimelineAssetSrc("compositions/scene-a.html", "assets/photo.png")).toBe( + "../assets/photo.png", + ); + }); +}); + +describe("insertTimelineAssetIntoSource", () => { + it("appends the new asset inside the root composition", () => { + const source = `
`; + const html = insertTimelineAssetIntoSource( + source, + '', + ); + + expect(html).toContain('
): string { + const baseName = assetPath.split("/").pop() ?? "asset"; + const normalized = baseName + .replace(/\.[^.]+$/, "") + .replace(/[^a-zA-Z0-9_-]+/g, "_") + .replace(/^_+|_+$/g, "") + .toLowerCase(); + const baseId = normalized || "asset"; + const ids = new Set(existingIds); + if (!ids.has(baseId)) return baseId; + let suffix = 2; + while (ids.has(`${baseId}_${suffix}`)) suffix += 1; + return `${baseId}_${suffix}`; +} + +export function resolveTimelineAssetSrc(targetPath: string, assetPath: string): string { + const targetDir = targetPath.includes("/") + ? targetPath.slice(0, targetPath.lastIndexOf("/")) + : ""; + if (!targetDir) return assetPath; + + const fromParts = targetDir.split("/").filter(Boolean); + const toParts = assetPath.split("/").filter(Boolean); + while (fromParts.length > 0 && toParts.length > 0 && fromParts[0] === toParts[0]) { + fromParts.shift(); + toParts.shift(); + } + + const up = fromParts.map(() => ".."); + const relative = [...up, ...toParts].join("/"); + return relative || assetPath.split("/").pop() || assetPath; +} + +export function buildTimelineAssetInsertHtml(input: { + id: string; + assetPath: string; + kind: TimelineAssetKind; + start: number; + duration: number; + track: number; + zIndex: number; +}): string { + const sharedAttrs = `id="${input.id}" class="clip" src="${input.assetPath}" data-start="${input.start}" data-duration="${input.duration}" data-track-index="${input.track}"`; + + if (input.kind === "image") { + return ``; + } + + if (input.kind === "video") { + return ``; + } + + return ``; +} + +export function insertTimelineAssetIntoSource(source: string, assetHtml: string): string { + const rootOpenTag = /<[^>]*data-composition-id="[^"]+"[^>]*>/i; + const match = rootOpenTag.exec(source); + if (!match || match.index == null) { + throw new Error("No composition root found in target source"); + } + const insertAt = match.index + match[0].length; + return `${source.slice(0, insertAt)}${assetHtml}${source.slice(insertAt)}`; +} From 273e7d349bf229403cc88d7ff3fa587fda56713f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 23 Apr 2026 19:04:18 -0400 Subject: [PATCH 2/7] feat(studio): support delete key for timeline clips --- .../studio-api/helpers/sourceMutation.test.ts | 30 +++++ .../src/studio-api/helpers/sourceMutation.ts | 42 +++++++ packages/core/src/studio-api/routes/files.ts | 38 ++++++ packages/studio/src/App.tsx | 115 ++++++++++++++++++ .../studio/src/components/nle/NLELayout.tsx | 3 + .../src/player/components/Timeline.test.ts | 21 ++++ .../studio/src/player/components/Timeline.tsx | 63 ++++++++++ 7 files changed, 312 insertions(+) create mode 100644 packages/core/src/studio-api/helpers/sourceMutation.test.ts create mode 100644 packages/core/src/studio-api/helpers/sourceMutation.ts diff --git a/packages/core/src/studio-api/helpers/sourceMutation.test.ts b/packages/core/src/studio-api/helpers/sourceMutation.test.ts new file mode 100644 index 000000000..407d948a5 --- /dev/null +++ b/packages/core/src/studio-api/helpers/sourceMutation.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { removeElementFromHtml } from "./sourceMutation.js"; + +describe("removeElementFromHtml", () => { + it("removes a self-closing element by id", () => { + const html = `
`; + + const updated = removeElementFromHtml(html, { id: "photo" }); + + expect(updated).not.toContain(`id="photo"`); + expect(updated).toContain(`id="rest"`); + }); + + it("removes a matched composition host by selector", () => { + const html = `
Scene A
`; + + const updated = removeElementFromHtml(html, { + selector: '[data-composition-id="scene-a"]', + }); + + expect(updated).not.toContain(`data-composition-id="scene-a"`); + expect(updated).toContain(`data-composition-id="scene-b"`); + }); + + it("supports fragment html by returning updated body markup", () => { + const html = `
`; + + expect(removeElementFromHtml(html, { id: "photo" })).toBe(`
`); + }); +}); diff --git a/packages/core/src/studio-api/helpers/sourceMutation.ts b/packages/core/src/studio-api/helpers/sourceMutation.ts new file mode 100644 index 000000000..cefda05ce --- /dev/null +++ b/packages/core/src/studio-api/helpers/sourceMutation.ts @@ -0,0 +1,42 @@ +import { parseHTML } from "linkedom"; + +export interface SourceMutationTarget { + id?: string | null; + selector?: string; + selectorIndex?: number; +} + +function parseSourceDocument(source: string): { document: Document; wrappedFragment: boolean } { + const hasDocumentShell = /]/i.test(source); + if (hasDocumentShell) { + return { document: parseHTML(source).document, wrappedFragment: false }; + } + return { + document: parseHTML(`${source}`).document, + wrappedFragment: true, + }; +} + +function findTargetElement(document: Document, target: SourceMutationTarget): Element | null { + if (target.id) { + const byId = document.getElementById(target.id); + if (byId) return byId; + } + + if (!target.selector) return null; + try { + const matches = Array.from(document.querySelectorAll(target.selector)); + return matches[target.selectorIndex ?? 0] ?? null; + } catch { + return null; + } +} + +export function removeElementFromHtml(source: string, target: SourceMutationTarget): string { + const { document, wrappedFragment } = parseSourceDocument(source); + const element = findTargetElement(document, target); + if (!element) return source; + + element.remove(); + return wrappedFragment ? document.body.innerHTML || "" : document.toString(); +} diff --git a/packages/core/src/studio-api/routes/files.ts b/packages/core/src/studio-api/routes/files.ts index 0abfec0d0..760eff75f 100644 --- a/packages/core/src/studio-api/routes/files.ts +++ b/packages/core/src/studio-api/routes/files.ts @@ -14,6 +14,7 @@ import { import { resolve, dirname, join } from "node:path"; import type { StudioApiAdapter } from "../types.js"; import { isSafePath } from "../helpers/safePath.js"; +import { removeElementFromHtml } from "../helpers/sourceMutation.js"; // ── Shared helpers ────────────────────────────────────────────────────────── @@ -184,6 +185,43 @@ export function registerFileRoutes(api: Hono, adapter: StudioApiAdapter): void { return c.json({ ok: true }); }); + api.post("/projects/:id/file-mutations/remove-element/*", async (c) => { + const id = c.req.param("id"); + const project = await adapter.resolveProject(id); + if (!project) return c.json({ error: "not found" }, 404); + + const filePath = decodeURIComponent( + c.req.path.replace(`/projects/${project.id}/file-mutations/remove-element/`, ""), + ); + if (filePath.includes("\0")) { + return c.json({ error: "forbidden" }, 403); + } + + const absPath = resolve(project.dir, filePath); + if (!isSafePath(project.dir, absPath)) { + return c.json({ error: "forbidden" }, 403); + } + if (!existsSync(absPath)) { + return c.json({ error: "not found" }, 404); + } + + const body = (await c.req.json().catch(() => null)) as { + target?: { id?: string | null; selector?: string; selectorIndex?: number }; + } | null; + if (!body?.target) { + return c.json({ error: "target required" }, 400); + } + + const originalContent = readFileSync(absPath, "utf-8"); + const patchedContent = removeElementFromHtml(originalContent, body.target); + if (patchedContent === originalContent) { + return c.json({ ok: true, changed: false, content: originalContent }); + } + + writeFileSync(absPath, patchedContent, "utf-8"); + return c.json({ ok: true, changed: true, content: patchedContent }); + }); + // ── Rename / Move ── api.patch("/projects/:id/files/*", async (c) => { diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index cb8fe189f..1e6fe183b 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -781,6 +781,120 @@ export function StudioApp() { toastTimerRef.current = setTimeout(() => setAppToast(null), 4000); }, []); + const handleTimelineElementDelete = useCallback( + async (element: TimelineElement) => { + const pid = projectIdRef.current; + if (!pid) throw new Error("No active project"); + + const targetPath = element.sourceFile || activeCompPath || "index.html"; + try { + const response = await fetch( + `/api/projects/${pid}/files/${encodeURIComponent(targetPath)}`, + ); + if (!response.ok) { + throw new Error(`Failed to read ${targetPath}`); + } + + const data = (await response.json()) as { content?: string }; + const originalContent = data.content; + if (typeof originalContent !== "string") { + throw new Error(`Missing file contents for ${targetPath}`); + } + + const patchTarget = element.domId + ? { id: element.domId, selector: element.selector, selectorIndex: element.selectorIndex } + : element.selector + ? { selector: element.selector, selectorIndex: element.selectorIndex } + : null; + if (!patchTarget) { + throw new Error(`Timeline element ${element.id} is missing a patchable target`); + } + + const resolvedTargetPath = targetPath || "index.html"; + const remainingElements = timelineElements.filter( + (timelineElement) => + (timelineElement.key ?? timelineElement.id) !== (element.key ?? element.id) && + (timelineElement.sourceFile || activeCompPath || "index.html") === resolvedTargetPath, + ); + const trackZIndices = buildTrackZIndexMap( + remainingElements.map((timelineElement) => timelineElement.track), + ); + + const removeResponse = await fetch( + `/api/projects/${pid}/file-mutations/remove-element/${encodeURIComponent(targetPath)}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ target: patchTarget }), + }, + ); + if (!removeResponse.ok) { + throw new Error(`Failed to delete ${element.id} from ${targetPath}`); + } + + const removeData = (await removeResponse.json()) as { + changed?: boolean; + content?: string; + }; + let patchedContent = + typeof removeData.content === "string" ? removeData.content : originalContent; + for (const timelineElement of remainingElements) { + const elementTarget = timelineElement.domId + ? { + id: timelineElement.domId, + selector: timelineElement.selector, + selectorIndex: timelineElement.selectorIndex, + } + : timelineElement.selector + ? { + selector: timelineElement.selector, + selectorIndex: timelineElement.selectorIndex, + } + : null; + if (!elementTarget) continue; + const nextZIndex = trackZIndices.get(timelineElement.track); + if (nextZIndex == null) continue; + patchedContent = applyPatchByTarget(patchedContent, elementTarget, { + type: "inline-style", + property: "z-index", + value: String(nextZIndex), + }); + } + + const saveResponse = await fetch( + `/api/projects/${pid}/files/${encodeURIComponent(targetPath)}`, + { + method: "PUT", + headers: { "Content-Type": "text/plain" }, + body: patchedContent, + }, + ); + if (!saveResponse.ok) { + throw new Error(`Failed to save ${targetPath}`); + } + + if (editingPathRef.current === targetPath) { + setEditingFile({ path: targetPath, content: patchedContent }); + } + + usePlayerStore + .getState() + .setElements( + timelineElements.filter( + (timelineElement) => + (timelineElement.key ?? timelineElement.id) !== (element.key ?? element.id), + ), + ); + usePlayerStore.getState().setSelectedElementId(null); + setRefreshKey((k) => k + 1); + } catch (error) { + const message = error instanceof Error ? error.message : "Failed to delete timeline clip"; + showToast(message); + } + }, + [activeCompPath, showToast, timelineElements], + ); + const handleBlockedTimelineEdit = useCallback( (_element: TimelineElement) => { const now = Date.now(); @@ -1315,6 +1429,7 @@ export function StudioApp() { activeCompositionPath={activeCompPath} timelineToolbar={timelineToolbar} renderClipContent={renderClipContent} + onDeleteElement={handleTimelineElementDelete} onAssetDrop={handleTimelineAssetDrop} onMoveElement={handleTimelineElementMove} onResizeElement={handleTimelineElementResize} diff --git a/packages/studio/src/components/nle/NLELayout.tsx b/packages/studio/src/components/nle/NLELayout.tsx index 2ece9ff16..b94c50e7f 100644 --- a/packages/studio/src/components/nle/NLELayout.tsx +++ b/packages/studio/src/components/nle/NLELayout.tsx @@ -28,6 +28,7 @@ interface NLELayoutProps { element: TimelineElement, style: { clip: string; label: string }, ) => ReactNode; + onDeleteElement?: (element: TimelineElement) => Promise | void; onAssetDrop?: ( assetPath: string, placement: Pick, @@ -65,6 +66,7 @@ export const NLELayout = memo(function NLELayout({ onIframeRef, onCompositionChange, renderClipContent, + onDeleteElement, onAssetDrop, onMoveElement, onResizeElement, @@ -398,6 +400,7 @@ export const NLELayout = memo(function NLELayout({ onSeek={seek} onDrillDown={handleDrillDown} renderClipContent={renderClipContent} + onDeleteElement={onDeleteElement} onAssetDrop={onAssetDrop} onMoveElement={onMoveElement} onResizeElement={onResizeElement} diff --git a/packages/studio/src/player/components/Timeline.test.ts b/packages/studio/src/player/components/Timeline.test.ts index d5eca23de..3151f4a0b 100644 --- a/packages/studio/src/player/components/Timeline.test.ts +++ b/packages/studio/src/player/components/Timeline.test.ts @@ -6,6 +6,7 @@ import { resolveTimelineAssetDrop, getTimelinePlayheadLeft, getTimelineScrollLeftForZoomTransition, + shouldHandleTimelineDeleteKey, shouldAutoScrollTimeline, } from "./Timeline"; import { formatTime } from "../lib/time"; @@ -165,6 +166,26 @@ describe("getTimelineCanvasHeight", () => { }); }); +describe("shouldHandleTimelineDeleteKey", () => { + it("handles Delete and Backspace when focus is not in an editor", () => { + expect(shouldHandleTimelineDeleteKey({ key: "Delete" })).toBe(true); + expect(shouldHandleTimelineDeleteKey({ key: "Backspace" })).toBe(true); + }); + + it("ignores modifier shortcuts", () => { + expect(shouldHandleTimelineDeleteKey({ key: "Delete", metaKey: true })).toBe(false); + expect(shouldHandleTimelineDeleteKey({ key: "Backspace", ctrlKey: true })).toBe(false); + }); + + it("ignores input and editable targets", () => { + const input = { tagName: "INPUT", isContentEditable: false }; + const editable = { tagName: "DIV", isContentEditable: true }; + + expect(shouldHandleTimelineDeleteKey({ key: "Delete", target: input })).toBe(false); + expect(shouldHandleTimelineDeleteKey({ key: "Delete", target: editable })).toBe(false); + }); +}); + describe("getDefaultDroppedTrack", () => { it("defaults to track 0 when there are no rows yet", () => { expect(getDefaultDroppedTrack([])).toBe(0); diff --git a/packages/studio/src/player/components/Timeline.tsx b/packages/studio/src/player/components/Timeline.tsx index 225d1689b..6821aff1e 100644 --- a/packages/studio/src/player/components/Timeline.tsx +++ b/packages/studio/src/player/components/Timeline.tsx @@ -141,6 +141,34 @@ export function getTimelineCanvasHeight(trackCount: number): number { return RULER_H + Math.max(0, trackCount) * TRACK_H + TIMELINE_SCROLL_BUFFER; } +export function shouldHandleTimelineDeleteKey(input: { + key: string; + metaKey?: boolean; + ctrlKey?: boolean; + altKey?: boolean; + target?: EventTarget | null; +}): boolean { + if (input.key !== "Delete" && input.key !== "Backspace") return false; + if (input.metaKey || input.ctrlKey || input.altKey) return false; + const target = + input.target && typeof input.target === "object" + ? (input.target as { + tagName?: string; + isContentEditable?: boolean; + closest?: (selector: string) => Element | null; + }) + : null; + if (target) { + const tag = target.tagName?.toLowerCase() ?? ""; + if (target.isContentEditable) return false; + if (["input", "textarea", "select"].includes(tag)) return false; + if (typeof target.closest === "function" && target.closest("[contenteditable='true']")) { + return false; + } + } + return true; +} + export function getDefaultDroppedTrack(trackOrder: number[], rowIndex?: number): number { if (trackOrder.length === 0) return 0; if (rowIndex == null || rowIndex < 0) return trackOrder[0]; @@ -198,6 +226,9 @@ interface TimelineProps { placement: { start: number; track: number }, ) => Promise | void; /** Persist a clip move back into source HTML */ + onDeleteElement?: ( + element: import("../store/playerStore").TimelineElement, + ) => Promise | void; onMoveElement?: ( element: import("../store/playerStore").TimelineElement, updates: Pick, @@ -256,6 +287,7 @@ export const Timeline = memo(function Timeline({ renderClipOverlay, onFileDrop, onAssetDrop, + onDeleteElement, onMoveElement, onResizeElement, onBlockedEditAttempt, @@ -306,10 +338,13 @@ export const Timeline = memo(function Timeline({ const resizingClipRef = useRef(null); resizingClipRef.current = resizingClip; const blockedClipRef = useRef(null); + const deleteInFlightRef = useRef(false); const onMoveElementRef = useRef(onMoveElement); onMoveElementRef.current = onMoveElement; const onResizeElementRef = useRef(onResizeElement); onResizeElementRef.current = onResizeElement; + const onDeleteElementRef = useRef(onDeleteElement); + onDeleteElementRef.current = onDeleteElement; const suppressClickRef = useRef(false); const [showPopover, setShowPopover] = useState(false); const [viewportWidth, setViewportWidth] = useState(0); @@ -380,6 +415,12 @@ export const Timeline = memo(function Timeline({ } return [...trackOrder, draggedClip.previewTrack].sort((a, b) => a - b); }, [draggedClip, trackOrder]); + const selectedElement = useMemo( + () => elements.find((element) => (element.key ?? element.id) === selectedElementId) ?? null, + [elements, selectedElementId], + ); + const selectedElementRef = useRef(selectedElement); + selectedElementRef.current = selectedElement; // Calculate effective pixels per second // In fit mode, use clientWidth (excludes scrollbar) with a small padding @@ -786,6 +827,28 @@ export const Timeline = memo(function Timeline({ }; }); + useMountEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + if (!shouldHandleTimelineDeleteKey(event)) return; + const selected = selectedElementRef.current; + const onDelete = onDeleteElementRef.current; + if (!selected || !onDelete || deleteInFlightRef.current) return; + event.preventDefault(); + deleteInFlightRef.current = true; + suppressClickRef.current = true; + setShowPopover(false); + setRangeSelection(null); + Promise.resolve(onDelete(selected)).finally(() => { + deleteInFlightRef.current = false; + requestAnimationFrame(() => { + suppressClickRef.current = false; + }); + }); + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }); + const handlePointerDown = useCallback( (e: React.PointerEvent) => { if (e.button !== 0) return; From 82a70a098828aad22dfb20c30f3442ef025ebc73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 23 Apr 2026 19:24:30 -0400 Subject: [PATCH 3/7] feat(studio): place external file drops onto timeline --- packages/studio/src/App.tsx | 105 +++++++++++------- .../studio/src/components/nle/NLELayout.tsx | 6 + .../studio/src/player/components/Timeline.tsx | 26 ++++- .../src/utils/timelineAssetDrop.test.ts | 11 ++ .../studio/src/utils/timelineAssetDrop.ts | 10 ++ 5 files changed, 117 insertions(+), 41 deletions(-) diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index 1e6fe183b..53d216ddc 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -16,6 +16,7 @@ import { isMediaFile } from "./utils/mediaTypes"; import { buildTimelineAssetId, buildTimelineAssetInsertHtml, + buildTimelineFileDropPlacements, getTimelineAssetKind, insertTimelineAssetIntoSource, resolveTimelineAssetSrc, @@ -905,6 +906,52 @@ export function StudioApp() { [showToast], ); + const refreshFileTree = useCallback(async () => { + const pid = projectIdRef.current; + if (!pid) return; + const res = await fetch(`/api/projects/${pid}`); + const data = await res.json(); + if (data.files) setFileTree(data.files); + }, []); + + const uploadProjectFiles = useCallback( + async (files: Iterable, dir?: string): Promise => { + const pid = projectIdRef.current; + const fileList = Array.from(files); + if (!pid || fileList.length === 0) return []; + + const formData = new FormData(); + for (const file of fileList) { + formData.append("file", file); + } + + const qs = dir ? `?dir=${encodeURIComponent(dir)}` : ""; + try { + const res = await fetch(`/api/projects/${pid}/upload${qs}`, { + method: "POST", + body: formData, + }); + if (res.ok) { + const data = await res.json(); + if (data.skipped?.length) { + showToast(`Skipped (too large): ${data.skipped.join(", ")}`); + } + await refreshFileTree(); + setRefreshKey((k) => k + 1); + return Array.isArray(data.files) ? data.files : []; + } else if (res.status === 413) { + showToast("Upload rejected: payload too large"); + } else { + showToast(`Upload failed (${res.status})`); + } + } catch { + showToast("Upload failed: network error"); + } + return []; + }, + [refreshFileTree, showToast], + ); + const handleTimelineAssetDrop = useCallback( async (assetPath: string, placement: Pick) => { const pid = projectIdRef.current; @@ -1011,15 +1058,22 @@ export function StudioApp() { [activeCompPath, showToast, timelineElements], ); - // ── File Management Handlers ── + const handleTimelineFileDrop = useCallback( + async (files: File[], placement?: Pick) => { + const uploaded = await uploadProjectFiles(files); + if (uploaded.length === 0) return; + const placements = buildTimelineFileDropPlacements( + placement ?? { start: 0, track: 0 }, + uploaded.length, + ); + for (const [index, assetPath] of uploaded.entries()) { + await handleTimelineAssetDrop(assetPath, placements[index] ?? placements[0]); + } + }, + [handleTimelineAssetDrop, uploadProjectFiles], + ); - const refreshFileTree = useCallback(async () => { - const pid = projectIdRef.current; - if (!pid) return; - const res = await fetch(`/api/projects/${pid}`); - const data = await res.json(); - if (data.files) setFileTree(data.files); - }, []); + // ── File Management Handlers ── const handleCreateFile = useCallback( async (path: string) => { @@ -1135,38 +1189,10 @@ export function StudioApp() { const handleMoveFile = handleRenameFile; const handleImportFiles = useCallback( - async (files: FileList, dir?: string) => { - const pid = projectIdRef.current; - if (!pid || files.length === 0) return; - - const formData = new FormData(); - for (const file of Array.from(files)) { - formData.append("file", file); - } - - const qs = dir ? `?dir=${encodeURIComponent(dir)}` : ""; - try { - const res = await fetch(`/api/projects/${pid}/upload${qs}`, { - method: "POST", - body: formData, - }); - if (res.ok) { - const data = await res.json(); - if (data.skipped?.length) { - showToast(`Skipped (too large): ${data.skipped.join(", ")}`); - } - await refreshFileTree(); - setRefreshKey((k) => k + 1); - } else if (res.status === 413) { - showToast("Upload rejected: payload too large"); - } else { - showToast(`Upload failed (${res.status})`); - } - } catch { - showToast("Upload failed: network error"); - } + async (files: FileList | File[], dir?: string) => { + void uploadProjectFiles(Array.from(files), dir); }, - [refreshFileTree, showToast], + [uploadProjectFiles], ); const handleLint = useCallback(async () => { @@ -1431,6 +1457,7 @@ export function StudioApp() { renderClipContent={renderClipContent} onDeleteElement={handleTimelineElementDelete} onAssetDrop={handleTimelineAssetDrop} + onFileDrop={handleTimelineFileDrop} onMoveElement={handleTimelineElementMove} onResizeElement={handleTimelineElementResize} onBlockedEditAttempt={handleBlockedTimelineEdit} diff --git a/packages/studio/src/components/nle/NLELayout.tsx b/packages/studio/src/components/nle/NLELayout.tsx index b94c50e7f..2f29ff2cd 100644 --- a/packages/studio/src/components/nle/NLELayout.tsx +++ b/packages/studio/src/components/nle/NLELayout.tsx @@ -28,6 +28,10 @@ interface NLELayoutProps { element: TimelineElement, style: { clip: string; label: string }, ) => ReactNode; + onFileDrop?: ( + files: File[], + placement?: Pick, + ) => Promise | void; onDeleteElement?: (element: TimelineElement) => Promise | void; onAssetDrop?: ( assetPath: string, @@ -66,6 +70,7 @@ export const NLELayout = memo(function NLELayout({ onIframeRef, onCompositionChange, renderClipContent, + onFileDrop, onDeleteElement, onAssetDrop, onMoveElement, @@ -400,6 +405,7 @@ export const NLELayout = memo(function NLELayout({ onSeek={seek} onDrillDown={handleDrillDown} renderClipContent={renderClipContent} + onFileDrop={onFileDrop} onDeleteElement={onDeleteElement} onAssetDrop={onAssetDrop} onMoveElement={onMoveElement} diff --git a/packages/studio/src/player/components/Timeline.tsx b/packages/studio/src/player/components/Timeline.tsx index 6821aff1e..7dba1e138 100644 --- a/packages/studio/src/player/components/Timeline.tsx +++ b/packages/studio/src/player/components/Timeline.tsx @@ -219,7 +219,10 @@ interface TimelineProps { /** Optional overlay renderer for clips (e.g. badges, cursors) */ renderClipOverlay?: (element: import("../store/playerStore").TimelineElement) => ReactNode; /** Called when files are dropped onto the empty timeline */ - onFileDrop?: (files: File[]) => void; + onFileDrop?: ( + files: File[], + placement?: { start: number; track: number }, + ) => Promise | void; /** Called when an existing asset is dropped from the Assets tab */ onAssetDrop?: ( assetPath: string, @@ -955,7 +958,26 @@ export const Timeline = memo(function Timeline({ e.preventDefault(); setIsDragOver(false); if (onFileDrop && e.dataTransfer.files.length > 0) { - onFileDrop(Array.from(e.dataTransfer.files)); + const scroll = scrollRef.current; + const rect = scroll?.getBoundingClientRect(); + const placement = + scroll && rect + ? resolveTimelineAssetDrop( + { + rectLeft: rect.left, + rectTop: rect.top, + scrollLeft: scroll.scrollLeft, + scrollTop: scroll.scrollTop, + pixelsPerSecond: ppsRef.current, + duration: durationRef.current, + trackHeight: TRACK_H, + trackOrder: trackOrderRef.current, + }, + e.clientX, + e.clientY, + ) + : undefined; + void onFileDrop(Array.from(e.dataTransfer.files), placement); return; } diff --git a/packages/studio/src/utils/timelineAssetDrop.test.ts b/packages/studio/src/utils/timelineAssetDrop.test.ts index e045abf71..e8aad53b7 100644 --- a/packages/studio/src/utils/timelineAssetDrop.test.ts +++ b/packages/studio/src/utils/timelineAssetDrop.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + buildTimelineFileDropPlacements, buildTimelineAssetInsertHtml, getTimelineAssetKind, insertTimelineAssetIntoSource, @@ -56,6 +57,16 @@ describe("resolveTimelineAssetSrc", () => { }); }); +describe("buildTimelineFileDropPlacements", () => { + it("uses the dropped start and stacks multiple files onto successive tracks", () => { + expect(buildTimelineFileDropPlacements({ start: 1.5, track: 2 }, 3)).toEqual([ + { start: 1.5, track: 2 }, + { start: 1.5, track: 3 }, + { start: 1.5, track: 4 }, + ]); + }); +}); + describe("insertTimelineAssetIntoSource", () => { it("appends the new asset inside the root composition", () => { const source = `
`; diff --git a/packages/studio/src/utils/timelineAssetDrop.ts b/packages/studio/src/utils/timelineAssetDrop.ts index 4a1489665..9466158fd 100644 --- a/packages/studio/src/utils/timelineAssetDrop.ts +++ b/packages/studio/src/utils/timelineAssetDrop.ts @@ -44,6 +44,16 @@ export function resolveTimelineAssetSrc(targetPath: string, assetPath: string): return relative || assetPath.split("/").pop() || assetPath; } +export function buildTimelineFileDropPlacements( + placement: { start: number; track: number }, + count: number, +): Array<{ start: number; track: number }> { + return Array.from({ length: Math.max(0, count) }, (_, index) => ({ + start: placement.start, + track: placement.track + index, + })); +} + export function buildTimelineAssetInsertHtml(input: { id: string; assetPath: string; From 461ed9ad0b9c49b77b01535a06ffd52e751981be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 23 Apr 2026 19:35:56 -0400 Subject: [PATCH 4/7] fix(studio): reject invalid media uploads --- .../helpers/mediaValidation.test.ts | 53 ++++++++++++++++ .../src/studio-api/helpers/mediaValidation.ts | 61 +++++++++++++++++++ packages/core/src/studio-api/routes/files.ts | 10 ++- packages/studio/src/App.tsx | 4 ++ packages/studio/vite.config.ts | 1 + 5 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/studio-api/helpers/mediaValidation.test.ts create mode 100644 packages/core/src/studio-api/helpers/mediaValidation.ts diff --git a/packages/core/src/studio-api/helpers/mediaValidation.test.ts b/packages/core/src/studio-api/helpers/mediaValidation.test.ts new file mode 100644 index 000000000..dcba462ee --- /dev/null +++ b/packages/core/src/studio-api/helpers/mediaValidation.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { validateUploadedMedia } from "./mediaValidation.js"; + +describe("validateUploadedMedia", () => { + it("passes through non-media files", () => { + expect( + validateUploadedMedia("/tmp/test.svg", () => ({ status: 0, stdout: "", stderr: "" })), + ).toEqual({ + ok: true, + }); + }); + + it("accepts video files with a video stream", () => { + expect( + validateUploadedMedia("/tmp/test.mp4", () => ({ + status: 0, + stdout: JSON.stringify({ streams: [{ codec_type: "video" }] }), + stderr: "", + })), + ).toEqual({ ok: true }); + }); + + it("rejects video files with no supported video stream", () => { + expect( + validateUploadedMedia("/tmp/test.mp4", () => ({ + status: 0, + stdout: JSON.stringify({ streams: [] }), + stderr: "", + })), + ).toEqual({ ok: false, reason: "no supported video stream found" }); + }); + + it("accepts audio files with an audio stream", () => { + expect( + validateUploadedMedia("/tmp/test.wav", () => ({ + status: 0, + stdout: JSON.stringify({ streams: [{ codec_type: "audio" }] }), + stderr: "", + })), + ).toEqual({ ok: true }); + }); + + it("does not block upload when ffprobe is unavailable", () => { + expect( + validateUploadedMedia("/tmp/test.mp4", () => ({ + status: null, + stdout: "", + stderr: "", + error: { code: "ENOENT" } as NodeJS.ErrnoException, + })), + ).toEqual({ ok: true }); + }); +}); diff --git a/packages/core/src/studio-api/helpers/mediaValidation.ts b/packages/core/src/studio-api/helpers/mediaValidation.ts new file mode 100644 index 000000000..3f84b3133 --- /dev/null +++ b/packages/core/src/studio-api/helpers/mediaValidation.ts @@ -0,0 +1,61 @@ +import { spawnSync } from "node:child_process"; + +const VIDEO_EXT = /\.(mp4|webm|mov)$/i; +const AUDIO_EXT = /\.(mp3|wav|ogg|m4a|aac)$/i; + +type FfprobeRunner = ( + command: string, + args: string[], +) => { + status: number | null; + stdout: string | Buffer; + stderr: string | Buffer; + error?: NodeJS.ErrnoException; +}; + +export function validateUploadedMedia( + filePath: string, + runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner, +): { ok: true } | { ok: false; reason: string } { + const isVideo = VIDEO_EXT.test(filePath); + const isAudio = AUDIO_EXT.test(filePath); + if (!isVideo && !isAudio) { + return { ok: true }; + } + + const result = runner("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=codec_type", + "-of", + "json", + filePath, + ]); + + if (result.error?.code === "ENOENT") { + return { ok: true }; + } + if (result.status !== 0) { + return { ok: false, reason: "ffprobe failed to read the media file" }; + } + + try { + const parsed = JSON.parse(String(result.stdout || "{}")) as { + streams?: Array<{ codec_type?: string }>; + }; + const streams = parsed.streams ?? []; + const hasVideo = streams.some((stream) => stream.codec_type === "video"); + const hasAudio = streams.some((stream) => stream.codec_type === "audio"); + + if (isVideo && !hasVideo) { + return { ok: false, reason: "no supported video stream found" }; + } + if (isAudio && !hasAudio) { + return { ok: false, reason: "no supported audio stream found" }; + } + return { ok: true }; + } catch { + return { ok: false, reason: "ffprobe returned unreadable media metadata" }; + } +} diff --git a/packages/core/src/studio-api/routes/files.ts b/packages/core/src/studio-api/routes/files.ts index 760eff75f..7b4b17c5e 100644 --- a/packages/core/src/studio-api/routes/files.ts +++ b/packages/core/src/studio-api/routes/files.ts @@ -13,6 +13,7 @@ import { } from "node:fs"; import { resolve, dirname, join } from "node:path"; import type { StudioApiAdapter } from "../types.js"; +import { validateUploadedMedia } from "../helpers/mediaValidation.js"; import { isSafePath } from "../helpers/safePath.js"; import { removeElementFromHtml } from "../helpers/sourceMutation.js"; @@ -301,6 +302,7 @@ export function registerFileRoutes(api: Hono, adapter: StudioApiAdapter): void { const formData = await c.req.formData(); const uploaded: string[] = []; const skipped: string[] = []; + const invalid: Array<{ name: string; reason: string }> = []; for (const [, value] of formData.entries()) { if (!(value instanceof File)) continue; @@ -338,10 +340,16 @@ export function registerFileRoutes(api: Hono, adapter: StudioApiAdapter): void { const buffer = Buffer.from(await value.arrayBuffer()); writeFileSync(finalPath, buffer); + const validation = validateUploadedMedia(finalPath); + if (!validation.ok) { + unlinkSync(finalPath); + invalid.push({ name: finalName, reason: validation.reason }); + continue; + } uploaded.push(subDir ? join(subDir, finalName) : finalName); } - return c.json({ ok: true, files: uploaded, skipped }, 201); + return c.json({ ok: true, files: uploaded, skipped, invalid }, 201); }, ); } diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index 53d216ddc..7bd4d9854 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -936,6 +936,10 @@ export function StudioApp() { if (data.skipped?.length) { showToast(`Skipped (too large): ${data.skipped.join(", ")}`); } + if (data.invalid?.length) { + const names = data.invalid.map((entry: { name: string }) => entry.name).join(", "); + showToast(`Unsupported media skipped: ${names}`); + } await refreshFileTree(); setRefreshKey((k) => k + 1); return Array.isArray(data.files) ? data.files : []; diff --git a/packages/studio/vite.config.ts b/packages/studio/vite.config.ts index 93b70915b..57c8c215d 100644 --- a/packages/studio/vite.config.ts +++ b/packages/studio/vite.config.ts @@ -99,6 +99,7 @@ function createViteAdapter(dataDir: string, server: ViteDevServer): StudioApiAda return { listProjects() { + if (!existsSync(dataDir)) return []; const sessionsDir = resolve(dataDir, "../sessions"); const sessionMap = new Map(); if (existsSync(sessionsDir)) { From 67fceaff72caf2d5805dde0021385ef8fbfbe754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 23 Apr 2026 20:07:10 -0400 Subject: [PATCH 5/7] fix(studio): preserve binary uploads in api bridge --- .../helpers/mediaValidation.test.ts | 20 ++++++++++++++++++- .../src/studio-api/helpers/mediaValidation.ts | 19 ++++++++++++++++++ packages/core/src/studio-api/routes/files.ts | 7 +++---- packages/studio/vite.config.ts | 10 ++++------ packages/studio/vite.request-body.test.ts | 20 +++++++++++++++++++ packages/studio/vite.request-body.ts | 11 ++++++++++ 6 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 packages/studio/vite.request-body.test.ts create mode 100644 packages/studio/vite.request-body.ts diff --git a/packages/core/src/studio-api/helpers/mediaValidation.test.ts b/packages/core/src/studio-api/helpers/mediaValidation.test.ts index dcba462ee..45ddd4a00 100644 --- a/packages/core/src/studio-api/helpers/mediaValidation.test.ts +++ b/packages/core/src/studio-api/helpers/mediaValidation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { validateUploadedMedia } from "./mediaValidation.js"; +import { validateUploadedMedia, validateUploadedMediaBuffer } from "./mediaValidation.js"; describe("validateUploadedMedia", () => { it("passes through non-media files", () => { @@ -51,3 +51,21 @@ describe("validateUploadedMedia", () => { ).toEqual({ ok: true }); }); }); + +describe("validateUploadedMediaBuffer", () => { + it("validates media from a temp file that preserves the extension", () => { + let inspectedPath = ""; + expect( + validateUploadedMediaBuffer("raycast.mp4", new Uint8Array([0, 1, 2]), (_command, args) => { + inspectedPath = args.at(-1) ?? ""; + return { + status: 0, + stdout: JSON.stringify({ streams: [{ codec_type: "video" }] }), + stderr: "", + }; + }), + ).toEqual({ ok: true }); + + expect(inspectedPath).toMatch(/raycast\.mp4$/); + }); +}); diff --git a/packages/core/src/studio-api/helpers/mediaValidation.ts b/packages/core/src/studio-api/helpers/mediaValidation.ts index 3f84b3133..6b3a1d6da 100644 --- a/packages/core/src/studio-api/helpers/mediaValidation.ts +++ b/packages/core/src/studio-api/helpers/mediaValidation.ts @@ -1,4 +1,7 @@ import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; const VIDEO_EXT = /\.(mp4|webm|mov)$/i; const AUDIO_EXT = /\.(mp3|wav|ogg|m4a|aac)$/i; @@ -59,3 +62,19 @@ export function validateUploadedMedia( return { ok: false, reason: "ffprobe returned unreadable media metadata" }; } } + +export function validateUploadedMediaBuffer( + fileName: string, + buffer: Uint8Array, + runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner, +): { ok: true } | { ok: false; reason: string } { + const tempDir = mkdtempSync(join(tmpdir(), "hyperframes-upload-")); + const tempPath = join(tempDir, basename(fileName)); + + try { + writeFileSync(tempPath, buffer); + return validateUploadedMedia(tempPath, runner); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +} diff --git a/packages/core/src/studio-api/routes/files.ts b/packages/core/src/studio-api/routes/files.ts index 7b4b17c5e..7fc63ae47 100644 --- a/packages/core/src/studio-api/routes/files.ts +++ b/packages/core/src/studio-api/routes/files.ts @@ -13,7 +13,7 @@ import { } from "node:fs"; import { resolve, dirname, join } from "node:path"; import type { StudioApiAdapter } from "../types.js"; -import { validateUploadedMedia } from "../helpers/mediaValidation.js"; +import { validateUploadedMediaBuffer } from "../helpers/mediaValidation.js"; import { isSafePath } from "../helpers/safePath.js"; import { removeElementFromHtml } from "../helpers/sourceMutation.js"; @@ -339,13 +339,12 @@ export function registerFileRoutes(api: Hono, adapter: StudioApiAdapter): void { } const buffer = Buffer.from(await value.arrayBuffer()); - writeFileSync(finalPath, buffer); - const validation = validateUploadedMedia(finalPath); + const validation = validateUploadedMediaBuffer(finalName, buffer); if (!validation.ok) { - unlinkSync(finalPath); invalid.push({ name: finalName, reason: validation.reason }); continue; } + writeFileSync(finalPath, buffer); uploaded.push(subDir ? join(subDir, finalName) : finalName); } diff --git a/packages/studio/vite.config.ts b/packages/studio/vite.config.ts index 57c8c215d..4926f7a8b 100644 --- a/packages/studio/vite.config.ts +++ b/packages/studio/vite.config.ts @@ -15,6 +15,7 @@ import type { RenderJobState, } from "@hyperframes/core/studio-api"; import { createRetryingModuleLoader, ensureProducerDist } from "./vite.producer"; +import { readNodeRequestBody } from "./vite.request-body.js"; // ── Shared Puppeteer browser ───────────────────────────────────────────────── @@ -446,13 +447,10 @@ function devProjectApi(): Plugin { url.pathname = url.pathname.slice(4); // Read body for non-GET/HEAD - let body: string | undefined; + let body: Buffer | undefined; if (req.method !== "GET" && req.method !== "HEAD") { - body = await new Promise((resolve) => { - let data = ""; - req.on("data", (chunk: Buffer) => (data += chunk.toString())); - req.on("end", () => resolve(data)); - }); + const bytes = await readNodeRequestBody(req); + body = bytes.byteLength > 0 ? bytes : undefined; } const headers: Record = {}; diff --git a/packages/studio/vite.request-body.test.ts b/packages/studio/vite.request-body.test.ts new file mode 100644 index 000000000..605b8037a --- /dev/null +++ b/packages/studio/vite.request-body.test.ts @@ -0,0 +1,20 @@ +import { Readable } from "node:stream"; +import { describe, expect, it } from "vitest"; +import { readNodeRequestBody } from "./vite.request-body.js"; + +describe("readNodeRequestBody", () => { + it("preserves binary request bytes", async () => { + const source = Buffer.from([0x00, 0xff, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); + const body = await readNodeRequestBody( + Readable.from([source.subarray(0, 3), source.subarray(3)]), + ); + + expect(Buffer.compare(body, source)).toBe(0); + }); + + it("returns an empty buffer when the request has no body", async () => { + const body = await readNodeRequestBody(Readable.from([])); + + expect(body.byteLength).toBe(0); + }); +}); diff --git a/packages/studio/vite.request-body.ts b/packages/studio/vite.request-body.ts new file mode 100644 index 000000000..df470e341 --- /dev/null +++ b/packages/studio/vite.request-body.ts @@ -0,0 +1,11 @@ +export async function readNodeRequestBody( + req: AsyncIterable, +): Promise { + const chunks: Uint8Array[] = []; + + for await (const chunk of req) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk)); + } + + return Buffer.concat(chunks); +} From 9754c3b3d372074fe89ba360fd90bd89d6ad8977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 23 Apr 2026 20:27:09 -0400 Subject: [PATCH 6/7] fix(core): use live child timelines for runtime visibility --- packages/core/src/runtime/init.test.ts | 93 ++++++++++++++++++++++++++ packages/core/src/runtime/init.ts | 11 ++- 2 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/runtime/init.test.ts diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts new file mode 100644 index 000000000..1b145768d --- /dev/null +++ b/packages/core/src/runtime/init.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { initSandboxRuntimeModular } from "./init"; +import type { RuntimeTimelineLike } from "./types"; + +function createMockTimeline(duration: number): RuntimeTimelineLike { + const state = { time: 0, paused: true }; + return { + play: () => { + state.paused = false; + }, + pause: () => { + state.paused = true; + }, + seek: (time: number) => { + state.time = time; + }, + totalTime: (time: number) => { + state.time = time; + }, + time: () => state.time, + duration: () => duration, + add: () => {}, + paused: (value?: boolean) => { + if (typeof value === "boolean") { + state.paused = value; + } + return state.paused; + }, + timeScale: () => {}, + set: () => {}, + getChildren: () => [], + }; +} + +describe("initSandboxRuntimeModular", () => { + const originalRequestAnimationFrame = window.requestAnimationFrame; + const originalCancelAnimationFrame = window.cancelAnimationFrame; + + beforeEach(() => { + document.body.innerHTML = ""; + (globalThis as typeof globalThis & { CSS?: { escape?: (value: string) => string } }).CSS ??= {}; + globalThis.CSS.escape ??= (value: string) => value; + window.requestAnimationFrame = ((callback: FrameRequestCallback) => { + callback(0); + return 1; + }) as typeof window.requestAnimationFrame; + window.cancelAnimationFrame = (() => {}) as typeof window.cancelAnimationFrame; + }); + + afterEach(() => { + (window as Window & { __hfRuntimeTeardown?: (() => void) | null }).__hfRuntimeTeardown?.(); + document.body.innerHTML = ""; + delete (window as Window & { __timelines?: Record }).__timelines; + delete (window as Window & { __player?: unknown }).__player; + delete (window as Window & { __playerReady?: boolean }).__playerReady; + delete (window as Window & { __renderReady?: boolean }).__renderReady; + window.requestAnimationFrame = originalRequestAnimationFrame; + window.cancelAnimationFrame = originalCancelAnimationFrame; + }); + + it("uses live child timeline duration for composition-host visibility", () => { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const child = document.createElement("div"); + child.setAttribute("data-composition-id", "slide-1"); + child.setAttribute("data-start", "0"); + child.setAttribute("data-hf-authored-duration", "14"); + root.appendChild(child); + + (window as Window & { __timelines?: Record }).__timelines = { + main: createMockTimeline(20), + "slide-1": createMockTimeline(8), + }; + + initSandboxRuntimeModular(); + + const player = ( + window as Window & { + __player?: { renderSeek: (timeSeconds: number) => void }; + } + ).__player; + expect(player).toBeDefined(); + + player?.renderSeek(9); + + expect(child.style.visibility).toBe("hidden"); + }); +}); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 0f6ecf9bc..7b7f0e35c 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -368,13 +368,16 @@ export function initSandboxRuntimeModular(): void { return resolver.resolveStartForElement(element, fallback); }; - const resolveDurationForElement = (element: Element): number | null => { + const resolveDurationForElement = ( + element: Element, + opts?: { includeAuthoredTimingAttrs?: boolean }, + ): number | null => { const resolver = createRuntimeStartTimeResolver({ timelineRegistry: (window.__timelines ?? {}) as Record< string, RuntimeTimelineLike | undefined >, - includeAuthoredTimingAttrs: true, + includeAuthoredTimingAttrs: opts?.includeAuthoredTimingAttrs ?? true, }); return resolver.resolveDurationForElement(element); }; @@ -1234,7 +1237,9 @@ export function initSandboxRuntimeModular(): void { } const start = resolveStartForElement(rawNode, 0); - const duration = resolveDurationForElement(rawNode); + const duration = resolveDurationForElement(rawNode, { + includeAuthoredTimingAttrs: false, + }); const end = duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY; // For composition hosts, use the composition timeline's duration to compute end let computedEnd = end; From 88da51c5254bd90ab714d52b4d640dc398a9afc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 23 Apr 2026 20:49:30 -0400 Subject: [PATCH 7/7] fix(core): respect host timing in runtime visibility --- packages/core/src/runtime/init.test.ts | 35 +++++++++++++++++++++++++- packages/core/src/runtime/init.ts | 26 ++++++++++++------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 1b145768d..8ce97e748 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -58,7 +58,7 @@ describe("initSandboxRuntimeModular", () => { window.cancelAnimationFrame = originalCancelAnimationFrame; }); - it("uses live child timeline duration for composition-host visibility", () => { + it("uses the shorter live child timeline when the authored window is longer", () => { const root = document.createElement("div"); root.setAttribute("data-composition-id", "main"); root.setAttribute("data-root", "true"); @@ -90,4 +90,37 @@ describe("initSandboxRuntimeModular", () => { expect(child.style.visibility).toBe("hidden"); }); + + it("uses the shorter authored host window when the child timeline is longer", () => { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const child = document.createElement("div"); + child.setAttribute("data-composition-id", "slide-1"); + child.setAttribute("data-start", "0"); + child.setAttribute("data-hf-authored-duration", "2"); + root.appendChild(child); + + (window as Window & { __timelines?: Record }).__timelines = { + main: createMockTimeline(20), + "slide-1": createMockTimeline(8), + }; + + initSandboxRuntimeModular(); + + const player = ( + window as Window & { + __player?: { renderSeek: (timeSeconds: number) => void }; + } + ).__player; + expect(player).toBeDefined(); + + player?.renderSeek(3); + + expect(child.style.visibility).toBe("hidden"); + }); }); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 7b7f0e35c..e87292c87 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -1237,20 +1237,28 @@ export function initSandboxRuntimeModular(): void { } const start = resolveStartForElement(rawNode, 0); - const duration = resolveDurationForElement(rawNode, { - includeAuthoredTimingAttrs: false, - }); - const end = duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY; - // For composition hosts, use the composition timeline's duration to compute end - let computedEnd = end; + let duration = resolveDurationForElement(rawNode); const compId = rawNode.getAttribute("data-composition-id"); - if (compId && !Number.isFinite(end)) { + if (compId) { const compTimeline = (window.__timelines ?? {})[compId]; + let liveDuration: number | null = null; if (compTimeline && typeof compTimeline.duration === "function") { - const compDur = compTimeline.duration(); - if (compDur > 0) computedEnd = start + compDur; + const compDur = Number(compTimeline.duration()); + if (Number.isFinite(compDur) && compDur > 0) { + liveDuration = compDur; + } + } + + // Composition hosts must respect both the authored clip window in the parent + // composition and the child composition's own live timeline duration. + if (duration != null && duration > 0 && liveDuration != null) { + duration = Math.min(duration, liveDuration); + } else if ((duration == null || duration <= 0) && liveDuration != null) { + duration = liveDuration; } } + const computedEnd = + duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY; const isVisibleNow = state.currentTime >= start && (Number.isFinite(computedEnd) ? state.currentTime < computedEnd : true);