feat(annotate): instrument input actions and show subtitle overlay

CLAUDE.md

@@ -126,6 +126,7 @@ EOF
 
 Never add Co-Authored-By agents in commit message.
 Never add "Generated with" in commit message.
+Never add test plan to PR description. Keep PR description short — a few bullet points at most.
 Branch naming for issue fixes: `fix-<issue-number>`
 
 ## Development Guides

docs/src/test-api/class-testoptions.md

@@ -625,6 +625,8 @@ export default defineConfig({
   - `size` ?<[Object]> Size of the recorded video. Optional.
     - `width` <[int]>
     - `height` <[int]>
+  - `annotate` ?<[Object]> If specified, visually annotates actions in the video with element highlights and action title subtitles.
+    - `delay` ?<[int]> How long each annotation is displayed in milliseconds. Defaults to `500`.
 
 Whether to record video for each test. Defaults to `'off'`.
 * `'off'`: Do not record video.
@@ -634,6 +636,8 @@ Whether to record video for each test. Defaults to `'off'`.
 
 To control video size, pass an object with `mode` and `size` properties. If video size is not specified, it will be equal to [`property: TestOptions.viewport`] scaled down to fit into 800x800. If `viewport` is not configured explicitly the video size defaults to 800x450. Actual picture of each page will be scaled down if necessary to fit the specified size.
 
+To annotate actions in the video with element highlights and action title subtitles, pass `annotate` with an optional `delay` in milliseconds (defaults to `500`).
+
 **Usage**
 
 ```js title="playwright.config.ts"

docs/src/videos.md

@@ -36,15 +36,18 @@ const context = await browser.newContext({ recordVideo: { dir: 'videos/' } });
 await context.close();
 ```
 
-You can also specify video size. The video size defaults to the viewport size scaled down to fit 800x800. The video of the viewport is placed in the top-left corner of the output video, scaled down to fit if necessary. You may need to set the viewport size to match your desired video size.
+You can also specify video size and annotation. The video size defaults to the viewport size scaled down to fit 800x800. The video of the viewport is placed in the top-left corner of the output video, scaled down to fit if necessary. You may need to set the viewport size to match your desired video size.
+
+When `annotate` is specified, each action will be visually highlighted in the video with the element outline and action title subtitle. The optional `delay` property controls how long each annotation is displayed (defaults to `500`ms).
 
 ```js tab=js-test title="playwright.config.ts"
 import { defineConfig } from '@playwright/test';
 export default defineConfig({
   use: {
     video: {
       mode: 'on-first-retry',
-      size: { width: 640, height: 480 }
+      size: { width: 640, height: 480 },
+      annotate: { delay: 500 },
     }
   },
 });
@@ -55,6 +58,7 @@ const context = await browser.newContext({
   recordVideo: {
     dir: 'videos/',
     size: { width: 640, height: 480 },
+    annotate: { delay: 500 },
   }
 });
 ```

examples/todomvc/playwright.config.ts

@@ -47,6 +47,7 @@ export default defineConfig({
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'on-first-retry',
+    video: { mode: 'on', annotate: { delay: 500 } },
   },
 
   /* Configure projects for major browsers */

packages/injected/src/highlight.css

@@ -107,6 +107,23 @@ x-pw-action-point {
   z-index: 2;
 }
 
+x-pw-subtitle {
+  position: absolute;
+  bottom: 55px;
+  left: 50%;
+  transform: translateX(-50%);
+  backdrop-filter: blur(5px);
+  background-color: rgba(0, 0, 0, 0.7);
+  color: white;
+  border-radius: 6px;
+  padding: 6px 14px;
+  font-size: 14px;
+  line-height: 1.4;
+  white-space: nowrap;
+  user-select: none;
+  z-index: 3;
+}
+
 @keyframes pw-fade-out {
   from { opacity: 1; }
   to { opacity: 0; }

packages/injected/src/highlight.ts

@@ -24,8 +24,10 @@ import type { ParsedSelector } from '@isomorphic/selectorParser';
 import type { InjectedScript } from './injectedScript';
 
 
+type Rect = { x: number, y: number, width: number, height: number };
+
 type RenderedHighlightEntry = {
-  targetElement: Element,
+  targetElement?: Element,
   color: string,
   borderColor?: string,
   fadeDuration?: number,
@@ -38,7 +40,8 @@ type RenderedHighlightEntry = {
 };
 
 export type HighlightEntry = {
-  element: Element,
+  element?: Element,
+  box?: Rect,
   color: string,
   borderColor?: string,
   fadeDuration?: number,
@@ -50,6 +53,7 @@ export class Highlight {
   private _glassPaneShadow: ShadowRoot;
   private _renderedEntries: RenderedHighlightEntry[] = [];
   private _actionPointElement: HTMLElement;
+  private _subtitleElement: HTMLElement;
   private _isUnderTest: boolean;
   private _injectedScript: InjectedScript;
   private _rafRequest: number | undefined;
@@ -75,6 +79,8 @@ export class Highlight {
     this._glassPaneElement.style.backgroundColor = 'transparent';
     this._actionPointElement = document.createElement('x-pw-action-point');
     this._actionPointElement.setAttribute('hidden', 'true');
+    this._subtitleElement = document.createElement('x-pw-subtitle');
+    this._subtitleElement.setAttribute('hidden', 'true');
     this._glassPaneShadow = this._glassPaneElement.attachShadow({ mode: this._isUnderTest ? 'open' : 'closed' });
     // workaround for firefox: when taking screenshots, it complains adoptedStyleSheets.push
     // is not a function, so we fallback to style injection
@@ -88,6 +94,7 @@ export class Highlight {
       this._glassPaneShadow.appendChild(styleElement);
     }
     this._glassPaneShadow.appendChild(this._actionPointElement);
+    this._glassPaneShadow.appendChild(this._subtitleElement);
   }
 
   install() {
@@ -141,6 +148,17 @@ export class Highlight {
     this._actionPointElement.hidden = true;
   }
 
+  showSubtitle(text: string, fadeDuration: number) {
+    this._subtitleElement.textContent = text;
+    this._subtitleElement.hidden = false;
+    const fadeTime = fadeDuration / 4;
+    this._subtitleElement.style.animation = `pw-fade-out ${fadeTime}ms ease-out ${fadeDuration - fadeTime}ms forwards`;
+  }
+
+  hideSubtitle() {
+    this._subtitleElement.hidden = true;
+  }
+
   clearHighlight() {
     for (const entry of this._renderedEntries) {
       entry.highlightElement?.remove();
@@ -178,12 +196,14 @@ export class Highlight {
         lineElement.textContent = entry.tooltipText;
         tooltipElement.appendChild(lineElement);
       }
-      this._renderedEntries.push({ targetElement: entry.element, color: entry.color, borderColor: entry.borderColor, fadeDuration: entry.fadeDuration, tooltipElement, highlightElement });
+      this._renderedEntries.push({ targetElement: entry.element, box: toDOMRect(entry.box), color: entry.color, borderColor: entry.borderColor, fadeDuration: entry.fadeDuration, tooltipElement, highlightElement });
     }
 
     // 2. Trigger layout while positioning tooltips and computing bounding boxes.
     for (const entry of this._renderedEntries) {
-      entry.box = entry.targetElement.getBoundingClientRect();
+      if (!entry.box && !entry.targetElement)
+        continue;
+      entry.box = entry.box || entry.targetElement!.getBoundingClientRect();
       if (!entry.tooltipElement)
         continue;
 
@@ -271,7 +291,7 @@ export class Highlight {
       const oldBox = this._renderedEntries[i].box;
       if (!oldBox)
         return false;
-      const box = entries[i].element.getBoundingClientRect();
+      const box = entries[i].box ? toDOMRect(entries[i].box!) : entries[i].element!.getBoundingClientRect();
       if (box.top !== oldBox.top || box.right !== oldBox.right || box.bottom !== oldBox.bottom || box.left !== oldBox.left)
         return false;
     }
@@ -298,3 +318,11 @@ export class Highlight {
     this._glassPaneElement.removeEventListener('click', handler);
   }
 }
+
+function toDOMRect(box: Rect): DOMRect;
+function toDOMRect(box: Rect | undefined): DOMRect | undefined;
+function toDOMRect(box: Rect | undefined): DOMRect | undefined {
+  if (!box)
+    return undefined;
+  return new DOMRect(box.x, box.y, box.width, box.height);
+}

packages/injected/src/injectedScript.ts

@@ -1313,21 +1313,22 @@ export class InjectedScript {
     this._highlight.runHighlightOnRaf(selector);
   }
 
-  highlightNode(node: Node, point?: { x: number, y: number }, delay?: number) {
+  annotate(annotation: { point?: channels.Point, box?: channels.Rect, title?: string, delay?: number }) {
     const highlight = this._createHighlight();
-    const fadeDuration = delay ?? 500;
+    const fadeDuration = annotation.delay ?? 500;
 
-    if (node.nodeType === Node.ELEMENT_NODE) {
-      const element = node as Element;
+    if (annotation.box) {
       highlight.updateHighlight([{
-        element,
+        box: annotation.box,
         color: 'rgba(0, 128, 255, 0.15)',
         borderColor: 'rgba(0, 128, 255, 0.6)',
         fadeDuration,
       }]);
     }
-    if (point)
-      highlight.showActionPoint(point.x, point.y, fadeDuration);
+    if (annotation.point)
+      highlight.showActionPoint(annotation.point.x, annotation.point.y, fadeDuration);
+    if (annotation.title)
+      highlight.showSubtitle(annotation.title, fadeDuration);
   }
 
   hideHighlight() {

packages/playwright-core/src/client/channelOwner.ts

@@ -16,7 +16,7 @@
 
 import { EventEmitter } from './eventEmitter';
 import { ValidationError, maybeFindValidator  } from '../protocol/validator';
-import { getMetainfo } from '../utils/isomorphic/protocolFormatter';
+import { getMetainfo } from '../utils/isomorphic/protocolMetainfo';
 import { captureLibraryStackTrace } from './clientStackTrace';
 import { stringifyStackFrames } from '../utils/isomorphic/stackTrace';
 

packages/playwright-core/src/server/debugger.ts

@@ -17,7 +17,7 @@
 import { SdkObject } from './instrumentation';
 import { monotonicTime } from '../utils';
 import { BrowserContext } from './browserContext';
-import { getMetainfo } from '../utils/isomorphic/protocolFormatter';
+import { getMetainfo } from '../utils/isomorphic/protocolMetainfo';
 
 import type { CallMetadata, InstrumentationListener } from './instrumentation';
 
@@ -29,7 +29,7 @@ export class Debugger extends SdkObject implements InstrumentationListener {
   private _pauseAt: PauseAt = {};
   private _pausedCallsMetadata = new Map<CallMetadata, { resolve: () => void, sdkObject: SdkObject }>();
   private _enabled = false;
-  private _pauseBeforeInputActions = false;  // instead of inside input actions
+  private _pauseBeforeWaitingActions = false;  // instead of inside input actions
   private _context: BrowserContext;
 
   static Events = {
@@ -52,24 +52,27 @@ export class Debugger extends SdkObject implements InstrumentationListener {
   }
 
   async onBeforeCall(sdkObject: SdkObject, metadata: CallMetadata): Promise<void> {
-    if (this._muted)
+    if (this._muted || metadata.internal)
       return;
+    const metainfo = getMetainfo(metadata);
     const pauseOnPauseCall = this._enabled && metadata.type === 'BrowserContext' && metadata.method === 'pause';
-    const pauseOnNextStep = !!this._pauseAt.next && shouldPauseBeforeStep(metadata, this._pauseBeforeInputActions);
+    const pauseBeforeAction = !!this._pauseAt.next && !!metainfo?.pause && (this._pauseBeforeWaitingActions || !metainfo?.isAutoWaiting);
     const pauseOnLocation = !!this._pauseAt.location && matchesLocation(metadata, this._pauseAt.location);
-    if (pauseOnPauseCall || pauseOnNextStep || pauseOnLocation)
+    if (pauseOnPauseCall || pauseBeforeAction || pauseOnLocation)
       await this._pause(sdkObject, metadata);
   }
 
   async onBeforeInputAction(sdkObject: SdkObject, metadata: CallMetadata): Promise<void> {
-    if (this._muted)
+    if (this._muted || metadata.internal)
       return;
-    if (!!this._pauseAt.next && !this._pauseBeforeInputActions)
+    const metainfo = getMetainfo(metadata);
+    const pauseBeforeInput = !!this._pauseAt.next && !!metainfo?.pause && !!metainfo?.isAutoWaiting && !this._pauseBeforeWaitingActions;
+    if (pauseBeforeInput)
       await this._pause(sdkObject, metadata);
   }
 
   private async _pause(sdkObject: SdkObject, metadata: CallMetadata) {
-    if (this._muted)
+    if (this._muted || metadata.internal)
       return;
     this._pauseAt = {};
     metadata.pauseStartTime = monotonicTime();
@@ -93,8 +96,8 @@ export class Debugger extends SdkObject implements InstrumentationListener {
     this.emit(Debugger.Events.PausedStateChanged);
   }
 
-  setPauseBeforeInputActions() {
-    this._pauseBeforeInputActions = true;
+  setPauseBeforeWaitingActions() {
+    this._pauseBeforeWaitingActions = true;
   }
 
   setPauseAt(at: { next?: boolean, location?: { file: string, line?: number, column?: number } } = {}) {
@@ -121,10 +124,3 @@ function matchesLocation(metadata: CallMetadata, location: { file: string, line?
       (location.line === undefined || metadata.location.line === location.line) &&
       (location.column === undefined || metadata.location.column === location.column);
 }
-
-function shouldPauseBeforeStep(metadata: CallMetadata, includeInputActions: boolean): boolean {
-  if (metadata.internal)
-    return false;
-  const metainfo = getMetainfo(metadata);
-  return !!metainfo?.pausesBeforeAction || (includeInputActions && !!metainfo?.pausesBeforeInput);
-}

packages/playwright-core/src/server/dispatchers/debuggerDispatcher.ts

@@ -53,7 +53,7 @@ export class DebuggerDispatcher extends Dispatcher<Debugger, channels.DebuggerCh
   async pause(params: channels.DebuggerPauseParams, progress: Progress): Promise<void> {
     if (this._object.isPaused())
       throw new Error('Debugger is already paused');
-    this._object.setPauseBeforeInputActions();
+    this._object.setPauseBeforeWaitingActions();
     this._object.setPauseAt({ next: true });
   }
 
@@ -66,15 +66,15 @@ export class DebuggerDispatcher extends Dispatcher<Debugger, channels.DebuggerCh
   async next(params: channels.DebuggerNextParams, progress: Progress): Promise<void> {
     if (!this._object.isPaused())
       throw new Error('Debugger is not paused');
-    this._object.setPauseBeforeInputActions();
+    this._object.setPauseBeforeWaitingActions();
     this._object.setPauseAt({ next: true });
     this._object.resume();
   }
 
   async runTo(params: channels.DebuggerRunToParams, progress: Progress): Promise<void> {
     if (!this._object.isPaused())
       throw new Error('Debugger is not paused');
-    this._object.setPauseBeforeInputActions();
+    this._object.setPauseBeforeWaitingActions();
     this._object.setPauseAt({ location: params.location });
     this._object.resume();
   }

packages/playwright-core/src/server/dispatchers/dispatcher.ts

@@ -24,7 +24,7 @@ import { TargetClosedError, isTargetClosedError, serializeError } from '../error
 import { createRootSdkObject, SdkObject } from '../instrumentation';
 import { isProtocolError } from '../protocolError';
 import { compressCallLog } from '../callLog';
-import { getMetainfo } from '../../utils/isomorphic/protocolFormatter';
+import { getMetainfo } from '../../utils/isomorphic/protocolMetainfo';
 import { Progress, ProgressController } from '../progress';
 
 import type { CallMetadata } from '../instrumentation';