Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/yummy-games-act.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@browserbasehq/stagehand": patch
---

fix issue where screenshot mask was only being applied to the first element that the locator resolved to. masks now apply to all matching elements.
30 changes: 17 additions & 13 deletions packages/core/lib/v3/tests/page-screenshot.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,13 @@ test.describe("Page.screenshot options", () => {
<meta charset="utf-8" />
<style>
body { background: #aaccee; margin: 0; height: 100vh; display: flex; flex-direction: column; align-items: flex-start; }
#mask-target { width: 80px; height: 80px; margin: 40px; background: rgb(0, 180, 60); animation: pulse 1s infinite alternate; }
.mask-target { width: 80px; height: 80px; margin: 40px; background: rgb(0, 180, 60); animation: pulse 1s infinite alternate; }
@keyframes pulse { from { transform: scale(1); } to { transform: scale(1.2); } }
</style>
</head>
<body>
<div id="mask-target"></div>
<div class="mask-target"></div>
<div class="mask-target"></div>
<input id="focus-me" value="focus" />
<script>document.getElementById('focus-me').focus();</script>
</body>
Expand All @@ -104,7 +105,7 @@ test.describe("Page.screenshot options", () => {

await page.goto("data:text/html," + encodeURIComponent(html));

const maskLocator = page.locator("#mask-target");
const maskLocator = page.locator(".mask-target");
const tempPath = path.join(
os.tmpdir(),
`stagehand-screenshot-${Date.now()}-${Math.random().toString(36).slice(2)}.jpeg`,
Expand Down Expand Up @@ -242,16 +243,19 @@ test.describe("Page.screenshot options", () => {
cssArgs.some((css) => css.includes("border: 3px solid black")),
).toBeTruthy();

expect(
evaluateCalls.some((entry) => {
const arg = entry.arg;
return (
arg &&
typeof arg === "object" &&
"rects" in (arg as Record<string, unknown>)
);
}),
).toBeTruthy();
const maskCalls = evaluateCalls.filter((entry) => {
const arg = entry.arg;
return (
arg &&
typeof arg === "object" &&
"rects" in (arg as Record<string, unknown>)
);
});
expect(maskCalls.length).toBeGreaterThan(0);
const rects = (maskCalls[0]?.arg as { rects?: unknown } | undefined)
?.rects;
expect(Array.isArray(rects)).toBeTruthy();
expect((rects as unknown[]).length).toBe(2);
} finally {
Frame.prototype.screenshot = originalScreenshot;
Frame.prototype.evaluate = originalEvaluate;
Expand Down
26 changes: 22 additions & 4 deletions packages/core/lib/v3/understudy/frameLocator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ import { Locator } from "./locator";
import type { Page } from "./page";
import { Frame } from "./frame";
import { executionContexts } from "./executionContextRegistry";
import { ContentFrameNotFoundError } from "../types/public/sdkErrors";
import {
ContentFrameNotFoundError,
StagehandInvalidArgumentError,
} from "../types/public/sdkErrors";

/**
* FrameLocator: resolves iframe elements to their child Frames and allows
Expand Down Expand Up @@ -92,11 +95,14 @@ class LocatorDelegate {
constructor(
private readonly fl: FrameLocator,
private readonly sel: string,
private readonly nthIndex: number = -1,
) {}

private async real(): Promise<Locator> {
const frame = await this.fl.resolveFrame();
return frame.locator(this.sel);
const locator = frame.locator(this.sel);
if (this.nthIndex < 0) return locator;
return locator.nth(this.nthIndex);
}

// Locator API delegates
Expand Down Expand Up @@ -143,8 +149,20 @@ class LocatorDelegate {
return (await this.real()).count();
}
first(): LocatorDelegate {
// Underlying querySelector already returns the first; keep chaining stable
return this;
return this.nth(0);
}
nth(index: number): LocatorDelegate {
const value = Number(index);
if (!Number.isFinite(value) || value < 0) {
throw new StagehandInvalidArgumentError(
"locator().nth() expects a non-negative index",
);
}

const nextIndex = Math.floor(value);
if (nextIndex === this.nthIndex) return this;

return new LocatorDelegate(this.fl, this.sel, nextIndex);
}
}

Expand Down
49 changes: 41 additions & 8 deletions packages/core/lib/v3/understudy/locator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,20 +43,19 @@ export class Locator {

private readonly selectorQuery: SelectorQuery;

// -1 means "no explicit nth()"; default locator resolves to first match for actions.
private readonly nthIndex: number;

constructor(
private readonly frame: Frame,
private readonly selector: string,
private readonly options?: { deep?: boolean; depth?: number },
nthIndex: number = 0,
nthIndex: number = -1,
) {
this.selectorResolver = new FrameSelectorResolver(this.frame);
this.selectorQuery = FrameSelectorResolver.parseSelector(selector);
this.nthIndex = Math.max(
0,
Math.floor(Number.isFinite(nthIndex) ? nthIndex : 0),
);
const normalized = Number.isFinite(nthIndex) ? Math.floor(nthIndex) : -1;
this.nthIndex = normalized < 0 ? -1 : normalized;
}

/** Return the owning Frame for this locator (typed accessor, no private access). */
Expand Down Expand Up @@ -821,10 +820,10 @@ export class Locator {
}

/**
* For API parity, returns the same locator (querySelector already returns the first match).
* Return a locator narrowed to the first match.
*/
first(): Locator {
return this;
return this.nth(0);
}

/** Return a locator narrowed to the element at the given zero-based index. */
Expand Down Expand Up @@ -859,9 +858,10 @@ export class Locator {
await session.send("Runtime.enable");
await session.send("DOM.enable");

const index = this.nthIndex < 0 ? 0 : this.nthIndex;
const resolved = await this.selectorResolver.resolveAtIndex(
this.selectorQuery,
this.nthIndex,
index,
);
if (!resolved) {
throw new StagehandElementNotFoundError([this.selector]);
Expand All @@ -870,6 +870,39 @@ export class Locator {
return resolved;
}

/**
* Resolve all matching nodes for this locator.
* If the locator is narrowed via nth(), only that index is returned.
*/
public async resolveNodesForMask(): Promise<
Array<{
nodeId: Protocol.DOM.NodeId | null;
objectId: Protocol.Runtime.RemoteObjectId;
}>
> {
const session = this.frame.session;

await session.send("Runtime.enable");
await session.send("DOM.enable");

if (this.nthIndex >= 0) {
const resolved = await this.selectorResolver.resolveAtIndex(
this.selectorQuery,
this.nthIndex,
);
if (!resolved) {
throw new StagehandElementNotFoundError([this.selector]);
}
return [resolved];
}

const resolved = await this.selectorResolver.resolveAll(this.selectorQuery);
if (!resolved.length) {
throw new StagehandElementNotFoundError([this.selector]);
}
return resolved;
}

/** Compute a center point from a BoxModel content quad */
private centerFromBoxContent(content: number[]): { cx: number; cy: number } {
// content is [x1,y1, x2,y2, x3,y3, x4,y4]
Expand Down
127 changes: 73 additions & 54 deletions packages/core/lib/v3/understudy/screenshotUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -209,10 +209,10 @@ export async function applyMaskOverlays(

for (const locator of locators) {
try {
const info = await resolveMaskRect(locator);
const info = await resolveMaskRects(locator);
if (!info) continue;
const list = rectsByFrame.get(info.frame) ?? [];
list.push(info.rect);
list.push(...info.rects);
rectsByFrame.set(info.frame, list);
} catch {
// ignore individual locator failures
Expand Down Expand Up @@ -282,71 +282,90 @@ export async function applyMaskOverlays(
};
}

async function resolveMaskRect(
async function resolveMaskRects(
locator: Locator,
): Promise<{ frame: Frame; rect: ScreenshotClip } | null> {
): Promise<{ frame: Frame; rects: ScreenshotClip[] } | null> {
const frame = locator.getFrame();
const session = frame.session;
let objectId: Protocol.Runtime.RemoteObjectId | null = null;
let resolved: Array<{
objectId: Protocol.Runtime.RemoteObjectId;
nodeId: Protocol.DOM.NodeId | null;
}> = [];

try {
const resolved = await locator.resolveNode();
objectId = resolved.objectId;

const result = await session.send<Protocol.Runtime.CallFunctionOnResponse>(
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
if (!this || typeof this.getBoundingClientRect !== 'function') return null;
const rect = this.getBoundingClientRect();
if (!rect) return null;
const style = window.getComputedStyle(this);
if (!style) return null;
if (style.visibility === 'hidden' || style.display === 'none') return null;
if (rect.width <= 0 || rect.height <= 0) return null;
return {
x: rect.left + window.scrollX,
y: rect.top + window.scrollY,
width: rect.width,
height: rect.height,
};
}`,
returnByValue: true,
},
);

if (result.exceptionDetails) {
return null;
resolved = await locator.resolveNodesForMask();
const rects: ScreenshotClip[] = [];

for (const { objectId } of resolved) {
try {
const rect = await resolveMaskRectForObject(session, objectId);
if (rect) rects.push(rect);
} catch {
// ignore individual element failures
} finally {
await session
.send<never>("Runtime.releaseObject", { objectId })
.catch(() => {});
}
}

const rect = result.result.value as ScreenshotClip | null;
if (!rect) return null;

const { x, y, width, height } = rect;
if (
!Number.isFinite(x) ||
!Number.isFinite(y) ||
!Number.isFinite(width) ||
!Number.isFinite(height) ||
width <= 0 ||
height <= 0
) {
return null;
}
if (!rects.length) return null;

return { frame, rect: { x, y, width, height } };
return { frame, rects };
} catch {
return null;
} finally {
if (objectId) {
await session
.send<never>("Runtime.releaseObject", { objectId })
.catch(() => {});
}
}
}

async function resolveMaskRectForObject(
session: CDPSessionLike,
objectId: Protocol.Runtime.RemoteObjectId,
): Promise<ScreenshotClip | null> {
const result = await session.send<Protocol.Runtime.CallFunctionOnResponse>(
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
if (!this || typeof this.getBoundingClientRect !== 'function') return null;
const rect = this.getBoundingClientRect();
if (!rect) return null;
const style = window.getComputedStyle(this);
if (!style) return null;
if (style.visibility === 'hidden' || style.display === 'none') return null;
if (rect.width <= 0 || rect.height <= 0) return null;
return {
x: rect.left + window.scrollX,
y: rect.top + window.scrollY,
width: rect.width,
height: rect.height,
};
}`,
returnByValue: true,
},
);

if (result.exceptionDetails) {
return null;
}

const rect = result.result.value as ScreenshotClip | null;
if (!rect) return null;

const { x, y, width, height } = rect;
if (
!Number.isFinite(x) ||
!Number.isFinite(y) ||
!Number.isFinite(width) ||
!Number.isFinite(height) ||
width <= 0 ||
height <= 0
) {
return null;
}

return { x, y, width, height };
}

export async function runScreenshotCleanups(
cleanups: ScreenshotCleanup[],
): Promise<void> {
Expand Down
Loading