From 6b5369343273010978d195bc114a6279047b86fb Mon Sep 17 00:00:00 2001 From: miguel Date: Thu, 17 Sep 2026 00:06:04 -0700 Subject: [PATCH] Reorder #2896: keep frame support before capture recovery --- .changeset/eval-frame-snapshot-maps.md | 6 + .../bench/observe/observe_vantechjournal.ts | 5 +- packages/evals/tests/cli.test.ts | 169 ++-- .../understudy/a11y/snapshot/capture.test.ts | 203 ++++- .../understudy/a11y/snapshot/capture.ts | 23 +- .../understudy/a11y/snapshot/domTree.test.ts | 34 +- .../understudy/a11y/snapshot/domTree.ts | 6 +- packages/extension/understudy/frame.ts | 14 +- packages/extension/understudy/page.ts | 44 +- .../understudy/screenshotUtils.test.ts | 205 +++++ .../extension/understudy/screenshotUtils.ts | 79 ++ packages/integrations/core/README.md | 57 ++ .../core/integration/facade-frames.test.ts | 172 ++++ .../integrations/core/src/facade/runtime.ts | 752 ++++++++++++++++++ .../tests/facade-frame-composition.test.ts | 118 +++ .../core/tests/facade-tools.test.ts | 233 ++++++ .../extensionassets/stagehand-extension.zip | Bin 441540 -> 441955 bytes .../stagehandLaunchConnectSmoke.test.ts | 35 + 18 files changed, 2052 insertions(+), 103 deletions(-) create mode 100644 .changeset/eval-frame-snapshot-maps.md create mode 100644 packages/extension/understudy/screenshotUtils.test.ts create mode 100644 packages/integrations/core/README.md create mode 100644 packages/integrations/core/integration/facade-frames.test.ts create mode 100644 packages/integrations/core/tests/facade-frame-composition.test.ts diff --git a/.changeset/eval-frame-snapshot-maps.md b/.changeset/eval-frame-snapshot-maps.md new file mode 100644 index 0000000000..ebc35be0a5 --- /dev/null +++ b/.changeset/eval-frame-snapshot-maps.md @@ -0,0 +1,6 @@ +--- +"@browserbasehq/stagehand-extension": patch +"@browserbasehq/stagehand-go": patch +--- + +Snapshot references remain valid across same-origin and out-of-process frame captures diff --git a/packages/evals/tasks/bench/observe/observe_vantechjournal.ts b/packages/evals/tasks/bench/observe/observe_vantechjournal.ts index d3fd59e455..4a88d24ffa 100644 --- a/packages/evals/tasks/bench/observe/observe_vantechjournal.ts +++ b/packages/evals/tasks/bench/observe/observe_vantechjournal.ts @@ -19,9 +19,10 @@ export default defineBenchTask( }; } + // The archive layout changes independently of the pagination link. const expectedLocators = [ - "xpath=/html/body/div[2]/div/div/section/div/div/div[3]/a", - "xpath=/html/body/div[2]/div/div/section/div/div/div[3]/a/span", + "xpath=//a[@href='/archive?page=2' and normalize-space(.)='Load more']", + "xpath=//a[@href='/archive?page=2' and normalize-space(.)='Load more']/span", ]; // v3 compares backendNodeIds (first observation vs. each expected diff --git a/packages/evals/tests/cli.test.ts b/packages/evals/tests/cli.test.ts index fb106d7bb3..c559eab80f 100644 --- a/packages/evals/tests/cli.test.ts +++ b/packages/evals/tests/cli.test.ts @@ -10,6 +10,7 @@ const repoRoot = path.resolve(__dirname, "..", "..", ".."); const CLI_PATH = path.join(repoRoot, "packages", "evals", "cli.ts"); const SOURCE_CONFIG = path.join(repoRoot, "packages", "evals", "evals.config.json"); const CLI_CHILD_TIMEOUT_MS = 15_000; +const CLI_TEST_TIMEOUT_MS = CLI_CHILD_TIMEOUT_MS + 2_000; // File-level snapshot/restore: any `evals run …` invocation through the // real CLI writes `_meta.firstRunCompletedAt` into the source config @@ -60,7 +61,7 @@ function readSourceWelcomeCompletedAt(): string | undefined { return config._meta?.firstRunCompletedAt; } -describe("CLI entrypoint", () => { +describe("CLI entrypoint", { timeout: CLI_TEST_TIMEOUT_MS }, () => { it( "shows help", async () => { @@ -72,7 +73,7 @@ describe("CLI entrypoint", () => { expect(stdout).toContain("config"); expect(stdout).toContain("experiments"); }, - CLI_CHILD_TIMEOUT_MS + 2_000, + CLI_TEST_TIMEOUT_MS, ); it("shows experiments overview help", async () => { @@ -197,19 +198,23 @@ describe("CLI entrypoint", () => { expect(stdout).toContain(contains); }); - it("does not mark first-run complete for nested help invocations", async () => { - resetSourceWelcomeMeta(); - - for (const args of [ - ["config", "set", "--help"], - ["experiments", "compare", "--help"], - ]) { - const { stdout, code } = await runCli(args); - expect(code).toBe(0); - expect(stdout).toContain("evals"); - expect(readSourceWelcomeCompletedAt()).toBeUndefined(); - } - }); + it( + "does not mark first-run complete for nested help invocations", + async () => { + resetSourceWelcomeMeta(); + + for (const args of [ + ["config", "set", "--help"], + ["experiments", "compare", "--help"], + ]) { + const { stdout, code } = await runCli(args); + expect(code).toBe(0); + expect(stdout).toContain("evals"); + expect(readSourceWelcomeCompletedAt()).toBeUndefined(); + } + }, + 2 * CLI_CHILD_TIMEOUT_MS + 2_000, + ); // Regression: help interception must not reach into value positions. // `config set ` must surface a parse/value error, not silently @@ -289,19 +294,27 @@ describe("CLI entrypoint", () => { expect(stdout.trim()).toBe(path.join(repoRoot, "packages", "evals", "evals.config.json")); }); - it("treats `>` as equivalent to a space separator (argv form)", async () => { - const direct = await runCli(["config", "path"]); - const piped = await runCli(["config", ">", "path"]); - expect(piped.code).toBe(0); - expect(piped.stdout).toBe(direct.stdout); - }); + it( + "treats `>` as equivalent to a space separator (argv form)", + async () => { + const direct = await runCli(["config", "path"]); + const piped = await runCli(["config", ">", "path"]); + expect(piped.code).toBe(0); + expect(piped.stdout).toBe(direct.stdout); + }, + 2 * CLI_CHILD_TIMEOUT_MS + 2_000, + ); - it("supports `>` chaining across multiple levels", async () => { - const direct = await runCli(["config", "core", "path"]); - const piped = await runCli(["config", ">", "core", ">", "path"]); - expect(piped.code).toBe(0); - expect(piped.stdout).toBe(direct.stdout); - }); + it( + "supports `>` chaining across multiple levels", + async () => { + const direct = await runCli(["config", "core", "path"]); + const piped = await runCli(["config", ">", "core", ">", "path"]); + expect(piped.code).toBe(0); + expect(piped.stdout).toBe(direct.stdout); + }, + 2 * CLI_CHILD_TIMEOUT_MS + 2_000, + ); it("strips a leading `evals` sigil token (no-op at root)", async () => { // From a shell, `evals evals run act --dry-run` should resolve like @@ -313,7 +326,7 @@ describe("CLI entrypoint", () => { }); }); -describe.sequential("core config", () => { +describe.sequential("core config", { timeout: CLI_TEST_TIMEOUT_MS }, () => { // Tests mutate packages/evals/evals.config.json. Snapshot beforeAll, // reset to snapshot before each test, restore afterAll. let snapshot: string; @@ -348,15 +361,19 @@ describe.sequential("core config", () => { expect(saved.core?.tool).toBe("understudy_code"); }); - it("flows persisted core.tool into run dry-run output", async () => { - resetConfig(); - await runCli(["config", "core", "set", "tool", "understudy_code"]); + it( + "flows persisted core.tool into run dry-run output", + async () => { + resetConfig(); + await runCli(["config", "core", "set", "tool", "understudy_code"]); - const { stdout, code } = await runCli(["run", "navigation/open", "--dry-run"]); - expect(code).toBe(0); - const payload = JSON.parse(stdout); - expect(payload.runOptions.coreToolSurface).toBe("understudy_code"); - }, 15_000); + const { stdout, code } = await runCli(["run", "navigation/open", "--dry-run"]); + expect(code).toBe(0); + const payload = JSON.parse(stdout); + expect(payload.runOptions.coreToolSurface).toBe("understudy_code"); + }, + 2 * CLI_CHILD_TIMEOUT_MS + 2_000, + ); it("rejects unknown tool", async () => { resetConfig(); @@ -384,42 +401,54 @@ describe.sequential("core config", () => { expect(stdout + stderr).toContain("Cannot set startup without a tool"); }); - it("rejects startup unsupported by the chosen tool", async () => { - resetConfig(); - // cdp_code does not support tool_create_browserbase. - await runCli(["config", "core", "set", "tool", "cdp_code"]); - const { stdout, stderr, code } = await runCli([ - "config", - "core", - "set", - "startup", - "tool_create_browserbase", - ]); - expect(code).toBe(1); - expect(stdout + stderr).toContain('Tool "cdp_code" does not support startup'); - }, 30_000); + it( + "rejects startup unsupported by the chosen tool", + async () => { + resetConfig(); + // cdp_code does not support tool_create_browserbase. + await runCli(["config", "core", "set", "tool", "cdp_code"]); + const { stdout, stderr, code } = await runCli([ + "config", + "core", + "set", + "startup", + "tool_create_browserbase", + ]); + expect(code).toBe(1); + expect(stdout + stderr).toContain('Tool "cdp_code" does not support startup'); + }, + 2 * CLI_CHILD_TIMEOUT_MS + 2_000, + ); - it("auto-resets startup when a tool change invalidates it", async () => { - resetConfig(); - // cdp_code supports tool_attach_local_cdp; browse_cli does not. - await runCli(["config", "core", "set", "tool", "cdp_code"]); - await runCli(["config", "core", "set", "startup", "tool_attach_local_cdp"]); - const { stdout, code } = await runCli(["config", "core", "set", "tool", "browse_cli"]); - expect(code).toBe(0); - expect(stdout).toContain("Resetting startup"); + it( + "auto-resets startup when a tool change invalidates it", + async () => { + resetConfig(); + // cdp_code supports tool_attach_local_cdp; browse_cli does not. + await runCli(["config", "core", "set", "tool", "cdp_code"]); + await runCli(["config", "core", "set", "startup", "tool_attach_local_cdp"]); + const { stdout, code } = await runCli(["config", "core", "set", "tool", "browse_cli"]); + expect(code).toBe(0); + expect(stdout).toContain("Resetting startup"); - const saved = JSON.parse(fs.readFileSync(SOURCE_CONFIG, "utf-8")); - expect(saved.core?.tool).toBe("browse_cli"); - expect(saved.core?.startup).toBeUndefined(); - }, 30_000); + const saved = JSON.parse(fs.readFileSync(SOURCE_CONFIG, "utf-8")); + expect(saved.core?.tool).toBe("browse_cli"); + expect(saved.core?.startup).toBeUndefined(); + }, + 3 * CLI_CHILD_TIMEOUT_MS + 2_000, + ); - it("reset clears the whole core section", async () => { - resetConfig(); - await runCli(["config", "core", "set", "tool", "understudy_code"]); - const { code } = await runCli(["config", "core", "reset"]); - expect(code).toBe(0); + it( + "reset clears the whole core section", + async () => { + resetConfig(); + await runCli(["config", "core", "set", "tool", "understudy_code"]); + const { code } = await runCli(["config", "core", "reset"]); + expect(code).toBe(0); - const saved = JSON.parse(fs.readFileSync(SOURCE_CONFIG, "utf-8")); - expect(saved.core).toBeUndefined(); - }, 15_000); + const saved = JSON.parse(fs.readFileSync(SOURCE_CONFIG, "utf-8")); + expect(saved.core).toBeUndefined(); + }, + 2 * CLI_CHILD_TIMEOUT_MS + 2_000, + ); }); diff --git a/packages/extension/understudy/a11y/snapshot/capture.test.ts b/packages/extension/understudy/a11y/snapshot/capture.test.ts index b67bd73ea2..543228b015 100644 --- a/packages/extension/understudy/a11y/snapshot/capture.test.ts +++ b/packages/extension/understudy/a11y/snapshot/capture.test.ts @@ -1,13 +1,20 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { FrameContext, FrameDomMaps } from "../../../types/private/index.js"; +import type { FrameContext, FrameDomMaps, SessionDomIndex } from "../../../types/private/index.js"; import type { StagehandLogger } from "../../../logger.js"; import type { Page } from "../../page.js"; import { FrameSelectorResolver } from "../../selectorResolver.js"; import { a11yForFrame } from "./a11yTree.js"; -import { mergeFramesIntoSnapshot, resolveIgnoredNodes, tryScopedSnapshot } from "./capture.js"; +import { + buildFrameExclusionIntervals, + collectPerFrameMaps, + mergeFramesIntoSnapshot, + resolveIgnoredNodes, + tryScopedSnapshot, +} from "./capture.js"; import { domMapsForSession } from "./domTree.js"; +import * as focusSelectors from "./focusSelectors.js"; import { resolveCssFocusFrameAndTail } from "./focusSelectors.js"; -import { ownerSession } from "./sessions.js"; +import { ownerSession, parentSession } from "./sessions.js"; vi.mock("./a11yTree.js", async (importOriginal) => ({ ...(await importOriginal()), @@ -24,6 +31,7 @@ vi.mock("./focusSelectors.js", async (importOriginal) => ({ vi.mock("./sessions.js", async (importOriginal) => ({ ...(await importOriginal()), ownerSession: vi.fn(), + parentSession: vi.fn(), })); const emptyMaps = (): FrameDomMaps => ({ @@ -33,6 +41,195 @@ const emptyMaps = (): FrameDomMaps => ({ urlMap: {}, }); +describe("snapshot frame document slicing", () => { + const documentIndex = (rootBackend = 1): SessionDomIndex => ({ + rootBackend, + absByBe: new Map([ + [rootBackend, "/"], + [rootBackend + 1, "/html[1]"], + [rootBackend + 2, "/html[1]/body[1]"], + ]), + tagByBe: new Map(), + scrollByBe: new Map(), + docRootOf: new Map([ + [rootBackend, rootBackend], + [rootBackend + 1, rootBackend], + [rootBackend + 2, rootBackend], + ]), + contentDocRootByIframe: new Map(), + enterByBe: new Map(), + exitByBe: new Map(), + }); + + beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(a11yForFrame).mockResolvedValue({ + outline: "[0-3] button: Root", + urlMap: {}, + scopeApplied: false, + }); + }); + + it.each(["owner lookup fails", "owner is missing", "content document is missing"])( + "does not duplicate the parent map when a same-session child's %s", + async (failure) => { + const session = { + id: "root-session", + send: vi.fn(async () => { + if (failure === "owner lookup fails") throw new Error("No frame owner found"); + return failure === "owner is missing" ? {} : { backendNodeId: 4 }; + }), + }; + vi.mocked(ownerSession).mockReturnValue(session as never); + const context: FrameContext = { + rootId: "root", + frames: ["root", "child-a", "child-b"], + parentByFrame: new Map([ + ["root", null], + ["child-a", "root"], + ["child-b", "root"], + ]), + }; + const page = { getOrdinal: (id: string) => context.frames.indexOf(id) } as Page; + const { perFrameMaps, perFrameOutlines } = await collectPerFrameMaps( + page, + context, + new Map([[session.id, documentIndex()]]), + undefined, + true, + context.frames, + new Map(), + ); + const snapshot = mergeFramesIntoSnapshot( + context, + perFrameMaps, + perFrameOutlines, + new Map([["root", ""]]), + new Map(), + context.frames, + ); + + expect(snapshot.combinedTree).toBe("[0-3] button: Root"); + expect(snapshot.combinedXpathMap).toEqual({ + "0-1": "/", + "0-2": "/html[1]", + "0-3": "/html[1]/body[1]", + }); + expect([...perFrameMaps.keys()]).toEqual(["root"]); + expect(a11yForFrame).toHaveBeenCalledTimes(1); + }, + ); + + it("does not resolve an ignored child document to the parent document", async () => { + const session = { id: "root-session", send: vi.fn().mockResolvedValue({ backendNodeId: 4 }) }; + vi.mocked(ownerSession).mockReturnValue(session as never); + const focus = vi + .spyOn(focusSelectors, "resolveFocusFrameAndTail") + .mockResolvedValue({ targetFrameId: "child", tailXPath: "/" } as never); + const context: FrameContext = { + rootId: "root", + frames: ["root", "child"], + parentByFrame: new Map([ + ["root", null], + ["child", "root"], + ]), + }; + try { + const ignored = await resolveIgnoredNodes( + {} as Page, + [{ selector: "xpath=/iframe/" }], + context, + new Map([[session.id, documentIndex()]]), + ); + expect(ignored.size).toBe(0); + } finally { + focus.mockRestore(); + } + }); + + it("does not give an unresolved excluded child the parent's whole interval", async () => { + const session = { id: "root-session", send: vi.fn().mockResolvedValue({ backendNodeId: 4 }) }; + vi.mocked(ownerSession).mockReturnValue(session as never); + vi.mocked(parentSession).mockReturnValue(session as never); + const context: FrameContext = { + rootId: "root", + frames: ["root", "child"], + parentByFrame: new Map([ + ["root", null], + ["child", "root"], + ]), + }; + const index = documentIndex(); + index.enterByBe.set(1, 1); + index.exitByBe.set(1, 100); + index.enterByBe.set(4, 10); + index.exitByBe.set(4, 20); + const intervals = await buildFrameExclusionIntervals( + {} as Page, + context, + new Map([[session.id, index]]), + new Map([["root", new Set([4])]]), + ); + expect(intervals.get("root")).toEqual([{ start: 10, end: 20 }]); + expect(intervals.has("child")).toBe(false); + }); + + it("retains root and OOPIF session roots and slices a resolved same-session child", async () => { + const rootSession = { + id: "root-session", + send: vi.fn().mockResolvedValue({ backendNodeId: 4 }), + }; + const oopifSession = { id: "oopif-session", send: vi.fn() }; + vi.mocked(ownerSession).mockImplementation( + (_page, id) => (id === "oopif" ? oopifSession : rootSession) as never, + ); + const context: FrameContext = { + rootId: "root", + frames: ["root", "child", "oopif"], + parentByFrame: new Map([ + ["root", null], + ["child", "root"], + ["oopif", "root"], + ]), + }; + const rootIndex = documentIndex(); + rootIndex.contentDocRootByIframe.set(4, 10); + rootIndex.absByBe.set(10, "/html[1]/body[1]/iframe[1]"); + rootIndex.absByBe.set(11, "/html[1]/body[1]/iframe[1]/html[1]"); + rootIndex.docRootOf.set(10, 10); + rootIndex.docRootOf.set(11, 10); + const { perFrameMaps } = await collectPerFrameMaps( + { getOrdinal: (id: string) => context.frames.indexOf(id) } as Page, + context, + new Map([ + [rootSession.id, rootIndex], + [oopifSession.id, documentIndex(20)], + ]), + undefined, + true, + context.frames, + new Map(), + ); + + expect(perFrameMaps.get("root")?.xpathMap).toEqual({ + "0-1": "/", + "0-2": "/html[1]", + "0-3": "/html[1]/body[1]", + }); + expect(perFrameMaps.get("child")?.xpathMap).toEqual({ "1-10": "/", "1-11": "/html[1]" }); + expect(perFrameMaps.get("oopif")?.xpathMap).toEqual({ + "2-20": "/", + "2-21": "/html[1]", + "2-22": "/html[1]/body[1]", + }); + expect(rootSession.send).toHaveBeenCalledExactlyOnceWith("DOM.getFrameOwner", { + frameId: "child", + }); + expect(oopifSession.send).not.toHaveBeenCalled(); + expect(a11yForFrame).toHaveBeenCalledTimes(3); + }); +}); + describe("snapshot Unicode repair", () => { beforeEach(() => { vi.resetAllMocks(); diff --git a/packages/extension/understudy/a11y/snapshot/capture.ts b/packages/extension/understudy/a11y/snapshot/capture.ts index 0335ed6391..6ac06d3f0d 100644 --- a/packages/extension/understudy/a11y/snapshot/capture.ts +++ b/packages/extension/understudy/a11y/snapshot/capture.ts @@ -356,11 +356,14 @@ export async function collectPerFrameMaps( const sameSessionAsParent = !!parentId && ownerSession(page, parentId) === sess; const docRootBe = await resolveFrameDocRootBackendId(page, frameId, idx, sameSessionAsParent); + if (docRootBe === undefined) continue; const tagNameMap: Record = {}; const xpathMap: Record = {}; const scrollableMap: Record = {}; - const isIgnoredBackendNode = makeIsIgnoredBackendNode(frameId, idx, exclusionIntervalsByFrame); + const isExcluded = makeIsIgnoredBackendNode(frameId, idx, exclusionIntervalsByFrame); + const isIgnoredBackendNode = (backendId: number): boolean => + isExcluded?.(backendId) === true || (!pierce && idx.docRootOf.get(backendId) !== docRootBe); const enc = (be: number) => `${page.getOrdinal(frameId)}-${be}`; const baseAbs = idx.absByBe.get(docRootBe) ?? "/"; @@ -454,6 +457,7 @@ async function resolveIgnoredNodesForLocator( idx, sameSessionAsParent, ); + if (backendNodeId === undefined) return []; return [{ frameId: targetFrameId, backendNodeId }]; } return resolveIgnoredNodesInFrame( @@ -558,10 +562,12 @@ export async function buildFrameExclusionIntervals( const sameSessionAsParent = !!parentId && ownerSession(page, parentId) === ownerSession(page, frameId); const docRootBe = await resolveFrameDocRootBackendId(page, frameId, idx, sameSessionAsParent); - const start = idx.enterByBe.get(docRootBe); - const end = idx.exitByBe.get(docRootBe); - if (typeof start === "number" && typeof end === "number") { - pushInterval(frameId, start, end); + if (docRootBe !== undefined) { + const start = idx.enterByBe.get(docRootBe); + const end = idx.exitByBe.get(docRootBe); + if (typeof start === "number" && typeof end === "number") { + pushInterval(frameId, start, end); + } } for (const childFrameId of listChildrenOf(context.parentByFrame, frameId)) { @@ -692,7 +698,7 @@ async function resolveFrameDocRootBackendId( frameId: string, idx: SessionDomIndex, sameSessionAsParent: boolean, -): Promise { +): Promise { if (!sameSessionAsParent) return idx.rootBackend; const session = ownerSession(page, frameId); try { @@ -704,9 +710,10 @@ async function resolveFrameDocRootBackendId( if (typeof docRootBe === "number") return docRootBe; } } catch { - // + // The frame may have detached since the session DOM index was captured. } - return idx.rootBackend; + // A same-session child's missing document must not duplicate the parent document. + return undefined; } /** diff --git a/packages/extension/understudy/a11y/snapshot/domTree.test.ts b/packages/extension/understudy/a11y/snapshot/domTree.test.ts index 719d3168ae..0258ecf369 100644 --- a/packages/extension/understudy/a11y/snapshot/domTree.test.ts +++ b/packages/extension/understudy/a11y/snapshot/domTree.test.ts @@ -1,7 +1,7 @@ import type { Protocol } from "devtools-protocol"; import { describe, expect, it, vi } from "vitest"; import type { CDPSessionLike } from "../../cdp.js"; -import { getDomTreeWithFallback, hydrateDomTree } from "./domTree.js"; +import { buildSessionDomIndex, getDomTreeWithFallback, hydrateDomTree } from "./domTree.js"; describe("DOM tree adaptive retries", () => { it("throws the last original DOM.getDocument retry error", async () => { @@ -53,3 +53,35 @@ describe("DOM tree adaptive retries", () => { expect(send).toHaveBeenCalledOnce(); }); }); + +describe("session DOM index frame and shadow traversal", () => { + it.each([false, true])("keeps iframe documents with pierceShadow=%s", async (pierce) => { + const node = ( + id: number, + name: string, + children: Protocol.DOM.Node[] = [], + ): Protocol.DOM.Node => ({ + nodeId: id, + backendNodeId: id, + nodeType: name === "#document" ? 9 : 1, + nodeName: name, + localName: name.toLowerCase(), + nodeValue: "", + childNodeCount: children.length, + children, + }); + const child = node(5, "#document", [node(6, "HTML", [node(7, "INPUT")])]); + const frame = { ...node(4, "IFRAME"), contentDocument: child }; + const host = { + ...node(8, "DIV"), + shadowRoots: [node(9, "#document-fragment", [node(10, "BUTTON")])], + }; + const root = node(1, "#document", [node(2, "HTML", [node(3, "BODY", [frame, host])])]); + const send = vi.fn(async (method: string) => (method === "DOM.getDocument" ? { root } : {})); + const index = await buildSessionDomIndex({ send } as unknown as CDPSessionLike, pierce); + expect(send).toHaveBeenCalledWith("DOM.getDocument", { depth: -1, pierce: true }); + expect(index.contentDocRootByIframe.get(4)).toBe(5); + expect(index.docRootOf.get(7)).toBe(5); + expect(index.absByBe.has(10)).toBe(pierce); + }); +}); diff --git a/packages/extension/understudy/a11y/snapshot/domTree.ts b/packages/extension/understudy/a11y/snapshot/domTree.ts index 17fe646919..287709095f 100644 --- a/packages/extension/understudy/a11y/snapshot/domTree.ts +++ b/packages/extension/understudy/a11y/snapshot/domTree.ts @@ -235,7 +235,9 @@ export async function buildSessionDomIndex( pierce: boolean, ): Promise { await session.send("DOM.enable").catch(() => {}); - const root = await getDomTreeWithFallback(session, pierce); + // CDP uses one piercing flag for both shadow roots and iframe documents. + // Fetch both, then independently omit shadow roots while building the index. + const root = await getDomTreeWithFallback(session, true); const absByBe = new Map(); const tagByBe = new Map(); @@ -289,7 +291,7 @@ export async function buildSessionDomIndex( } } - for (const sr of node.shadowRoots ?? []) { + for (const sr of pierce ? (node.shadowRoots ?? []) : []) { stack.push({ node: sr, xp: joinXPath(xp, "//"), diff --git a/packages/extension/understudy/frame.ts b/packages/extension/understudy/frame.ts index eae60f09d5..bfa7dc2c45 100644 --- a/packages/extension/understudy/frame.ts +++ b/packages/extension/understudy/frame.ts @@ -2,6 +2,7 @@ import { Protocol } from "devtools-protocol"; import type { CDPSessionLike } from "./cdp.js"; import { Locator } from "./locator.js"; +import { waitForScreenshot } from "./screenshotUtils.js"; import { executionContexts } from "./executionContextRegistry.js"; import type { StagehandLogger } from "../logger.js"; @@ -223,8 +224,11 @@ export class Frame implements FrameManager { type?: "png" | "jpeg"; quality?: number; scale?: number; + signal?: AbortSignal; }): Promise { - await this.session.send("Page.enable"); + const signal = options?.signal; + signal?.throwIfAborted(); + await waitForScreenshot(this.session.send("Page.enable"), signal); const format = options?.type ?? "png"; const params: Protocol.Page.CaptureScreenshotRequest & { scale?: number } = { format, @@ -255,9 +259,11 @@ export class Frame implements FrameManager { params.quality = Math.min(100, Math.max(0, q)); } - const { data } = await this.session.send( - "Page.captureScreenshot", - params, + // Headless Chrome can wait indefinitely for a background tab to produce a frame. + await waitForScreenshot(this.session.send("Page.bringToFront"), signal); + const { data } = await waitForScreenshot( + this.session.send("Page.captureScreenshot", params), + signal, ); return base64ToBytes(data); } diff --git a/packages/extension/understudy/page.ts b/packages/extension/understudy/page.ts index 1fd5e650c9..8591e86a2d 100644 --- a/packages/extension/understudy/page.ts +++ b/packages/extension/understudy/page.ts @@ -51,10 +51,11 @@ import { normalizeScreenshotClip, runScreenshotCleanups, setTransparentBackground, + withScreenshotLock, + waitForScreenshot, type ScreenshotCleanup, } from "./screenshotUtils.js"; import { InitScriptSource } from "../types/private/index.js"; -import { withTimeout } from "../timeoutConfig.js"; /** * Page @@ -1242,48 +1243,65 @@ export class Page { const scaleMode: NonNullable = opts.scale ?? "device"; const frames = collectFramesForScreenshot(this); const clip = opts.clip ? normalizeScreenshotClip(opts.clip) : undefined; - const captureScale = await computeScreenshotScale(this, scaleMode); const maskLocators = opts.mask ?? []; const cleanupTasks: ScreenshotCleanup[] = []; - const exec = async (): Promise => { + const exec = async (signal: AbortSignal): Promise => { try { + const captureScale = await waitForScreenshot( + computeScreenshotScale(this, scaleMode), + signal, + ); if (opts.omitBackground) { + signal.throwIfAborted(); cleanupTasks.push(await setTransparentBackground(this.mainSession)); } if (animationsMode === "disabled") { + signal.throwIfAborted(); cleanupTasks.push(await disableAnimations(frames)); } if (caretMode === "hide") { + signal.throwIfAborted(); cleanupTasks.push(await hideCaret(frames)); } if (opts.style && opts.style.trim()) { + signal.throwIfAborted(); cleanupTasks.push(await applyStyleToFrames(frames, opts.style, "custom")); } if (maskLocators.length > 0) { + signal.throwIfAborted(); cleanupTasks.push(await applyMaskOverlays(maskLocators, opts.maskColor ?? "#FF00FF")); } - const buffer = await this.mainFrameWrapper.screenshot({ - fullPage: opts.fullPage, - clip, - type, - quality: type === "jpeg" ? opts.quality : undefined, - scale: captureScale, - }); - - return buffer; + // Setup and cleanup mutate this page only. Hold the browser-wide lock solely + // while activating and capturing, so a stalled page cannot block other tabs. + return await waitForScreenshot( + withScreenshotLock( + this.conn, + () => + this.mainFrameWrapper.screenshot({ + fullPage: opts.fullPage, + clip, + type, + quality: type === "jpeg" ? opts.quality : undefined, + scale: captureScale, + signal, + }), + undefined, + ), + signal, + ); } finally { await runScreenshotCleanups(cleanupTasks); } }; - return await withTimeout(exec(), opts.timeout, "screenshot"); + return await withScreenshotLock(this, exec, opts.timeout); } /** diff --git a/packages/extension/understudy/screenshotUtils.test.ts b/packages/extension/understudy/screenshotUtils.test.ts new file mode 100644 index 0000000000..47235c1689 --- /dev/null +++ b/packages/extension/understudy/screenshotUtils.test.ts @@ -0,0 +1,205 @@ +import { trace } from "@opentelemetry/api"; +import { Page } from "./page.js"; +import { CdpConnection } from "./cdp.js"; +import { Frame } from "./frame.js"; +import { StagehandLogger } from "../logger.js"; +import { describe, expect, it, vi } from "vitest"; +import { withScreenshotLock } from "./screenshotUtils.js"; + +function connection() { + return { send: vi.fn(), on: vi.fn(), off: vi.fn(), close: vi.fn(), id: null }; +} + +function captureGate() { + let release!: (value: Uint8Array) => void; + const pending = new Promise((resolve) => { + release = resolve; + }); + return { pending, release: () => release(new Uint8Array([1])) }; +} + +describe("screenshot serialization", () => { + it("keeps another page's capture queued until the first finishes", async () => { + const browser = connection(); + const first = captureGate(); + const nextCapture = vi.fn(async () => new Uint8Array([2])); + const a = withScreenshotLock(browser, () => first.pending, undefined); + const b = withScreenshotLock(browser, nextCapture, undefined); + await Promise.resolve(); + expect(nextCapture).not.toHaveBeenCalled(); + first.release(); + await expect(a).resolves.toEqual(new Uint8Array([1])); + await expect(b).resolves.toEqual(new Uint8Array([2])); + }); + + it("does not block a different browser", async () => { + const first = captureGate(); + const a = withScreenshotLock(connection(), () => first.pending, undefined); + try { + await expect( + withScreenshotLock(connection(), async () => new Uint8Array([2]), undefined), + ).resolves.toEqual(new Uint8Array([2])); + } finally { + first.release(); + await a; + } + }); + + it("continues after a failed capture", async () => { + const browser = connection(); + const a = withScreenshotLock( + browser, + async () => { + throw new Error("capture failed"); + }, + undefined, + ); + const b = withScreenshotLock(browser, async () => new Uint8Array([2]), undefined); + await expect(a).rejects.toThrow("capture failed"); + await expect(b).resolves.toEqual(new Uint8Array([2])); + }); + + it("releases a stalled capture only after cleanup, ignoring its late response", async () => { + vi.useFakeTimers(); + const browser = connection(); + let respond!: (value: { data: string }) => void; + const response = new Promise<{ data: string }>((resolve) => { + respond = resolve; + }); + browser.send.mockImplementation(async (method: string) => + method === "Page.captureScreenshot" ? response : {}, + ); + const frame = new Frame( + browser, + "frame", + "page", + false, + new StagehandLogger({ tracer: trace.getTracer("screenshot-test") }, () => {}), + ); + const cleanup = captureGate(); + const cleanupStarted = vi.fn(); + const first = withScreenshotLock( + browser, + async (signal) => { + try { + return await frame.screenshot({ signal }); + } finally { + cleanupStarted(); + await cleanup.pending; + } + }, + 10, + ); + const nextCapture = vi.fn(async () => new Uint8Array([2])); + const second = withScreenshotLock(browser, nextCapture, undefined); + const timedOut = expect(first).rejects.toThrow(/screenshot.*timed out/i); + const blocked = expect(second).rejects.toThrow(/still recovering/); + try { + await vi.advanceTimersByTimeAsync(10); + await timedOut; + expect(browser.send).toHaveBeenCalledWith("Page.captureScreenshot", expect.any(Object)); + expect(cleanupStarted).toHaveBeenCalledOnce(); + expect(nextCapture).not.toHaveBeenCalled(); + await blocked; + cleanup.release(); + await vi.advanceTimersByTimeAsync(0); + await expect(withScreenshotLock(browser, nextCapture, undefined)).resolves.toEqual( + new Uint8Array([2]), + ); + respond({ data: "AQ==" }); + await vi.advanceTimersByTimeAsync(0); + expect(cleanupStarted).toHaveBeenCalledOnce(); + expect(nextCapture).toHaveBeenCalledOnce(); + } finally { + cleanup.release(); + respond({ data: "AQ==" }); + vi.useRealTimers(); + } + }); + + it.each(["setup", "cleanup"] as const)( + "does not block another tab when page %s stalls", + async (phase) => { + vi.useFakeTimers(); + const logger = new StagehandLogger({ tracer: trace.getTracer("screenshot-test") }, () => {}); + const browser = new CdpConnection( + { + connected: true, + send: vi.fn(), + close: vi.fn(async () => {}), + onMessage: vi.fn(), + onClose: vi.fn(), + onError: vi.fn(), + }, + logger, + ); + const makePage = (id: string) => { + const session = connection(); + session.send.mockResolvedValue({ data: "AQ==" }); + return new Page(browser, session, id, id, logger); + }; + const firstPage = makePage("first"); + const otherPage = makePage("other"); + vi.spyOn(firstPage, "frames").mockReturnValue([firstPage.mainFrame()]); + const stalled = captureGate(); + const evaluate = vi.spyOn(firstPage.mainFrame(), "evaluate").mockResolvedValue(undefined); + if (phase === "cleanup") evaluate.mockResolvedValueOnce(undefined); + evaluate.mockImplementationOnce(() => stalled.pending); + const first = firstPage.screenshot({ timeout: 10 }); + const timedOut = expect(first).rejects.toThrow(/screenshot.*timed out/i); + try { + await vi.advanceTimersByTimeAsync(10); + await timedOut; + await expect(firstPage.screenshot({ caret: "initial" })).rejects.toThrow( + /still recovering/, + ); + let result: Uint8Array | undefined; + let error: unknown; + const other = otherPage.screenshot({ caret: "initial", timeout: 20 }).then( + (value) => { + result = value; + }, + (reason: unknown) => { + error = reason; + }, + ); + await vi.advanceTimersByTimeAsync(20); + await other; + expect(error).toBeUndefined(); + expect(result).toEqual(new Uint8Array([1])); + stalled.release(); + await vi.advanceTimersByTimeAsync(1); + await expect(firstPage.screenshot({ caret: "initial" })).resolves.toEqual( + new Uint8Array([1]), + ); + } finally { + stalled.release(); + await vi.advanceTimersByTimeAsync(0); + firstPage.dispose(); + otherPage.dispose(); + vi.useRealTimers(); + } + }, + ); + + it("does not activate a queued page after its timeout", async () => { + vi.useFakeTimers(); + const browser = connection(); + const first = captureGate(); + const a = withScreenshotLock(browser, () => first.pending, undefined); + const nextCapture = vi.fn(async () => new Uint8Array([2])); + const b = withScreenshotLock(browser, nextCapture, 10); + const timedOut = expect(b).rejects.toThrow(/screenshot.*timed out/i); + try { + await vi.advanceTimersByTimeAsync(10); + await timedOut; + first.release(); + await a; + await withScreenshotLock(browser, async () => new Uint8Array([3]), undefined); + expect(nextCapture).not.toHaveBeenCalled(); + } finally { + first.release(); + vi.useRealTimers(); + } + }); +}); diff --git a/packages/extension/understudy/screenshotUtils.ts b/packages/extension/understudy/screenshotUtils.ts index ed153d4ff0..d1f972801a 100644 --- a/packages/extension/understudy/screenshotUtils.ts +++ b/packages/extension/understudy/screenshotUtils.ts @@ -1,3 +1,4 @@ +import { TimeoutError } from "../errors.js"; import { Protocol } from "devtools-protocol"; import type { CDPSessionLike } from "./cdp.js"; import type { DeepLocatorDelegate } from "./deepLocator.js"; @@ -9,6 +10,84 @@ import { resolveMaskRect } from "../dom/screenshotScripts/index.js"; export type ScreenshotCleanup = () => Promise | void; +const screenshotQueues = new WeakMap; blocked: AbortController }>(); + +/** Serialize page mutations, or the browser-wide activation/capture critical section. */ +export async function withScreenshotLock( + owner: object, + capture: (signal: AbortSignal) => Promise, + timeout: number | undefined, +): Promise { + const queue = screenshotQueues.get(owner) ?? { + tail: Promise.resolve(), + blocked: new AbortController(), + }; + // Keep late setup/cleanup isolated, but never make callers wait indefinitely for it. + queue.blocked.signal.throwIfAborted(); + const controller = new AbortController(); + let started = false; + const timer = + typeof timeout === "number" && Number.isFinite(timeout) && timeout > 0 + ? setTimeout( + () => { + controller.abort(new TimeoutError("screenshot", timeout)); + if (started) { + queue.blocked.abort( + new Error("screenshot: a previous timed-out capture is still recovering"), + ); + } + }, + Math.min(timeout, 2_147_483_647), + ) + : undefined; + const pending = queue.tail.then(() => { + queue.blocked.signal.throwIfAborted(); + controller.signal.throwIfAborted(); + started = true; + return capture(controller.signal); + }); + const released = pending.then( + () => {}, + () => {}, + ); + queue.tail = released; + screenshotQueues.set(owner, queue); + try { + return await waitForScreenshot( + waitForScreenshot(pending, queue.blocked.signal), + controller.signal, + ); + } finally { + if (timer !== undefined) clearTimeout(timer); + void released.then(() => { + if (queue.tail === released) screenshotQueues.delete(owner); + }); + } +} + +/** Stop waiting for a stalled CDP capture; its eventual response cannot resume cleanup. */ +export async function waitForScreenshot(pending: Promise, signal?: AbortSignal): Promise { + if (!signal) return await pending; + return await new Promise((resolve, reject) => { + const onAbort = () => { + signal.removeEventListener("abort", onAbort); + reject(signal.reason); + }; + if (signal.aborted) onAbort(); + else signal.addEventListener("abort", onAbort, { once: true }); + pending.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error: unknown) => { + signal.removeEventListener("abort", onAbort); + reject(error); + }, + ); + }); +} + export function collectFramesForScreenshot(page: Page): Frame[] { const seen = new Map(); const main = page.mainFrame(); diff --git a/packages/integrations/core/README.md b/packages/integrations/core/README.md new file mode 100644 index 0000000000..326d89f002 --- /dev/null +++ b/packages/integrations/core/README.md @@ -0,0 +1,57 @@ +# Shared Stagehand facade + +`@browserbasehq/stagehand-integrations` defines the shared `run`, `snapshot`, and +`screenshot` contract. Harnesses mount these tools through the `stagehand-facade` +stdio server or use `StagehandFacadeTools` with an existing Stagehand instance. +Tool schemas, instructions, and browser behavior live in this package. + +## Browser ownership and configuration + +The stdio host owns one browser for its lifetime. It closes Stagehand and then the +browser explicitly. Browserbase sessions use `keepAlive: true` so transport loss +does not silently replace the session. A hidden `about:blank` keeper is excluded +from the facade's pages and page events; closing the last visible tab permits a +new visible page in the same browser. In-process callers can pass +`{ keeperPage: false }` to `StagehandFacadeTools` and remain responsible for cleanup. + +| Setting | Default and behavior | +| ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `STAGEHAND_BROWSER` | `browserbase` when `BROWSERBASE_API_KEY` is set, otherwise `local`; accepts these two values only. Local Chrome is headed. | +| `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID` | Existing Browserbase credentials and optional project ID; the API key is required for Browserbase. | +| `STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS` | `3600`; integer seconds from `1` through `21600`. | +| `STAGEHAND_BROWSERBASE_PROXIES`, `STAGEHAND_BROWSERBASE_VERIFIED` | Unset leaves the corresponding Browserbase API option unspecified. Accepts `1/true/yes/on` and `0/false/no/off`. The eval runner explicitly defaults both to true. | +| `STAGEHAND_BROWSERBASE_EXTENSION_ID` | Unset uploads a packaged extension for this session; set to reuse the host's uploaded extension. The host owns the shared upload's cleanup. | +| `STAGEHAND_MODEL_NAME`, `STAGEHAND_MODEL_API_KEY` | Optional Stagehand model configuration. A model API key requires an explicit model name. Plain facade navigation and inspection do not invoke a model. | + +`--surface=playwright` is the default stdio description. `--surface=legacy` selects +the earlier description with the same runtime. `EXPLICIT_SNAPSHOT_ACTIONS=1` +changes Playwright-surface instructions to prefer snapshot IDs for simple actions; +it does not introduce another implementation or change the legacy description. +The runner-only `session_info` call reports browser identity and is absent from +the model-facing tool list. + +A failed CDP attach releases a Browserbase session created by that launch even +with `keepAlive: true`. Failed attachment to an existing session does not release +it. There is still a shutdown limitation: the stdio host waits at most five seconds +for pending initialization before proceeding to bounded cleanup. If creation or +attachment has not returned a handle when the process exits, the host cannot +explicitly release that session. A cancellable launch/early lease API is a separate +follow-up; the configured remote session expiry remains the fallback. + +## Local regression checks + +From the repository root, with workspace dependencies and the extension, SDK, and +core builds available: + +```sh +pnpm --filter @browserbasehq/stagehand-integrations test:unit +pnpm --filter @browserbasehq/stagehand-integrations test:browser +``` + +The browser suite uses local Chrome and static loopback fixtures with no remote +sites, credentials, or model requests. It compares label/strict-locator behavior +with native Playwright and checks real same-origin and out-of-process frames +through the SDK and extension. DOM tests and hooks have 20-second limits; the +extension/frame test has a 45-second limit. Chrome must already be installed; +`PLAYWRIGHT_CHROMIUM_CHANNEL` selects the DOM suite's channel (default `chrome`). +The SDK's local browser launcher also supports `CHROME_PATH`. diff --git a/packages/integrations/core/integration/facade-frames.test.ts b/packages/integrations/core/integration/facade-frames.test.ts new file mode 100644 index 0000000000..9d617681d0 --- /dev/null +++ b/packages/integrations/core/integration/facade-frames.test.ts @@ -0,0 +1,172 @@ +import { createServer } from "node:http"; +import { expect, it, vi } from "vitest"; +import { localBrowser, Stagehand, type StagehandBrowser } from "@browserbasehq/stagehand"; +import { StagehandFacadeTools } from "../src/facade/tools.js"; + +type FrameState = { + fixture: string; + active: string; + click: number; + input: number; + change: number; + lastClick: string | null; + scrollY: number; + targetScroll: number; + targetInView: boolean; +}; + +it("runs the shared facade against same-origin and out-of-process local frames", async () => { + let port = 0; + const server = createServer((request, response) => { + response.setHeader("content-type", "text/html; charset=utf-8"); + if (request.url === "/nested") { + response.end(""); + return; + } + if (request.url === "/same" || request.url === "/cross") { + const name = request.url === "/same" ? "Same value" : "Cross value"; + response.end(` + + +
+
+
+
+
Scrollable target
+
+ `); + return; + } + response.end( + `

Local frame fixture

`, + ); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + // The fixture uses both 127.0.0.1 and localhost to force site isolation. + // An unspecified bind accepts IPv4 and IPv6 localhost resolution. + server.listen(0, resolve); + }); + port = (server.address() as { port: number }).port; + let browser: StagehandBrowser | undefined; + let stagehand: Stagehand | undefined; + const generate = vi.fn(async (): Promise => { + throw new Error("This local fixture must not call a model"); + }); + try { + browser = await localBrowser.launch({ headless: true, args: ["--site-per-process"] }); + stagehand = await Stagehand.create({ browser, model: { generate }, logging: { level: "off" } }); + const tools = new StagehandFacadeTools(stagehand); + await tools.run(`await page.goto(${JSON.stringify(`http://127.0.0.1:${port}/`)});`); + const page = await stagehand.browser.context.activePage(); + if (!page) throw new Error("Missing local fixture page"); + const inspect = () => + page.evaluate( + () => + new Promise((resolve, reject) => { + const replies: FrameState[] = []; + const timeout = setTimeout(() => { + removeEventListener("message", listener); + reject(new Error("Frame fixture did not report its state")); + }, 2_000); + const listener = (event: MessageEvent) => { + if (!["/same", "/cross"].includes(event.data?.fixture)) return; + replies.push(event.data as FrameState); + if (replies.length === 2) { + clearTimeout(timeout); + removeEventListener("message", listener); + resolve(replies.sort((a, b) => a.fixture.localeCompare(b.fixture))); + } + }; + addEventListener("message", listener); + for (const frame of document.querySelectorAll("iframe")) + frame.contentWindow?.postMessage("inspect-fixture", "*"); + }), + ); + + for (const id of ["same", "cross"]) { + await tools.run(`await page.frameLocator("#${id}").locator("#focus").focus();`); + const state = (await inspect()).find((state) => state.fixture === `/${id}`); + expect(state).toMatchObject({ active: "focus", click: 0, input: 0, change: 0 }); + } + for (const id of ["same", "cross"]) { + await tools.run( + `await page.frameLocator("#${id}").locator("#target").scrollIntoViewIfNeeded();`, + ); + } + for (const state of await inspect()) { + expect(state.targetInView).toBe(true); + expect(state.scrollY).toBeGreaterThan(0); + expect(state.targetScroll).toBe(45); + expect(state).toMatchObject({ click: 0, input: 0, change: 0 }); + } + for (const id of ["same", "cross"]) { + await expect( + tools.run(` + const frame = page.frameLocator("#${id}"); + const choices = frame.locator(".choice"); + return [await frame.getByRole("button", {name:"Frame action", exact:true}).count(), + await (await choices.nth(1).all())[0].textContent(), + await (await choices.last().all())[0].textContent()]; + `), + ).resolves.toEqual([1, "Second", "Second"]); + await tools.run( + `await page.frameLocator("#${id}").locator("#a, #b").locator("button").nth(0).click();`, + ); + await expect( + tools.run(`return await page.frameLocator("#${id}").getByPlaceholder(/value/i).count();`), + ).rejects.toThrow(/regular-expression attribute matching is not supported/); + await tools.run(`await page.frameLocator("#${id}").locator("#value").fill("${id}-origin");`); + } + for (const state of await inspect()) expect(state.lastClick).toBe("first"); + await expect( + tools.run(`return [await page.frameLocator("#same").locator("#value").inputValue(), + await page.frameLocator("#cross").locator("#value").inputValue()];`), + ).resolves.toEqual(["same-origin", "cross-origin"]); + const snapshot = await tools.snapshot({ includeIframes: true }); + expect(snapshot).toContain("Same value"); + expect(snapshot).toContain("Cross value"); + expect(snapshot).toContain("Shadow-only action"); + // Confirm the cross-site frame really is an OOPIF, rather than assuming + // that every cross-origin iframe has a distinct renderer process. + const transport = stagehand.rpcClient.cdp as unknown as { + sendCommand(method: string): Promise<{ targetInfos: Array<{ type: string; url: string }> }>; + }; + const { targetInfos } = await transport.sendCommand("Target.getTargets"); + expect( + targetInfos.some( + (target) => target.type === "iframe" && target.url === `http://localhost:${port}/cross`, + ), + ).toBe(true); + expect(generate).not.toHaveBeenCalled(); + } finally { + try { + await stagehand?.close(); + } finally { + try { + await browser?.close(); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + } + } +}, 60_000); diff --git a/packages/integrations/core/src/facade/runtime.ts b/packages/integrations/core/src/facade/runtime.ts index 2004b2e4b3..f4d9faf75b 100644 --- a/packages/integrations/core/src/facade/runtime.ts +++ b/packages/integrations/core/src/facade/runtime.ts @@ -62,6 +62,7 @@ type RawLocator = { innerHtml(): Promise; textContent(): Promise; scrollTo(percent: number): Promise; + centroid(): Promise<{ x: number; y: number }>; }; type CompatSelectOption = @@ -1136,6 +1137,30 @@ export async function createPlaywrightCompatRuntime( }); } + frameLocator(selector: string): CompatFrameLocator { + record("calls", "locator.frameLocator"); + return this.derived({ kind: "selector", value: selector }).contentFrame(); + } + + /** The iframe element this locator points at, entered as a frame. Only css-selector plans map onto a hop selector. */ + contentFrame(): CompatFrameLocator { + record("calls", "locator.contentFrame"); + if (this.plan.length === 0 || this.plan.some((step) => step.kind !== "selector")) { + return frameUnsupported( + "contentFrame", + "only css-selector locators can be entered as frames; use page.frameLocator(cssSelector)", + ); + } + const selectors = this.plan.map((step) => + (step as { kind: "selector"; value: string }).value.trim(), + ); + const hop = + selectors.length > 1 && selectors.some((selector) => selector.includes(",")) + ? selectors.map((selector) => `:is(${selector})`).join(" ") + : selectors.join(" "); + return frameLocatorProxy(new CompatFrameLocator([hop], this.state)); + } + first(): CompatLocator { record("calls", "locator.first"); return this.derived({ kind: "nth", index: 0 }); @@ -1617,6 +1642,729 @@ export async function createPlaywrightCompatRuntime( const locatorProxy = (locator: CompatLocator): CompatLocator => guard("locator", locator); + // --------------------------------------------------------------------------- + // page.frameLocator(): locators scoped to an