Standardize eval policy, budgets, session metadata, traces, and cost accounting

.changeset/eval-capture-deadlines.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Bound experimental batch and RPC deadlines so callers can stop waiting without replaying actions or accepting late capture state.

.changeset/eval-cdp-heartbeat.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Add configurable bounded CDP heartbeats and sanitized disconnect diagnostics with cleanup on shutdown.

.changeset/eval-frame-snapshot-maps.md

@@ -0,0 +1,7 @@
+---
+"@browserbasehq/stagehand-extension": patch
+"@browserbasehq/stagehand-go": patch
+---
+
+Snapshot references remain valid across same-origin and out-of-process frame captures, including when shadow-root piercing is disabled.
+Refresh the Go SDK's embedded extension to include the fix.

packages/evals/core/contracts/tool.ts

@@ -140,6 +140,11 @@ export interface BrowserSessionLoss {
   cause: string;
   tool?: string;
   at?: string;
+  provider?: "local" | "browserbase";
+  sessionId?: string;
+  /** Elapsed time since the facade started browser launch, including initialization. */
+  sessionAgeMs?: number;
+  sessionTimeoutMs?: number;
 }
 
 /** MCP content returned unchanged by a runner call into its existing surface. */

packages/evals/core/tools/browserSessionLoss.ts

@@ -33,6 +33,22 @@ export function parseSessionLossTelemetry(line: string): BrowserSessionLoss | un
       cause: sanitizeErrorMessage(parsed.cause),
       ...(typeof parsed.tool === "string" && { tool: parsed.tool }),
       ...(typeof parsed.at === "string" && { at: parsed.at }),
+      ...((parsed.provider === "local" || parsed.provider === "browserbase") && {
+        provider: parsed.provider,
+      }),
+      ...(typeof parsed.sessionId === "string" && {
+        sessionId: sanitizeErrorMessage(parsed.sessionId),
+      }),
+      ...(typeof parsed.sessionAgeMs === "number" &&
+        Number.isFinite(parsed.sessionAgeMs) &&
+        parsed.sessionAgeMs >= 0 && {
+          sessionAgeMs: parsed.sessionAgeMs,
+        }),
+      ...(typeof parsed.sessionTimeoutMs === "number" &&
+        Number.isFinite(parsed.sessionTimeoutMs) &&
+        parsed.sessionTimeoutMs >= 0 && {
+          sessionTimeoutMs: parsed.sessionTimeoutMs,
+        }),
     };
   } catch {
     return undefined;

packages/evals/docs/verifier-gates.md

@@ -0,0 +1,29 @@
+# Verifier evidence gates
+
+External harnesses use the existing V3 verifier with the task's precomputed rubric when available. A requested verification that errors or returns a verifier-uncertainty sentinel fails closed: `_success` is false, `verifierError` explains the failure, and `agentReportedSuccess` preserves the original self-report. Such rows are ungraded and must not be presented as verified benchmark outcomes.
+
+For completed grades, the raw judge verdict is retained under `judge` in `scores/result.json`. The top-level result, `task_data.json`, and the run row carry the same adjusted outcome. Failed or uncertain verification instead persists `{ graded: false, verifierError, judge? }`, without top-level outcome or process scores; `judge` is present only if the verifier returned a response. `scores/gates.json` records diagnostics for completed grades when trajectory persistence is enabled.
+
+## Outcome
+
+A judge pass is rejected when the final answer is empty or, if a mounted-tool matcher is available, the trajectory has no browser tool calls. Grounding checks are advisory by default. `EVAL_REQUIRE_GROUNDING=1` additionally rejects an answer whose checked numeric findings all lack matching observations from a known non-search page. Captured step and terminal accessibility trees are included; unknown-page text cannot establish grounding. A terminal match is recorded as `groundedAtFinalObservation`, while untrusted matches can be recorded as `seenOnUnknownPage`. The check is a text heuristic: images, paraphrases and valid snippet sources can escape its matching, so it is not a replacement for rubric verification.
+
+Execution state remains separate. When populated by the runner or adapter, `harnessStatus`, `harnessStopReason` and `terminationReason` describe completion, budget exhaustion, abort, SDK error and browser loss. The verifier does not synthesize missing lifecycle metadata; its absence means unavailable. A supported task completion can still pass after a disconnect; execution error alone does not erase earlier evidence. An unfinished task must fail its rubric.
+
+## Process
+
+`processScoreStrict` recomputes the weighted process score with explicit `evidenceInsufficient` criteria earning zero while retaining their maximum points in the denominator. `processScore` uses this score; `processScoreLenient` preserves the judge's aggregate. Not-applicable criteria are excluded. Without a criterion breakdown, the judge's aggregate is retained and `scoringIncomplete` flags a short result against the rubric.
+
+Blocker wording is recorded as `blockerMentioned` on criterion diagnostics. It never changes points by itself: permitted fallback and stop-boundary explanations can correctly mention a blocker. The rubric and observed evidence determine whether the requirement was satisfied. This replaces the campaign's overbroad blocker substring heuristic.
+
+## Reporting
+
+These fields depend on the producing runner; this verifier layer forwards them but does not make every harness emit them:
+
+- Where supplied, `facade_tool_calls` and `facade_tool_call_failures` count attempted and failed browser work. Missing counters are unknown, not measured zero. Run-level browser loss comes from runner-owned telemetry. Normalized steps do not provide trusted per-call loss attribution, so tool-output text cannot exclude failures or synthesize a count after session loss. A graded pass with an explicit zero browser-call count is shown in the batch summary; with `EVAL_MAX_UNVERIFIABLE_CRITERIA` enabled, it fails the batch gate.
+- Separate agent, evidence-capture and verifier wall times are available only when recorded by the producer.
+- Usage must be interpreted with the producer's presence marker and cache convention. Legacy runners may supply zero placeholders; without an explicit presence marker, zero does not establish measured usage. Historical Cursor CLI usage remains unreported.
+- A producer's `cost_source` distinguishes reported dollars from a catalog estimate (`computed`). Shared runner estimates use the dated catalog in `pricing/pricing.json`; they are not invoices. This verifier layer does not compute estimates. Without provenance, cost origin is unavailable; unknown, tier-dependent or subscription costs must not be inferred as zero.
+- `harnessImplementation` records adapter and SDK versions when supplied. Its absence means unknown implementation; historical labels are preserved.
+
+Use `VERIFIER_PERSIST_TRAJECTORIES=1` for reviewable evidence. HardBench's compatibility gate rejects verifier errors, uncertainty sentinels, missing criteria and self-report fallbacks before accepting a result. Offline transport checks establish integration compatibility; live rubric accuracy still requires the separately recorded live fixtures.

packages/evals/framework/agentToolRuntime.ts

@@ -3,6 +3,7 @@ import { prepareCoreBrowserTarget } from "../core/targets/index.js";
 import { getCoreTool } from "../core/tools/registry.js";
 import { EvalsError } from "../errors.js";
 import type { EvalLogger } from "../logger.js";
+import { browserSessionFromMetadata, type BrowserSessionInfo } from "./browserSession.js";
 
 export interface AgentToolRuntimeInput {
   toolSurface: ToolSurface;
@@ -13,6 +14,12 @@ export interface AgentToolRuntimeInput {
 
 export interface StartedAgentToolRuntime {
   running: ToolStartResult;
+  /**
+   * Browser behind the surface, whether the runner provided it (Browserbase
+   * CDP target) or the tool created it (facade, stagehand_code). Known before
+   * the agent starts so the session URL can head the task log.
+   */
+  browserSession: BrowserSessionInfo;
   /** Closes the tool-owned runtime, then the runner-owned browser target. */
   cleanup: () => Promise<void>;
 }
@@ -49,6 +56,10 @@ export async function startAgentToolRuntime(
   let cleanupPromise: Promise<void> | undefined;
   return {
     running,
+    browserSession: browserSessionFromMetadata(
+      { ...running.metadata, ...target.metadata },
+      input.environment,
+    ),
     cleanup: async () => {
       cleanupPromise ??= (async () => {
         try {

packages/evals/framework/benchHarness.ts

@@ -1,5 +1,6 @@
 import { V3, normalizeRubric, type AvailableModel, type TaskSpec } from "stagehand-v3";
 import { EvalsError } from "../errors.js";
+import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness";
 import type { EvalLogger } from "../logger.js";
 import type { StagehandInitResult } from "../initStagehand.js";
 import type { EvalInput } from "../types/evals.js";
@@ -26,7 +27,13 @@ import {
   buildExternalHarnessTaskPlan,
   type ExternalHarnessTaskPlan,
 } from "./externalHarnessPlan.js";
+import {
+  logBrowserSession,
+  withBrowserSession,
+  type BrowserSessionInfo,
+} from "./browserSession.js";
 import { withHarnessAgentSpan } from "./otel.js";
+import { verifierTraceEnabled } from "./verifierTrace.js";
 import type { DiscoveredTask, TaskResult } from "./types.js";
 import type { BenchMatrixRow, BenchTaskKind, Harness } from "./benchTypes.js";
 import { DEFAULT_BENCH_HARNESS } from "./benchTypes.js";
@@ -69,7 +76,7 @@ export interface BenchHarness {
   supportsApi: boolean;
   /**
    * Tool surfaces this harness can mount for the agent, in display order; the
-   * first entry is the default when --tool is omitted. An empty list means the
+   * facade is preferred when --tool is omitted, otherwise the first entry. An empty list means the
    * harness does not mount tool surfaces and the planner passes the requested
    * surface/profile through unchanged as row metadata (stagehand harness).
    */
@@ -105,7 +112,14 @@ export interface ExternalHarnessRunInput<TAdapter> {
   verifier: ExternalHarnessVerifierConfig;
 }
 
-export interface ExternalHarnessDefinition<TAdapter extends { cleanup: () => Promise<void> }> {
+/** What every prepared external-harness adapter must expose to the shared lifecycle. */
+export interface ExternalHarnessAdapterBase {
+  cleanup: () => Promise<void>;
+  /** Browser behind the mounted surface; logged before the agent starts. */
+  browserSession?: BrowserSessionInfo;
+}
+
+export interface ExternalHarnessDefinition<TAdapter extends ExternalHarnessAdapterBase> {
   harness: string;
   supportedToolSurfaces: ToolSurface[];
   defaultModels: AvailableModel[];
@@ -119,7 +133,7 @@ export interface ExternalHarnessDefinition<TAdapter extends { cleanup: () => Pro
  * Define the lifecycle common to external agent harnesses without registering
  * it; registry ownership stays explicit so list order remains deterministic.
  */
-export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<void> }>(
+export function defineExternalHarness<TAdapter extends ExternalHarnessAdapterBase>(
   definition: ExternalHarnessDefinition<TAdapter>,
 ): BenchHarness {
   const {
@@ -148,6 +162,9 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
       // the adapter and the carrier.
       const carrierV3 = buildVerifierCarrierV3(logger);
       let toolAdapter: TAdapter | undefined;
+      let browserSession: BrowserSessionInfo = {
+        provider: row.config.environment === "BROWSERBASE" ? "browserbase" : "local",
+      };
       try {
         toolAdapter = await prepareToolAdapter({
           toolSurface: row.config.toolSurface,
@@ -157,7 +174,9 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
           logger,
         });
         const preparedAdapter = toolAdapter;
-        return await withHarnessAgentSpan(
+        browserSession = preparedAdapter.browserSession ?? browserSession;
+        logBrowserSession(logger, browserSession);
+        const result = await withHarnessAgentSpan(
           {
             harness,
             model: input.modelName,
@@ -178,6 +197,18 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
               },
             }),
         );
+        return withBrowserSession(result, browserSession);
+      } catch (error) {
+        return withBrowserSession(
+          {
+            _success: false,
+            error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
+            harnessStatus: "sdk_error",
+            terminationReason: "sdk_error",
+            logs: logger.getLogs(),
+          },
+          browserSession,
+        );
       } finally {
         try {
           await toolAdapter?.cleanup();
@@ -208,7 +239,9 @@ function buildVerifierCarrierV3(logger: EvalLogger): V3 {
     disablePino: true,
     disableAPI: true,
     experimental: true,
-    verbose: 0,
+    // verbose 2 surfaces the judge's LLM request/response lines (level 2),
+    // which verifierAdapter routes to scores/verifier-trace.jsonl.
+    verbose: verifierTraceEnabled() ? 2 : 0,
   });
 }
 

packages/evals/framework/benchPlanner.ts

@@ -1,5 +1,6 @@
 import type { AvailableModel } from "stagehand-v3";
 import { EvalsError } from "../errors.js";
+import { explicitSnapshotActionsEnabled } from "@browserbasehq/stagehand-integrations/facade";
 import { buildOnlineMind2WebTestcases } from "../suites/onlineMind2Web.js";
 import { buildHardBenchmarkTestcases } from "../suites/hardbenchmark.js";
 import { buildWebTailBenchTestcases } from "../suites/webtailbench.js";
@@ -366,11 +367,15 @@ function withBenchMetadata(
 }
 
 function buildToolMetadata(row: BenchMatrixRow): Partial<Testcase["metadata"]> {
+  const promptVariant =
+    row.toolSurface === "stagehand_facade" && explicitSnapshotActionsEnabled()
+      ? { promptVariant: "explicit_snapshot_actions" }
+      : {};
   if (
     getBenchHarness(row.harness).supportedToolSurfaces.includes("browse_cli") &&
     row.toolSurface === "browse_cli"
   ) {
-    return getBrowseCliToolMetadata();
+    return { ...getBrowseCliToolMetadata(), ...promptVariant };
   }
-  return {};
+  return promptVariant;
 }

packages/evals/framework/browserSession.ts

@@ -0,0 +1,89 @@
+import type { LogLine } from "stagehand-v3";
+import type { TaskResult } from "./types.js";
+
+export const BROWSER_SESSION_LOG_CATEGORY = "session";
+
+/** Where the browser behind a run lives, resolved before the agent starts. */
+export interface BrowserSessionInfo {
+  provider: "browserbase" | "local";
+  sessionId?: string;
+  sessionUrl?: string;
+  debugUrl?: string;
+}
+
+export function browserbaseSessionUrl(sessionId: string): string {
+  return `https://www.browserbase.com/sessions/${encodeURIComponent(sessionId)}`;
+}
+
+/**
+ * Read the session fields core tools and runner-provided targets publish on
+ * their `metadata` (`browserbaseSessionId` / `browserbaseSessionUrl` /
+ * `browserbaseDebugUrl`). Falls back to the bare provider when a Browserbase
+ * surface does not report its session id (browse_cli).
+ */
+export function browserSessionFromMetadata(
+  metadata: Record<string, unknown> | undefined,
+  environment: "LOCAL" | "BROWSERBASE",
+): BrowserSessionInfo {
+  if (environment !== "BROWSERBASE") return { provider: "local" };
+  const rawUrl = readString(metadata?.browserbaseSessionUrl);
+  const sessionId =
+    readString(metadata?.browserbaseSessionId) ?? rawUrl?.match(/\/sessions\/([^/?#]+)/u)?.[1];
+  const sessionUrl = rawUrl ?? (sessionId ? browserbaseSessionUrl(sessionId) : undefined);
+  const debugUrl = readString(metadata?.browserbaseDebugUrl);
+  return {
+    provider: "browserbase",
+    ...(sessionId && { sessionId }),
+    ...(sessionUrl && { sessionUrl }),
+    ...(debugUrl && { debugUrl }),
+  };
+}
+
+export function formatBrowserSessionMessage(info: BrowserSessionInfo): string {
+  if (info.provider === "local") return "Browser: local";
+  if (!info.sessionUrl) return "Browser: browserbase (session id not reported by this surface)";
+  return `Browserbase session: ${info.sessionUrl}`;
+}
+
+/** Level-0 lines so the session pointer survives every log filter. */
+export function buildBrowserSessionLogLines(info: BrowserSessionInfo): LogLine[] {
+  const lines: LogLine[] = [
+    {
+      category: BROWSER_SESSION_LOG_CATEGORY,
+      level: 0,
+      message: formatBrowserSessionMessage(info),
+      auxiliary: {
+        provider: { value: info.provider, type: "string" },
+        ...(info.sessionId && { sessionId: { value: info.sessionId, type: "string" } }),
+        ...(info.sessionUrl && { sessionUrl: { value: info.sessionUrl, type: "string" } }),
+      },
+    },
+  ];
+  if (info.debugUrl) {
+    lines.push({
+      category: BROWSER_SESSION_LOG_CATEGORY,
+      level: 0,
+      message: `Browserbase debugger: ${info.debugUrl}`,
+    });
+  }
+  return lines;
+}
+
+export function logBrowserSession(sink: { log(line: LogLine): void }, info: BrowserSessionInfo) {
+  for (const line of buildBrowserSessionLogLines(info)) sink.log(line);
+}
+
+/** Surface the session on the TaskResult row so Braintrust output is filterable. */
+export function withBrowserSession(result: TaskResult, info: BrowserSessionInfo): TaskResult {
+  return {
+    ...result,
+    browserProvider: info.provider,
+    ...(info.sessionId && { browserbaseSessionId: info.sessionId }),
+    ...(info.sessionUrl && { sessionUrl: result.sessionUrl || info.sessionUrl }),
+    ...(info.debugUrl && { debugUrl: result.debugUrl || info.debugUrl }),
+  };
+}
+
+function readString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value : undefined;
+}