diff --git a/packages/evals/framework/defineTask.ts b/packages/evals/framework/defineTask.ts index b03d3e037..d660a98c4 100644 --- a/packages/evals/framework/defineTask.ts +++ b/packages/evals/framework/defineTask.ts @@ -11,6 +11,7 @@ import type { TaskDefinition, TaskMeta, TaskResult, + BenchV4TaskContext, } from "./types.js"; /** @@ -43,6 +44,31 @@ export function defineBenchTask( }; } +/** + * Define a bench tier task ported to the Stagehand v4 SDK. + * v4 bench tasks receive { stagehand, page, logger, input, ... } and return + * TaskResult. They live under tasks/bench/ and are selected via --sdk v4. + */ +export function defineBenchV4Task( + meta: BenchTaskMeta, + fn: (ctx: BenchV4TaskContext) => Promise, +): TaskDefinition { + return { + __taskDefinition: true, + meta, + // Fail fast with a clear message if a v3-context runner invokes a v4 + // task — the v4 init/dispatch path lands with the harness change. + fn: (ctx) => { + if (!ctx || !("stagehand" in ctx) || ctx.stagehand === undefined) { + throw new Error( + `Task "${meta.name}" requires the v4 harness (--sdk v4); it was invoked with a v3 context.`, + ); + } + return fn(ctx as unknown as BenchV4TaskContext); + }, + }; +} + /** * Generic defineTask — for cases where the tier is ambiguous at definition time. * Prefer defineCoreTask / defineBenchTask for better type inference. diff --git a/packages/evals/framework/types.ts b/packages/evals/framework/types.ts index e574312b1..bbdeea360 100644 --- a/packages/evals/framework/types.ts +++ b/packages/evals/framework/types.ts @@ -7,7 +7,8 @@ * * A third tier ("interpret") is planned but not yet implemented. */ -import type { AgentToolMode, AgentInstance, AvailableModel, LogLine, V3 } from "stagehand-v3"; +import type { AvailableModel, LogLine } from "stagehand-v3"; +import type { Page, Stagehand } from "@browserbasehq/stagehand"; import type { CorePageHandle, CoreSession, @@ -35,6 +36,13 @@ export interface TaskMeta { export interface BenchTaskMeta extends TaskMeta { /** Override the default model list for this specific task. */ models?: string[]; + /** + * Custom system prompt passed to the Stagehand instance at init time + * (v3 `V3Options.systemPrompt` / v4 `StagehandInitParams.systemPrompt`). + * Used by tasks that test instruction-following (e.g. combination/instructions). + * Additional instructions applied when initializing Stagehand for this task. + */ + systemPrompt?: string; } /** Context provided to core (tier 1) tasks. */ @@ -128,7 +136,7 @@ export interface TaskDefinition { /** User-provided metadata. */ meta: TaskMeta | BenchTaskMeta; /** The task function. */ - fn: (ctx: CoreTaskContext | BenchTaskContext) => Promise; + fn: (ctx: CoreTaskContext | BenchTaskContext | BenchV4TaskContext) => Promise; /** Which tier this task was defined for (set during discovery from directory). */ tier?: Tier; } @@ -162,3 +170,33 @@ export interface TaskRegistry { /** Lookup by category. */ byCategory: Map; } + +/** + * Context type for bench tasks ported to the V4Stagehand v4 SDK. + * + * Deliberately mirrors BenchTaskContext (types.ts) so per-task diffs between + * the v3 and v4 suites stay 1:1, with the v3 surface swapped for the v4 one: + * `v3` → `stagehand`, Playwright `page` → v4 `V4Page`. Keeping `page` typed as + * the v4 V4Page makes any Playwright API usage in a ported task a type error. + */ + +export interface BenchV4TaskContext { + /** V4Stagehand v4 client instance. */ + stagehand: V4Stagehand; + /** v4 page object (RPC-backed — url()/title() are async). */ + page: V4Page; + /** Eval logger. Note: the v4 SDK itself logs to the console, not here. */ + logger: EvalLogger; + /** Full eval input (name, modelName, params). */ + input: { + name: string; + modelName: AvailableModel; + params?: Record; + }; + /** Model used for this run. */ + modelName: AvailableModel; + /** Debug URL (unavailable from the v4 SDK — always empty for now). */ + debugUrl: string; + /** Session URL (Browserbase; constructed from the session ID). */ + sessionUrl: string; +} diff --git a/packages/evals/tasks/bench/act/amazon_add_to_cart.ts b/packages/evals/tasks/bench/act/amazon_add_to_cart.ts index dcaa1cc21..21d4f7218 100644 --- a/packages/evals/tasks/bench/act/amazon_add_to_cart.ts +++ b/packages/evals/tasks/bench/act/amazon_add_to_cart.ts @@ -1,17 +1,16 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "amazon_add_to_cart" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/amazon/"); - await v3.act("click the 'Add to Cart' button"); + await stagehand.act("click the 'Add to Cart' button"); - await v3.act("click the 'Proceed to checkout' button"); + await stagehand.act("click the 'Proceed to checkout' button"); - const currentUrl = page.url(); + const currentUrl = await page.url(); const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/sign-in.html"; @@ -27,13 +26,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/bidnet.ts b/packages/evals/tasks/bench/act/bidnet.ts index 59a8e5352..021363a9c 100644 --- a/packages/evals/tasks/bench/act/bidnet.ts +++ b/packages/evals/tasks/bench/act/bidnet.ts @@ -1,32 +1,34 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask({ name: "bidnet" }, async ({ logger, debugUrl, sessionUrl, v3 }) => { - try { - const page = v3.context.pages()[0]; - await page.goto("https://www.bidnetdirect.com/"); +export default defineBenchV4Task( + { name: "bidnet" }, + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { + try { + await page.goto("https://www.bidnetdirect.com/"); - await v3.act('Click on the "Construction" keyword'); + await stagehand.act('Click on the "Construction" keyword'); - const expectedUrl = - "https://www.bidnetdirect.com/public/solicitations/open?keywords=Construction"; - const currentUrl = page.url(); + const expectedUrl = + "https://www.bidnetdirect.com/public/solicitations/open?keywords=Construction"; + const currentUrl = await page.url(); - return { - _success: currentUrl.startsWith(expectedUrl), - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - return { - _success: false, - error: error, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } finally { - await v3.close(); - } -}); + return { + _success: currentUrl.startsWith(expectedUrl), + currentUrl, + debugUrl, + sessionUrl, + logs: logger.getLogs(), + }; + } catch (error) { + return { + _success: false, + error: error instanceof Error ? error.message : String(error), + debugUrl, + sessionUrl, + logs: logger.getLogs(), + }; + } finally { + await stagehand.close(); + } + }, +); diff --git a/packages/evals/tasks/bench/act/checkboxes.ts b/packages/evals/tasks/bench/act/checkboxes.ts index 03c6ca945..42c1437ca 100644 --- a/packages/evals/tasks/bench/act/checkboxes.ts +++ b/packages/evals/tasks/bench/act/checkboxes.ts @@ -1,15 +1,14 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "checkboxes" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/checkboxes/"); - await v3.act("click the 'baseball' option"); + await stagehand.act("click the 'baseball' option"); - await v3.act("click the 'netball' option"); + await stagehand.act("click the 'netball' option"); const baseballChecked = await page .locator('input[type="checkbox"][name="sports"][value="baseball"]') @@ -34,7 +33,7 @@ export default defineBenchTask( logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/csr_in_oopif.ts b/packages/evals/tasks/bench/act/csr_in_oopif.ts index a238d5162..5fba02008 100644 --- a/packages/evals/tasks/bench/act/csr_in_oopif.ts +++ b/packages/evals/tasks/bench/act/csr_in_oopif.ts @@ -1,21 +1,27 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "csr_in_oopif" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // click inside an CSR (closed mode shadow) root that is inside an // OOPIF (out of process iframe) try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-root-in-oopif/", ); - await v3.act("click the button"); + await stagehand.act("click the button"); await new Promise((resolve) => setTimeout(resolve, 1000)); - const extraction = await v3.extract("extract the entire page text"); + + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -38,13 +44,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/csr_in_spif.ts b/packages/evals/tasks/bench/act/csr_in_spif.ts index 5baaa12ff..9d2acadab 100644 --- a/packages/evals/tasks/bench/act/csr_in_spif.ts +++ b/packages/evals/tasks/bench/act/csr_in_spif.ts @@ -1,20 +1,25 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "csr_in_spif" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // click inside an CSR (closed mode shadow) root that is inside an // SPIF (same process iframe) try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-dom-in-spif/", ); - await v3.act("click the button"); + await stagehand.act("click the button"); - const extraction = await v3.extract("extract the entire page text"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -37,13 +42,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/custom_dropdown.ts b/packages/evals/tasks/bench/act/custom_dropdown.ts index de3e7522a..14fb38d50 100644 --- a/packages/evals/tasks/bench/act/custom_dropdown.ts +++ b/packages/evals/tasks/bench/act/custom_dropdown.ts @@ -1,8 +1,8 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "custom_dropdown" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { /** * This eval is meant to test whether we do not incorrectly attempt * the selectOptionFromDropdown method (defined in actHandlerUtils.ts) on a @@ -13,14 +13,12 @@ export default defineBenchTask( */ try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/expand-dropdown/"); - await v3.act("choose Canada from the 'Select a Country' dropdown"); + await stagehand.act("choose Canada from the 'Select a Country' dropdown"); - // to test, we'll grab the full a11y tree, and make sure it contains 'Canada' - const extraction = await v3.extract(); - const fullTree = extraction.pageText; + // read the rendered page text directly — no second model call + const fullTree = await page.locator("#chosenValue").innerText(); if (fullTree.includes("Canada")) { return { @@ -40,13 +38,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error attempting to select an option from the dropdown: ${error.message}`, + message: `error attempting to select an option from the dropdown: ${(error as Error).message}`, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/dropdown.ts b/packages/evals/tasks/bench/act/dropdown.ts index 7998a9f75..c74d6b4df 100644 --- a/packages/evals/tasks/bench/act/dropdown.ts +++ b/packages/evals/tasks/bench/act/dropdown.ts @@ -1,10 +1,9 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "dropdown" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/dropdown/"); // click the dropdown element to expand it @@ -13,7 +12,7 @@ export default defineBenchTask( // type into the input box (which should be hidden behind the // expanded dropdown) - await v3.act("type 'test fill' into the input field"); + await stagehand.act("type 'test fill' into the input field"); const input = page.locator(`xpath=/html/body/div/input`); const expectedValue = "test fill"; @@ -33,13 +32,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/google_flights.ts b/packages/evals/tasks/bench/act/google_flights.ts index 82e1aab06..45cc26f06 100644 --- a/packages/evals/tasks/bench/act/google_flights.ts +++ b/packages/evals/tasks/bench/act/google_flights.ts @@ -1,5 +1,5 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; -import { Action } from "stagehand-v3"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; +import type { Action } from "@browserbasehq/stagehand"; /** * This eval attempts to click on an element that should not pass the playwright actionability check @@ -11,11 +11,10 @@ import { Action } from "stagehand-v3"; * pass the actionability check. */ -export default defineBenchTask( +export default defineBenchV4Task( { name: "google_flights" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/google-flights/"); const observeResult: Action = { @@ -25,13 +24,12 @@ export default defineBenchTask( method: "click", arguments: [], }; - await v3.act(observeResult); - + await replayObservedAction(page, observeResult); const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/google-flights/return-flight.html"; - const currentUrl = page.url(); + const currentUrl = await page.url(); - await v3.close(); + await stagehand.close(); if (currentUrl === expectedUrl) { return { @@ -52,13 +50,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/heal_custom_dropdown.ts b/packages/evals/tasks/bench/act/heal_custom_dropdown.ts index c339e1a43..4c4a6f4ec 100644 --- a/packages/evals/tasks/bench/act/heal_custom_dropdown.ts +++ b/packages/evals/tasks/bench/act/heal_custom_dropdown.ts @@ -1,8 +1,8 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "heal_custom_dropdown" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { /** * This eval is meant to test whether we do not incorrectly attempt * the selectOptionFromDropdown method (defined in actHandlerUtils.ts) on a @@ -13,25 +13,25 @@ export default defineBenchTask( */ try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/expand-dropdown/"); - await v3.act({ + // Self-healing act(Action) replay (restored by + // stagehand#2427): same intentionally invalid selector as the v3 + // twin — healing must re-locate "The 'Select a country' dropdown" + // and click it to expand. + await stagehand.act({ description: "The 'Select a country' dropdown", selector: "/html/not-a-dropdown", arguments: [], method: "click", }); - // we are expecting stagehand to click the dropdown to expand it, - // and therefore the available options should now be contained in the full - // a11y tree. + // If the dropdown expanded, its options are now rendered in the DOM. + // (v3 checked the schemaless-extract page text; v4 extract requires a + // schema, so read the rendered text directly — same signal, no LLM.) + const pageText = await page.evaluate(() => document.body.innerText); - // to test, we'll grab the full a11y tree, and make sure it contains 'Canada' - const extraction = await v3.extract(); - const fullTree = extraction.pageText; - - if (fullTree.includes("Canada")) { + if (pageText.includes("Canada")) { return { _success: true, debugUrl, @@ -49,13 +49,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error attempting to select an option from the dropdown: ${error.message}`, + message: `error attempting to select an option from the dropdown: ${(error as Error).message}`, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/heal_scroll_50.ts b/packages/evals/tasks/bench/act/heal_scroll_50.ts index b5dadad48..47647a57e 100644 --- a/packages/evals/tasks/bench/act/heal_scroll_50.ts +++ b/packages/evals/tasks/bench/act/heal_scroll_50.ts @@ -1,12 +1,16 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "heal_scroll_50" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/"); - await v3.act({ + + // Self-healing act(Action) replay (restored by + // stagehand#2427): same supplied action as the v3 twin — a + // "scrollTo" with arguments ["50%"], exercising the deterministic + // executor with variable substitution and healing. + await stagehand.act({ description: "the element to scroll on", selector: "/html/body/div/div/button", arguments: ["50%"], @@ -15,7 +19,6 @@ export default defineBenchTask( await new Promise((resolve) => setTimeout(resolve, 5000)); - // Get the current scroll position and total scroll height const scrollInfo = await page.evaluate(() => { return { scrollTop: window.scrollY + window.innerHeight / 2, @@ -25,32 +28,28 @@ export default defineBenchTask( const halfwayScroll = scrollInfo.scrollHeight / 2; const halfwayReached = Math.abs(scrollInfo.scrollTop - halfwayScroll) <= 200; - const evaluationResult = halfwayReached - ? { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - } - : { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - message: `Scroll position (${scrollInfo.scrollTop}px) is not halfway down the page (${halfwayScroll}px).`, - }; - return evaluationResult; + return { + _success: halfwayReached, + ...(halfwayReached + ? {} + : { + message: `Scroll position (${scrollInfo.scrollTop}px) is not halfway down the page (${halfwayScroll}px).`, + }), + logs: logger.getLogs(), + debugUrl, + sessionUrl, + }; } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/heal_simple_google_search.ts b/packages/evals/tasks/bench/act/heal_simple_google_search.ts index 3e13fddc6..028be7617 100644 --- a/packages/evals/tasks/bench/act/heal_simple_google_search.ts +++ b/packages/evals/tasks/bench/act/heal_simple_google_search.ts @@ -1,25 +1,27 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "heal_simple_google_search" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/google/"); - await v3.act({ + // Self-healing act(Action) replay (restored by + // stagehand#2427): same intentionally invalid selector as the v3 + // twin — healing must re-locate "The search bar" and fill it. + await stagehand.act({ description: "The search bar", selector: "/html/not-the-search-bar", arguments: ["OpenAI"], method: "fill", }); - await v3.act("press enter"); + await stagehand.act("press enter"); await new Promise((resolve) => setTimeout(resolve, 3000)); const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/google/openai.html"; - const currentUrl = page.url(); + const currentUrl = await page.url(); return { _success: currentUrl.startsWith(expectedUrl), @@ -31,13 +33,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/hidden_input_dropdown.ts b/packages/evals/tasks/bench/act/hidden_input_dropdown.ts index a91897a59..ccd79a192 100644 --- a/packages/evals/tasks/bench/act/hidden_input_dropdown.ts +++ b/packages/evals/tasks/bench/act/hidden_input_dropdown.ts @@ -1,8 +1,9 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "hidden_input_dropdown" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { /** * This eval is meant to test whether we do not incorrectly attempt * the selectOptionFromDropdown method (defined in actHandlerUtils.ts) on a @@ -13,20 +14,24 @@ export default defineBenchTask( */ try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/hidden-input-dropdown/", ); - await v3.act("click to expand the 'Favourite Colour' dropdown"); + await stagehand.act("click to expand the 'Favourite Colour' dropdown"); // we are expecting stagehand to click the dropdown to expand it, // and therefore the available options should now be contained in the full // a11y tree. // to test, we'll grab the full a11y tree, and make sure it contains 'Green' - const extraction = await v3.extract(); - const fullTree = extraction.pageText; + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); + const fullTree = extraction.extraction; if (fullTree.includes("Green")) { return { @@ -46,13 +51,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error attempting click to expand the dropdown: ${error.message}`, + message: `error attempting click to expand the dropdown: ${error instanceof Error ? error.message : String(error)}`, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/iframe_form_filling.ts b/packages/evals/tasks/bench/act/iframe_form_filling.ts index b28d172c9..91ef5aaea 100644 --- a/packages/evals/tasks/bench/act/iframe_form_filling.ts +++ b/packages/evals/tasks/bench/act/iframe_form_filling.ts @@ -1,36 +1,36 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "iframe_form_filling" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-form-filling/", ); - await v3.act("type 'nunya' into the 'first name' field"); - await v3.act("type 'business' into the 'last name' field"); - await v3.act("type 'test@email.com' into the 'email' field"); - await v3.act("click 'phone' as the preferred contact method"); - await v3.act("type 'yooooooooooooooo' into the message box"); + await stagehand.act("type 'nunya' into the 'first name' field"); + await stagehand.act("type 'business' into the 'last name' field"); + await stagehand.act("type 'test@email.com' into the 'email' field"); + await stagehand.act("click 'phone' as the preferred contact method"); + await stagehand.act("type 'yooooooooooooooo' into the message box"); - const iframe = page.frameLocator("iframe"); + // The form lives in a cross-origin iframe, so main-frame evaluation + // cannot access its contentDocument. V4 locators resolve `>>` iframe + // hops server-side and can inspect the OOPIF directly. + const firstNameValue = await page.locator('iframe >> input[placeholder="Jane"]').inputValue(); - const firstNameValue: string = await iframe.locator('input[placeholder="Jane"]').inputValue(); + const lastNameValue = await page.locator('iframe >> input[placeholder="Doe"]').inputValue(); - const lastNameValue: string = await iframe.locator('input[placeholder="Doe"]').inputValue(); - - const emailValue: string = await iframe - .locator('input[placeholder="jane@example.com"]') + const emailValue = await page + .locator('iframe >> input[placeholder="jane@example.com"]') .inputValue(); - const contactValue: boolean = await iframe - .locator("xpath=/html/body/main/section[1]/form/fieldset/label[2]/input") + const contactValue = await page + .locator("iframe >> xpath=/html/body/main/section[1]/form/fieldset/label[2]/input") .isChecked(); - const messageValue: string = await iframe - .locator('textarea[placeholder="Say hello…"]') + const messageValue = await page + .locator('iframe >> textarea[placeholder="Say hello…"]') .inputValue(); const passed: boolean = @@ -49,13 +49,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/iframe_same_proc.ts b/packages/evals/tasks/bench/act/iframe_same_proc.ts index 988c85c26..59f79e935 100644 --- a/packages/evals/tasks/bench/act/iframe_same_proc.ts +++ b/packages/evals/tasks/bench/act/iframe_same_proc.ts @@ -1,25 +1,38 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "iframe_same_proc" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/iframe-same-proc/"); - await v3.act("type 'stagehand' into the 'your name' field"); + await stagehand.act("type 'stagehand' into the 'your name' field"); // overly specific prompting is okay here. we are just trying to evaluate whether // we are properly traversing iframes - await v3.act( + await stagehand.act( "select 'Green' from the favorite colour dropdown. Ensure the word 'Green' is capitalized. Choose the selectOption method.", ); - const iframe = page.frameLocator("iframe"); + // v3 used page.frameLocator("iframe") for these assertions; v4 has no + // frameLocator, so the same checks are re-expressed in-page via the + // same-origin iframe's contentDocument. + const { nameValue, colorValue } = await page.evaluate(() => { + const doc = document.querySelector("iframe")?.contentDocument; + if (!doc) throw new Error("could not access iframe contentDocument"); - const nameValue: string = await iframe.locator('input[placeholder="Alice"]').inputValue(); + const name = doc.querySelector('input[placeholder="Alice"]') as HTMLInputElement | null; + const color = doc.querySelector("select") as HTMLSelectElement | null; - const colorValue: string = await iframe.locator("select").inputValue(); + if (!name || !color) { + throw new Error("could not resolve form fields inside the iframe"); + } + + return { + nameValue: name.value, + colorValue: color.value, + }; + }); const passed: boolean = nameValue.toLowerCase().trim() === "stagehand" && @@ -34,13 +47,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/iframe_scroll.ts b/packages/evals/tasks/bench/act/iframe_scroll.ts index ba0295268..c05120ef0 100644 --- a/packages/evals/tasks/bench/act/iframe_scroll.ts +++ b/packages/evals/tasks/bench/act/iframe_scroll.ts @@ -1,25 +1,29 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "iframe_scroll" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-same-proc-scroll/", ); - await v3.act("scroll down 50% inside the iframe"); - - const frames = page.frames(); - const frame = frames[1]; + await stagehand.act("scroll down 50% inside the iframe"); await new Promise((resolve) => setTimeout(resolve, 5000)); - // Get the current scroll position and total scroll height - const scrollInfo = await frame.evaluate(() => { + // v3 evaluated inside page.frames()[1]; v4 exposes no frames() list, + // so the same measurement is re-expressed via the same-origin iframe's + // contentWindow/contentDocument. + const scrollInfo = await page.evaluate(() => { + const iframe = document.querySelector("iframe"); + const win = iframe?.contentWindow; + const doc = iframe?.contentDocument; + if (!win || !doc) { + throw new Error("could not access iframe content"); + } return { - scrollTop: window.scrollY + window.innerHeight / 2, - scrollHeight: document.documentElement.scrollHeight, + scrollTop: win.scrollY + win.innerHeight / 2, + scrollHeight: doc.documentElement.scrollHeight, }; }); @@ -44,13 +48,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/iframes_nested.ts b/packages/evals/tasks/bench/act/iframes_nested.ts index e66fc7ca4..3cf4719b5 100644 --- a/packages/evals/tasks/bench/act/iframes_nested.ts +++ b/packages/evals/tasks/bench/act/iframes_nested.ts @@ -1,20 +1,30 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "iframes_nested" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/nested-iframes/"); - await v3.act("type 'stagehand' into the 'username' field"); + await stagehand.act("type 'stagehand' into the 'username' field"); - const inner = page - .frameLocator("iframe.lvl1") // level 1 - .frameLocator("iframe.lvl2") // level 2 - .frameLocator("iframe.lvl3"); // level 3 – form lives here + // v3 chained frameLocator lvl1 -> lvl2 -> lvl3 (form lives in level 3); + // v4 has no frameLocator, so the same check is re-expressed in-page by + // walking the same-origin iframes' contentDocuments. + const usernameText = await page.evaluate(() => { + const lvl1 = (document.querySelector("iframe.lvl1") as HTMLIFrameElement | null) + ?.contentDocument; // level 1 + const lvl2 = (lvl1?.querySelector("iframe.lvl2") as HTMLIFrameElement | null) + ?.contentDocument; // level 2 + const lvl3 = (lvl2?.querySelector("iframe.lvl3") as HTMLIFrameElement | null) + ?.contentDocument; // level 3 – form lives here - const usernameText = await inner.locator('input[name="username"]').inputValue(); + const input = lvl3?.querySelector('input[name="username"]') as HTMLInputElement | null; + if (!input) { + throw new Error("could not resolve the username input in the nested iframes"); + } + return input.value; + }); const passed: boolean = usernameText.toLowerCase().trim() === "stagehand"; @@ -27,13 +37,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/ionwave.ts b/packages/evals/tasks/bench/act/ionwave.ts index 6d9651765..1e51de03a 100644 --- a/packages/evals/tasks/bench/act/ionwave.ts +++ b/packages/evals/tasks/bench/act/ionwave.ts @@ -1,17 +1,16 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "ionwave" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/ionwave/"); - await v3.act('Click on "Closed Bids"'); + await stagehand.act('Click on "Closed Bids"'); const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/ionwave/closed-bids.html"; - const currentUrl = page.url(); + const currentUrl = await page.url(); return { _success: currentUrl.startsWith(expectedUrl), @@ -23,13 +22,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/login.ts b/packages/evals/tasks/bench/act/login.ts index 9ee817e59..355585545 100644 --- a/packages/evals/tasks/bench/act/login.ts +++ b/packages/evals/tasks/bench/act/login.ts @@ -1,36 +1,38 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask({ name: "login" }, async ({ debugUrl, sessionUrl, v3, logger }) => { - try { - const page = v3.context.pages()[0]; - await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/login/"); +export default defineBenchV4Task( + { name: "login" }, + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { + try { + await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/login/"); - await v3.act("type %nunya% into the username field", { - variables: { nunya: "business" }, - }); + await stagehand.act("type %nunya% into the username field", { + variables: { nunya: "business" }, + }); - const xpath = "xpath=/html/body/main/form/div[1]/input"; - const actualValue = await page.locator(xpath).inputValue(); + const xpath = "xpath=/html/body/main/form/div[1]/input"; + const actualValue = await page.locator(xpath).inputValue(); - const expectedValue = "business"; + const expectedValue = "business"; - return { - _success: actualValue === expectedValue, - expectedValue, - actualValue, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - return { - _success: false, - error: error, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } finally { - await v3.close(); - } -}); + return { + _success: actualValue === expectedValue, + expectedValue, + actualValue, + debugUrl, + sessionUrl, + logs: logger.getLogs(), + }; + } catch (error) { + return { + _success: false, + error: error instanceof Error ? error.message : String(error), + debugUrl, + sessionUrl, + logs: logger.getLogs(), + }; + } finally { + await stagehand.close(); + } + }, +); diff --git a/packages/evals/tasks/bench/act/multi_tab.ts b/packages/evals/tasks/bench/act/multi_tab.ts index c5931780b..a281851bc 100644 --- a/packages/evals/tasks/bench/act/multi_tab.ts +++ b/packages/evals/tasks/bench/act/multi_tab.ts @@ -1,17 +1,21 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "multi_tab" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/"); - await v3.act("click the button to open the other page"); - await v3.act("click the button to open the other page"); - await v3.act("click the button to open the other page"); - await v3.act("click the button to open the other page"); - let activePage = await v3.context.awaitActivePage(); + // v3-parity form: activePage() polls until Chrome's active target + // registers (stagehand#2458), and act resolves its target through it, + // so back-to-back tab-opening acts chain without explicit waits. + await stagehand.act("click the button to open the other page"); + await stagehand.act("click the button to open the other page"); + await stagehand.act("click the button to open the other page"); + await stagehand.act("click the button to open the other page"); + let activePage = await stagehand.context.activePage(); + if (!activePage) throw new Error("no active page after opening tabs"); let currentPageUrl = await activePage.url(); let expectedUrl = @@ -28,11 +32,12 @@ export default defineBenchTask( } // try acting on the first page again - const pages = v3.context.pages(); + const pages = await stagehand.context.pages(); const page1 = pages[0]; - await v3.act("click the button to open the other page", { page: page1 }); + await stagehand.act("click the button to open the other page", { page: page1 }); - activePage = await v3.context.awaitActivePage(); + activePage = await stagehand.context.activePage(); + if (!activePage) throw new Error("no active page after acting on page 1"); currentPageUrl = await activePage.url(); expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/page2.html"; if (currentPageUrl !== expectedUrl) { @@ -45,10 +50,18 @@ export default defineBenchTask( }; } - const page2text = await v3.extract({ page: activePage }); + // V4 GAP: extract has no { page } option (v3: + // v3.extract({ page: activePage })) — the target page is already the + // active page here, so extract operates on it. v3 also used schemaless + // extract; v4 requires a schema. Single-word key to + // stay clear of the snake_case wire-casing bug (#14). + const { data: page2text } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const expectedPage2text = "You've made it to page 2"; - if (page2text.pageText.includes(expectedPage2text)) { + if (page2text.extraction.includes(expectedPage2text)) { return { _success: true, debugUrl, @@ -58,7 +71,7 @@ export default defineBenchTask( } return { _success: false, - message: `extracted page text: ${page2text.pageText} does not match expected page text: ${expectedPage2text}`, + message: `extracted page text: ${page2text.extraction} does not match expected page text: ${expectedPage2text}`, debugUrl, sessionUrl, logs: logger.getLogs(), @@ -66,13 +79,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: error.message, + message: (error as Error).message, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/namespace_xpath.ts b/packages/evals/tasks/bench/act/namespace_xpath.ts index 11e7168b8..ec7b94ece 100644 --- a/packages/evals/tasks/bench/act/namespace_xpath.ts +++ b/packages/evals/tasks/bench/act/namespace_xpath.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "namespace_xpath" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/namespaced-xpath/"); - await v3.act("fill 'nunya' into the 'type here' form"); + await stagehand.act("fill 'nunya' into the 'type here' form"); const inputValue = await page.locator("#ns-text").inputValue(); // confirm that the form was filled @@ -22,13 +21,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/nested_iframes_2.ts b/packages/evals/tasks/bench/act/nested_iframes_2.ts index 4466a6bea..6030730cc 100644 --- a/packages/evals/tasks/bench/act/nested_iframes_2.ts +++ b/packages/evals/tasks/bench/act/nested_iframes_2.ts @@ -1,19 +1,30 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "nested_iframes_2" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/nested-iframes-2/"); - await v3.act("click the button called 'click me (inner 2)'"); + await stagehand.act("click the button called 'click me (inner 2)'"); - const inner = page - .frameLocator('iframe[src="iframe2.html"]') - .frameLocator('iframe[src="inner2.html"]'); + // v3 chained frameLocator iframe2.html -> inner2.html; v4 has no + // frameLocator, so the same check is re-expressed in-page by walking + // the same-origin iframes' contentDocuments. + const messageText = await page.evaluate(() => { + const outer = ( + document.querySelector('iframe[src="iframe2.html"]') as HTMLIFrameElement | null + )?.contentDocument; + const inner = ( + outer?.querySelector('iframe[src="inner2.html"]') as HTMLIFrameElement | null + )?.contentDocument; - const messageText = await inner.locator("#msg").textContent(); + const msg = inner?.querySelector("#msg"); + if (!msg) { + throw new Error("could not resolve #msg in the nested iframes"); + } + return msg.textContent ?? ""; + }); const passed: boolean = messageText.toLowerCase().trim() === "clicked the button in the second inner iframe"; @@ -33,7 +44,7 @@ export default defineBenchTask( error, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/next_chunk.ts b/packages/evals/tasks/bench/act/next_chunk.ts index 4f3e9c681..ec24d8d60 100644 --- a/packages/evals/tasks/bench/act/next_chunk.ts +++ b/packages/evals/tasks/bench/act/next_chunk.ts @@ -1,14 +1,13 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "next_chunk" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://www.apartments.com/san-francisco-ca/", { waitUntil: "domcontentloaded", }); - await v3.act("click on the all filters button"); + await stagehand.act("click on the all filters button"); const { initialScrollTop, chunkHeight } = await page.evaluate(() => { const container = document.querySelector("#advancedFilters > div") as HTMLElement; @@ -22,7 +21,7 @@ export default defineBenchTask( }; }); - await v3.act("scroll down one chunk on the filters modal"); + await stagehand.act("scroll down one chunk on the filters modal"); await new Promise((resolve) => setTimeout(resolve, 2000)); @@ -55,13 +54,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/no_js_click.ts b/packages/evals/tasks/bench/act/no_js_click.ts index 2d4447c71..0ee23dd35 100644 --- a/packages/evals/tasks/bench/act/no_js_click.ts +++ b/packages/evals/tasks/bench/act/no_js_click.ts @@ -1,9 +1,9 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; -import { Action } from "stagehand-v3"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; +import type { Action } from "@browserbasehq/stagehand"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "no_js_click" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { /** * This eval is meant to test whether our `clickElement` function * (inside actHandlerUtils.ts) is able to click elements even if @@ -11,7 +11,6 @@ export default defineBenchTask( */ try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/no-js-click/"); const observeResult: Action = { @@ -20,7 +19,7 @@ export default defineBenchTask( description: "the button to click", arguments: [], }; - await v3.act(observeResult); + await stagehand.act(observeResult); const text = await page.locator("#success-msg").textContent(); if (text?.trim() === "click succeeded") { @@ -41,13 +40,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error attempting to click the button: ${error.message}`, + message: `error attempting to click the button: ${(error as Error).message}`, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/nonsense_action.ts b/packages/evals/tasks/bench/act/nonsense_action.ts index 449f51b10..6bd3796ca 100644 --- a/packages/evals/tasks/bench/act/nonsense_action.ts +++ b/packages/evals/tasks/bench/act/nonsense_action.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "nonsense_action" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://www.homedepot.com/"); - const result = await v3.act("what is the capital of the moon?"); + const { data: result } = await stagehand.act("what is the capital of the moon?"); return { _success: !result.success, // We expect this to fail @@ -18,13 +17,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/oopif_in_csr.ts b/packages/evals/tasks/bench/act/oopif_in_csr.ts index 9108a6b6c..374810dc1 100644 --- a/packages/evals/tasks/bench/act/oopif_in_csr.ts +++ b/packages/evals/tasks/bench/act/oopif_in_csr.ts @@ -1,20 +1,25 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "oopif_in_csr" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // fill a form inside a OOPIF (out of process iframe) that is inside an // CSR (closed mode shadow) root try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/oopif-in-open-shadow-dom/", ); - await v3.act("fill 'nunya' into the first name field"); + await stagehand.act("fill 'nunya' into the first name field"); - const extraction = await v3.extract("extract the entire page text"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -37,13 +42,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/oopif_in_osr.ts b/packages/evals/tasks/bench/act/oopif_in_osr.ts index dcd720d0c..14816fe7d 100644 --- a/packages/evals/tasks/bench/act/oopif_in_osr.ts +++ b/packages/evals/tasks/bench/act/oopif_in_osr.ts @@ -1,20 +1,25 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "oopif_in_osr" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // fill a form inside a OOPIF (out of process iframe) that is inside an // OSR (open mode shadow) root try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/oopif-in-open-shadow-dom/", ); - await v3.act("fill 'nunya' into the first name field"); + await stagehand.act("fill 'nunya' into the first name field"); - const extraction = await v3.extract("extract the entire page text"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -37,13 +42,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/os_dropdown.ts b/packages/evals/tasks/bench/act/os_dropdown.ts index bf9f6ab35..7beabe2bb 100644 --- a/packages/evals/tasks/bench/act/os_dropdown.ts +++ b/packages/evals/tasks/bench/act/os_dropdown.ts @@ -1,19 +1,25 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "os_dropdown" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { /** * This eval is meant to test whether we can correctly select an element * from an OS level dropdown */ try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/nested-dropdown/"); - await v3.act("choose 'Smog Check Technician' from the 'License Type' dropdown"); - const selectedOption = await page.locator("#licenseType >> option:checked").textContent(); + await stagehand.act("choose 'Smog Check Technician' from the 'License Type' dropdown"); + // v3 used page.locator("#licenseType >> option:checked"); v4 locator has + // no ">>" chaining, so the same check is re-expressed in-page. + const selectedOption = await page.evaluate(() => { + const option = document.querySelector( + "#licenseType option:checked", + ) as HTMLOptionElement | null; + return option?.textContent ?? null; + }); if (selectedOption === "Smog Check Technician") { return { @@ -33,13 +39,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error attempting to select an option from the dropdown: ${error.message}`, + message: `error attempting to select an option from the dropdown: ${(error as Error).message}`, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/osr_in_oopif.ts b/packages/evals/tasks/bench/act/osr_in_oopif.ts index c90a9d991..e200bb313 100644 --- a/packages/evals/tasks/bench/act/osr_in_oopif.ts +++ b/packages/evals/tasks/bench/act/osr_in_oopif.ts @@ -1,20 +1,25 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "osr_in_oopif" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // click inside an OSR (open mode shadow) root that is inside an // OOPIF (out of process iframe) try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/open-shadow-root-in-oopif/", ); - await v3.act("click the button"); + await stagehand.act("click the button"); - const extraction = await v3.extract("extract the entire page text"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -37,13 +42,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/osr_in_spif.ts b/packages/evals/tasks/bench/act/osr_in_spif.ts index 9e6582db8..bb390821f 100644 --- a/packages/evals/tasks/bench/act/osr_in_spif.ts +++ b/packages/evals/tasks/bench/act/osr_in_spif.ts @@ -1,20 +1,25 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "osr_in_spif" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // click inside an OSR (open mode shadow) root that is inside an // SPIF (same process iframe) try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/open-shadow-root-in-spif/", ); - await v3.act("click the button"); + await stagehand.act("click the button"); - const extraction = await v3.extract("extract the entire page text"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -37,13 +42,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/prev_chunk.ts b/packages/evals/tasks/bench/act/prev_chunk.ts index e912b8b97..aeae8be42 100644 --- a/packages/evals/tasks/bench/act/prev_chunk.ts +++ b/packages/evals/tasks/bench/act/prev_chunk.ts @@ -1,10 +1,9 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "prev_chunk" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/"); await new Promise((resolve) => setTimeout(resolve, 2000)); const { initialScrollTop, chunkHeight } = await page.evaluate(() => { @@ -24,7 +23,7 @@ export default defineBenchTask( }; }); await new Promise((resolve) => setTimeout(resolve, 2000)); - await v3.act("scroll up one chunk"); + await stagehand.act("scroll up one chunk"); await new Promise((resolve) => setTimeout(resolve, 5000)); @@ -54,13 +53,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/radio_btn.ts b/packages/evals/tasks/bench/act/radio_btn.ts index 5191bf866..622870ba9 100644 --- a/packages/evals/tasks/bench/act/radio_btn.ts +++ b/packages/evals/tasks/bench/act/radio_btn.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "radio_btn" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/paneer-pizza/"); - await v3.act("click the 'medium' option"); + await stagehand.act("click the 'medium' option"); // confirm that the Medium radio is now checked const radioBtnClicked = await page @@ -23,13 +22,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/scroll_50.ts b/packages/evals/tasks/bench/act/scroll_50.ts index 47e5573f9..82106cbe2 100644 --- a/packages/evals/tasks/bench/act/scroll_50.ts +++ b/packages/evals/tasks/bench/act/scroll_50.ts @@ -1,12 +1,11 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "scroll_50" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/"); - await v3.act("Scroll 50% down the page"); + await stagehand.act("Scroll 50% down the page"); await new Promise((resolve) => setTimeout(resolve, 5000)); @@ -39,13 +38,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/scroll_75.ts b/packages/evals/tasks/bench/act/scroll_75.ts index e397d930b..e689d10de 100644 --- a/packages/evals/tasks/bench/act/scroll_75.ts +++ b/packages/evals/tasks/bench/act/scroll_75.ts @@ -1,12 +1,11 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "scroll_75" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/"); - await v3.act("Scroll 75% down the page"); + await stagehand.act("Scroll 75% down the page"); await new Promise((resolve) => setTimeout(resolve, 5000)); @@ -39,13 +38,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/shadow_dom.ts b/packages/evals/tasks/bench/act/shadow_dom.ts index 3ae9db8ec..22bc5e8f3 100644 --- a/packages/evals/tasks/bench/act/shadow_dom.ts +++ b/packages/evals/tasks/bench/act/shadow_dom.ts @@ -1,13 +1,18 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "shadow_dom" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/shadow-dom/"); - await v3.act("click the button"); - const extraction = await v3.extract("extract the page text"); + await stagehand.act("click the button"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -28,13 +33,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + message: `error: ${error instanceof Error ? error.message : String(error)}`, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/simple_google_search.ts b/packages/evals/tasks/bench/act/simple_google_search.ts index b44766420..6b9153ade 100644 --- a/packages/evals/tasks/bench/act/simple_google_search.ts +++ b/packages/evals/tasks/bench/act/simple_google_search.ts @@ -1,20 +1,19 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "simple_google_search" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/google/"); - await v3.act('type "OpenAI" into the search bar'); + await stagehand.act('type "OpenAI" into the search bar'); - await v3.act("press enter"); + await stagehand.act("press enter"); await new Promise((resolve) => setTimeout(resolve, 3000)); const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/google/openai.html"; - const currentUrl = page.url(); + const currentUrl = await page.url(); return { _success: currentUrl.startsWith(expectedUrl), @@ -26,13 +25,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/spif_in_csr.ts b/packages/evals/tasks/bench/act/spif_in_csr.ts index f6c7926e2..a75a179de 100644 --- a/packages/evals/tasks/bench/act/spif_in_csr.ts +++ b/packages/evals/tasks/bench/act/spif_in_csr.ts @@ -1,19 +1,24 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "spif_in_csr" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // click inside a SPIF (same process iframe) that is inside an // CSR (closed mode shadow) root try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/spif-in-closed-shadow-dom/", ); - await v3.act("click the button"); + await stagehand.act("click the button"); - const extraction = await v3.extract("extract the entire page text"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -36,13 +41,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/spif_in_osr.ts b/packages/evals/tasks/bench/act/spif_in_osr.ts index ae49f321f..db56d295b 100644 --- a/packages/evals/tasks/bench/act/spif_in_osr.ts +++ b/packages/evals/tasks/bench/act/spif_in_osr.ts @@ -1,20 +1,25 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "spif_in_osr" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { // this eval is designed to test whether stagehand can successfully // click inside a SPIF (same process iframe) that is inside an // OSR (open mode shadow) root try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/spif-in-open-shadow-dom/", ); - await v3.act("click the button"); + await stagehand.act("click the button"); - const extraction = await v3.extract("extract the entire page text"); + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { data: extraction } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + ); const pageText = extraction.extraction; @@ -37,13 +42,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: `error: ${error.message}`, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/tab_handling.ts b/packages/evals/tasks/bench/act/tab_handling.ts index f8ed321d5..5165e68d1 100644 --- a/packages/evals/tasks/bench/act/tab_handling.ts +++ b/packages/evals/tasks/bench/act/tab_handling.ts @@ -1,25 +1,39 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "tab_handling" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/new-tab/"); - await v3.act("click the button to open the other page"); + await stagehand.act("click the button to open the other page"); - const pages = v3.context.pages(); + // activePage() polls until the popup registers (stagehand#2458), so + // the page list is complete after this settles. + await stagehand.context.activePage(); + + const pages = await stagehand.context.pages(); const page1 = pages[0]; const page2 = pages[1]; + // v3 used schemaless extract; v4 requires a schema. + // extract all the text from the first page - const extraction1 = await v3.extract({ page: page1 }); + const { data: extraction1 } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + { page: page1 }, + ); // extract all the text from the second page - const extraction2 = await v3.extract({ page: page2 }); + const { data: extraction2 } = await stagehand.extract( + "extract the entire page text", + z.object({ extraction: z.string() }), + { page: page2 }, + ); - const extraction1Success = extraction1.pageText.includes("Welcome!"); - const extraction2Success = extraction2.pageText.includes("You’re on the other page"); + const extraction1Success = extraction1.extraction.includes("Welcome!"); + const extraction2Success = extraction2.extraction.includes("You’re on the other page"); return { _success: extraction1Success && extraction2Success, @@ -30,13 +44,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - message: error.message, + message: (error as Error).message, debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/vantechjournal.ts b/packages/evals/tasks/bench/act/vantechjournal.ts index 22d433ff8..f9f34ff8f 100644 --- a/packages/evals/tasks/bench/act/vantechjournal.ts +++ b/packages/evals/tasks/bench/act/vantechjournal.ts @@ -1,16 +1,15 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "vantechjournal" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://vantechjournal.com"); - await v3.act("click on page 'recommendations'"); + await stagehand.act("click on page 'recommendations'"); const expectedUrl = "https://vantechjournal.com/recommendations"; - const currentUrl = page.url(); + const currentUrl = await page.url(); return { _success: currentUrl === expectedUrl, @@ -23,13 +22,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/act/wikipedia.ts b/packages/evals/tasks/bench/act/wikipedia.ts index b4e6023e8..55546f41d 100644 --- a/packages/evals/tasks/bench/act/wikipedia.ts +++ b/packages/evals/tasks/bench/act/wikipedia.ts @@ -1,17 +1,16 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "wikipedia" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto(`https://en.wikipedia.org/wiki/Baseball`); - await v3.act('click the "hit and run" link in this article', { + await stagehand.act('click the "hit and run" link in this article', { timeout: 360_000, }); const url = "https://en.wikipedia.org/wiki/Hit_and_run_(baseball)"; - const currentUrl = page.url(); + const currentUrl = await page.url(); return { _success: currentUrl === url, @@ -24,13 +23,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_aigrant_targeted.ts b/packages/evals/tasks/bench/extract/extract_aigrant_targeted.ts index 91f5b4e6b..ed93aa7e0 100644 --- a/packages/evals/tasks/bench/extract/extract_aigrant_targeted.ts +++ b/packages/evals/tasks/bench/extract/extract_aigrant_targeted.ts @@ -1,14 +1,15 @@ import { z } from "zod"; -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_aigrant_targeted" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/"); + // NOTE: v3 passes a bare XPath here; v4 documents options.selector as + // CSS-only. Ported verbatim on purpose. const selector = "/html/body/div/ul[5]/li[28]"; - const company = await v3.extract( + const { data: company } = await stagehand.extract( "Extract the company name.", z.object({ company_name: z.string(), @@ -57,13 +58,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_aigrant_targeted_2.ts b/packages/evals/tasks/bench/extract/extract_aigrant_targeted_2.ts index 14748477a..d900bcd72 100644 --- a/packages/evals/tasks/bench/extract/extract_aigrant_targeted_2.ts +++ b/packages/evals/tasks/bench/extract/extract_aigrant_targeted_2.ts @@ -1,14 +1,14 @@ import { z } from "zod"; -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_aigrant_targeted_2" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/"); + // NOTE: v3 passes a bare XPath here; ported verbatim on purpose. const selector = "/html/body/div/ul[5]/li[28]"; - const company = await v3.extract( + const { data: company } = await stagehand.extract( "Extract the name of the company that comes after 'Coframe'.", z.object({ company_name: z.string(), @@ -61,13 +61,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_apartments.ts b/packages/evals/tasks/bench/extract/extract_apartments.ts index d1f1f95b3..586481d0f 100644 --- a/packages/evals/tasks/bench/extract/extract_apartments.ts +++ b/packages/evals/tasks/bench/extract/extract_apartments.ts @@ -1,15 +1,14 @@ import { z } from "zod"; -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_apartments" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://www.apartments.com/san-francisco-ca/2-bedrooms/", { waitUntil: "load", }); - const apartment_listings = await v3.extract( + const { data: apartment_listings } = await stagehand.extract( "Extract all the apartment listings with their prices and their addresses.", z.object({ listings: z.array( @@ -57,13 +56,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_area_codes.ts b/packages/evals/tasks/bench/extract/extract_area_codes.ts index 5923188c0..9947dcd96 100644 --- a/packages/evals/tasks/bench/extract/extract_area_codes.ts +++ b/packages/evals/tasks/bench/extract/extract_area_codes.ts @@ -1,16 +1,15 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_area_codes" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/ncc-area-codes/", { waitUntil: "domcontentloaded", }); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract ALL the Primary Center names and their corresponding Area Code, and the name of their corresponding Zone.", z.object({ primary_center_list: z.array( @@ -141,13 +140,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_baptist_health.ts b/packages/evals/tasks/bench/extract/extract_baptist_health.ts index 48d36d369..dcd0baa93 100644 --- a/packages/evals/tasks/bench/extract/extract_baptist_health.ts +++ b/packages/evals/tasks/bench/extract/extract_baptist_health.ts @@ -1,15 +1,92 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; -import { compareStrings } from "../../../utils.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +/** + * Inlined behavior-identical copies of `normalizeString`/`compareStrings` from + * stagehand packages/evals/utils.ts and the `jaroWinkler` similarity from + * string-comparison@1.3.0 — v4 eval tasks may only import "zod" and + * "../../../framework/*.js". Pure computation, no behavior change. + */ +function normalizeString(str: string): string { + return str + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") + .replace(/\s*,\s*/g, ", ") + .trim(); +} + +function jaroSimilarity(first: string, second: string): number { + // string-comparison initParams: strip all whitespace + lowercase + const s1 = first.replace(/\s+/g, "").toLowerCase(); + const s2 = second.replace(/\s+/g, "").toLowerCase(); + if (!s1.length && !s2.length) return 1; + if (!s1.length || !s2.length) return 0; + if (s1 === s2) return 1; + const len1 = s1.length; + const len2 = s2.length; + if (len1 === 0 || len2 === 0) return 0; + const matchWindow = Math.floor(Math.max(len1, len2) / 2) - 1; + let matches = 0; + const s1Matches = new Array(s1.length).fill(0); + const s2Matches = new Array(s2.length).fill(0); + for (let i = 0; i < len1; i++) { + for (let j = Math.max(0, i - matchWindow); j < Math.min(len2, i + matchWindow + 1); j++) { + if (s1[i] === s2[j] && s2Matches[j] === 0) { + s1Matches[i] = 1; + s2Matches[j] = 1; + matches++; + break; + } + } + } + if (matches === 0) return 0; + let transpositions = 0; + let k = 0; + for (let i = 0; i < len1; i++) { + if (s1Matches[i] === 1) { + while (s2Matches[k] === 0) k++; + if (s1[i] !== s2[k++]) transpositions++; + } + } + transpositions /= 2; + return (matches / len1 + matches / len2 + (matches - transpositions) / matches) / 3; +} + +function jaroWinklerSimilarity(a: string, b: string): number { + let sim = jaroSimilarity(a, b); + if (sim > 0.7) { + // NOTE: string-comparison computes the common prefix on the raw inputs + // (pre-initParams), preserved here. + let prefix = 0; + for (let i = 0; i < Math.min(a.length, b.length) && a[i] === b[i]; i++) { + prefix++; + } + prefix = Math.min(4, prefix); + sim += 0.1 * prefix * (1 - sim); + } + return sim; +} + +function compareStrings( + actual: string, + expected: string, + similarityThreshold: number = 0.85, +): { similarity: number; meetsThreshold: boolean } { + const similarity = jaroWinklerSimilarity(normalizeString(actual), normalizeString(expected)); + return { + similarity, + meetsThreshold: similarity >= similarityThreshold, + }; +} + +export default defineBenchV4Task( { name: "extract_baptist_health" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/baptist-health/"); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract the address, phone number, and fax number of the healthcare location.", z.object({ address: z.string(), @@ -86,13 +163,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_csa.ts b/packages/evals/tasks/bench/extract/extract_csa.ts index 0d4bf2e63..da804be06 100644 --- a/packages/evals/tasks/bench/extract/extract_csa.ts +++ b/packages/evals/tasks/bench/extract/extract_csa.ts @@ -1,14 +1,13 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_csa" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/csa/"); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract all the publications on the page including the publication date, session type, publication type, and annotation", z.object({ publications: z.array( @@ -20,6 +19,7 @@ export default defineBenchTask( }), ), }), + { page }, ); const publications = result.publications; @@ -139,13 +139,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_geniusee.ts b/packages/evals/tasks/bench/extract/extract_geniusee.ts index 210b18b39..8d762c5bf 100644 --- a/packages/evals/tasks/bench/extract/extract_geniusee.ts +++ b/packages/evals/tasks/bench/extract/extract_geniusee.ts @@ -1,14 +1,14 @@ import { z } from "zod"; -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_geniusee" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/geniusee/"); + // NOTE: v3 passes a bare XPath here; ported verbatim on purpose. const selector = "/html/body/main/div[2]/div[2]/div[2]/table"; - const scalability = await v3.extract( + const { data: scalability } = await stagehand.extract( "Extract the scalability comment in the table for Gemini (Google)", z.object({ scalability: z.string(), @@ -57,13 +57,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_geniusee_2.ts b/packages/evals/tasks/bench/extract/extract_geniusee_2.ts index cb8cf9fd1..4400525da 100644 --- a/packages/evals/tasks/bench/extract/extract_geniusee_2.ts +++ b/packages/evals/tasks/bench/extract/extract_geniusee_2.ts @@ -1,14 +1,14 @@ import { z } from "zod"; -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_geniusee_2" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/geniusee/"); + // NOTE: v3 passes a bare XPath here; ported verbatim on purpose. const selector = "/html/body/main/div[2]/div[2]/div[2]/table/tbody/tr[9]"; - const scalability = await v3.extract( + const { data: scalability } = await stagehand.extract( "Extract the scalability comment in the table for Gemini (Google)", z.object({ scalability: z.string(), @@ -63,13 +63,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_github_stars.ts b/packages/evals/tasks/bench/extract/extract_github_stars.ts index 6c17c53d9..fbaf94161 100644 --- a/packages/evals/tasks/bench/extract/extract_github_stars.ts +++ b/packages/evals/tasks/bench/extract/extract_github_stars.ts @@ -1,14 +1,15 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_github_stars" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://github.com/facebook/react"); - const { stars } = await v3.extract( + const { + data: { stars }, + } = await stagehand.extract( "Extract the number of stars for the project", z.object({ stars: z.number().describe("the number of stars for the project"), @@ -37,13 +38,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_hamilton_weather.ts b/packages/evals/tasks/bench/extract/extract_hamilton_weather.ts index d6c8bcd12..97aeecf35 100644 --- a/packages/evals/tasks/bench/extract/extract_hamilton_weather.ts +++ b/packages/evals/tasks/bench/extract/extract_hamilton_weather.ts @@ -1,16 +1,94 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; -import { compareStrings } from "../../../utils.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +/** + * Inlined behavior-identical copies of `normalizeString`/`compareStrings` from + * stagehand packages/evals/utils.ts and the `jaroWinkler` similarity from + * string-comparison@1.3.0 — v4 eval tasks may only import "zod" and + * "../../../framework/*.js". Pure computation, no behavior change. + */ +function normalizeString(str: string): string { + return str + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") + .replace(/\s*,\s*/g, ", ") + .trim(); +} + +function jaroSimilarity(first: string, second: string): number { + // string-comparison initParams: strip all whitespace + lowercase + const s1 = first.replace(/\s+/g, "").toLowerCase(); + const s2 = second.replace(/\s+/g, "").toLowerCase(); + if (!s1.length && !s2.length) return 1; + if (!s1.length || !s2.length) return 0; + if (s1 === s2) return 1; + const len1 = s1.length; + const len2 = s2.length; + if (len1 === 0 || len2 === 0) return 0; + const matchWindow = Math.floor(Math.max(len1, len2) / 2) - 1; + let matches = 0; + const s1Matches = new Array(s1.length).fill(0); + const s2Matches = new Array(s2.length).fill(0); + for (let i = 0; i < len1; i++) { + for (let j = Math.max(0, i - matchWindow); j < Math.min(len2, i + matchWindow + 1); j++) { + if (s1[i] === s2[j] && s2Matches[j] === 0) { + s1Matches[i] = 1; + s2Matches[j] = 1; + matches++; + break; + } + } + } + if (matches === 0) return 0; + let transpositions = 0; + let k = 0; + for (let i = 0; i < len1; i++) { + if (s1Matches[i] === 1) { + while (s2Matches[k] === 0) k++; + if (s1[i] !== s2[k++]) transpositions++; + } + } + transpositions /= 2; + return (matches / len1 + matches / len2 + (matches - transpositions) / matches) / 3; +} + +function jaroWinklerSimilarity(a: string, b: string): number { + let sim = jaroSimilarity(a, b); + if (sim > 0.7) { + // NOTE: string-comparison computes the common prefix on the raw inputs + // (pre-initParams), preserved here. + let prefix = 0; + for (let i = 0; i < Math.min(a.length, b.length) && a[i] === b[i]; i++) { + prefix++; + } + prefix = Math.min(4, prefix); + sim += 0.1 * prefix * (1 - sim); + } + return sim; +} + +function compareStrings( + actual: string, + expected: string, + similarityThreshold: number = 0.85, +): { similarity: number; meetsThreshold: boolean } { + const similarity = jaroWinklerSimilarity(normalizeString(actual), normalizeString(expected)); + return { + similarity, + meetsThreshold: similarity >= similarityThreshold, + }; +} + +export default defineBenchV4Task( { name: "extract_hamilton_weather" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/hamilton-weather/"); + // NOTE: v3 passes a bare XPath here; ported verbatim on purpose. const xpath = "/html/body[1]/div[5]/main[1]/article[1]/div[6]/div[2]/div[1]/table[1]"; - const weatherData = await v3.extract( + const { data: weatherData } = await stagehand.extract( "extract the weather data for Sun, Feb 23 at 11PM", z.object({ temperature: z.string(), @@ -57,13 +135,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_jfk_links.ts b/packages/evals/tasks/bench/extract/extract_jfk_links.ts index 39dd0d0e1..7ac0a3dae 100644 --- a/packages/evals/tasks/bench/extract/extract_jfk_links.ts +++ b/packages/evals/tasks/bench/extract/extract_jfk_links.ts @@ -1,14 +1,13 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_jfk_links" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/jfk/"); - const extraction = await v3.extract( + const { data: extraction } = await stagehand.extract( "extract all the record file name and their corresponding links", z.object({ records: z.array( @@ -106,13 +105,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_jstor_news.ts b/packages/evals/tasks/bench/extract/extract_jstor_news.ts index c5507c48d..cbafac3a8 100644 --- a/packages/evals/tasks/bench/extract/extract_jstor_news.ts +++ b/packages/evals/tasks/bench/extract/extract_jstor_news.ts @@ -1,23 +1,16 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_jstor_news" }, - async ({ - logger, - - debugUrl, - sessionUrl, - v3, - }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/jstor/", { waitUntil: "load", }); - await v3.act("close the cookie"); + await stagehand.act("close the cookie"); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract ALL the news report titles and their dates.", z.object({ reports: z.array( @@ -130,13 +123,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_memorial_healthcare.ts b/packages/evals/tasks/bench/extract/extract_memorial_healthcare.ts index 70a2af861..f74037131 100644 --- a/packages/evals/tasks/bench/extract/extract_memorial_healthcare.ts +++ b/packages/evals/tasks/bench/extract/extract_memorial_healthcare.ts @@ -1,21 +1,92 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; -import { compareStrings } from "../../../utils.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( - { name: "extract_memorial_healthcare" }, - async ({ - logger, +/** + * Inlined behavior-identical copies of `normalizeString`/`compareStrings` from + * stagehand packages/evals/utils.ts and the `jaroWinkler` similarity from + * string-comparison@1.3.0 — v4 eval tasks may only import "zod" and + * "../../../framework/*.js". Pure computation, no behavior change. + */ +function normalizeString(str: string): string { + return str + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") + .replace(/\s*,\s*/g, ", ") + .trim(); +} + +function jaroSimilarity(first: string, second: string): number { + // string-comparison initParams: strip all whitespace + lowercase + const s1 = first.replace(/\s+/g, "").toLowerCase(); + const s2 = second.replace(/\s+/g, "").toLowerCase(); + if (!s1.length && !s2.length) return 1; + if (!s1.length || !s2.length) return 0; + if (s1 === s2) return 1; + const len1 = s1.length; + const len2 = s2.length; + if (len1 === 0 || len2 === 0) return 0; + const matchWindow = Math.floor(Math.max(len1, len2) / 2) - 1; + let matches = 0; + const s1Matches = new Array(s1.length).fill(0); + const s2Matches = new Array(s2.length).fill(0); + for (let i = 0; i < len1; i++) { + for (let j = Math.max(0, i - matchWindow); j < Math.min(len2, i + matchWindow + 1); j++) { + if (s1[i] === s2[j] && s2Matches[j] === 0) { + s1Matches[i] = 1; + s2Matches[j] = 1; + matches++; + break; + } + } + } + if (matches === 0) return 0; + let transpositions = 0; + let k = 0; + for (let i = 0; i < len1; i++) { + if (s1Matches[i] === 1) { + while (s2Matches[k] === 0) k++; + if (s1[i] !== s2[k++]) transpositions++; + } + } + transpositions /= 2; + return (matches / len1 + matches / len2 + (matches - transpositions) / matches) / 3; +} - debugUrl, - sessionUrl, - v3, - }) => { +function jaroWinklerSimilarity(a: string, b: string): number { + let sim = jaroSimilarity(a, b); + if (sim > 0.7) { + // NOTE: string-comparison computes the common prefix on the raw inputs + // (pre-initParams), preserved here. + let prefix = 0; + for (let i = 0; i < Math.min(a.length, b.length) && a[i] === b[i]; i++) { + prefix++; + } + prefix = Math.min(4, prefix); + sim += 0.1 * prefix * (1 - sim); + } + return sim; +} + +function compareStrings( + actual: string, + expected: string, + similarityThreshold: number = 0.85, +): { similarity: number; meetsThreshold: boolean } { + const similarity = jaroWinklerSimilarity(normalizeString(actual), normalizeString(expected)); + return { + similarity, + meetsThreshold: similarity >= similarityThreshold, + }; +} + +export default defineBenchV4Task( + { name: "extract_memorial_healthcare" }, + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/mycmh/"); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "extract a list of the first three healthcare centers on this page, with their name, full address, and phone number", z.object({ health_centers: z.array( @@ -185,13 +256,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_nhl_stats.ts b/packages/evals/tasks/bench/extract/extract_nhl_stats.ts index bf2600acf..ea66d4995 100644 --- a/packages/evals/tasks/bench/extract/extract_nhl_stats.ts +++ b/packages/evals/tasks/bench/extract/extract_nhl_stats.ts @@ -1,17 +1,29 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; -import { normalizeString } from "../../../utils.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +/** + * Inlined behavior-identical copy of `normalizeString` from stagehand + * packages/evals/utils.ts — v4 eval tasks may only import "zod" and + * "../../../framework/*.js". Pure computation, no behavior change. + */ +function normalizeString(str: string): string { + return str + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") + .replace(/\s*,\s*/g, ", ") + .trim(); +} + +export default defineBenchV4Task( { name: "extract_nhl_stats" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://www.hockeydb.com/ihdb/stats/top_league.php?lid=nhl1927&sid=1990", { waitUntil: "domcontentloaded", }); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract the name of the goal scoring leader, their number of goals they scored, and the team they played for.", z.object({ name: z.string(), @@ -109,13 +121,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_professional_info.ts b/packages/evals/tasks/bench/extract/extract_professional_info.ts index b64eed811..f23e9d9e5 100644 --- a/packages/evals/tasks/bench/extract/extract_professional_info.ts +++ b/packages/evals/tasks/bench/extract/extract_professional_info.ts @@ -1,17 +1,29 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; -import { normalizeString } from "../../../utils.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +/** + * Inlined behavior-identical copy of `normalizeString` from stagehand + * packages/evals/utils.ts — v4 eval tasks may only import "zod" and + * "../../../framework/*.js". Pure computation, no behavior change. + */ +function normalizeString(str: string): string { + return str + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") + .replace(/\s*,\s*/g, ", ") + .trim(); +} + +export default defineBenchV4Task( { name: "extract_professional_info" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/professional-info/", ); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract the list of Practices, phone number, and fax number of the professional.", z.object({ practices: z.array(z.string()), @@ -20,7 +32,8 @@ export default defineBenchTask( }), ); - await v3.close(); + // v3 closes mid-task here (and again in finally); preserved verbatim. + await stagehand.close(); const { practices, phone, fax } = result; @@ -119,13 +132,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_public_notices.ts b/packages/evals/tasks/bench/extract/extract_public_notices.ts index a83a9753d..eab9ad465 100644 --- a/packages/evals/tasks/bench/extract/extract_public_notices.ts +++ b/packages/evals/tasks/bench/extract/extract_public_notices.ts @@ -1,17 +1,94 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; -import { compareStrings } from "../../../utils.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +/** + * Inlined behavior-identical copies of `normalizeString`/`compareStrings` from + * stagehand packages/evals/utils.ts and the `jaroWinkler` similarity from + * string-comparison@1.3.0 — v4 eval tasks may only import "zod" and + * "../../../framework/*.js". Pure computation, no behavior change. + */ +function normalizeString(str: string): string { + return str + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") + .replace(/\s*,\s*/g, ", ") + .trim(); +} + +function jaroSimilarity(first: string, second: string): number { + // string-comparison initParams: strip all whitespace + lowercase + const s1 = first.replace(/\s+/g, "").toLowerCase(); + const s2 = second.replace(/\s+/g, "").toLowerCase(); + if (!s1.length && !s2.length) return 1; + if (!s1.length || !s2.length) return 0; + if (s1 === s2) return 1; + const len1 = s1.length; + const len2 = s2.length; + if (len1 === 0 || len2 === 0) return 0; + const matchWindow = Math.floor(Math.max(len1, len2) / 2) - 1; + let matches = 0; + const s1Matches = new Array(s1.length).fill(0); + const s2Matches = new Array(s2.length).fill(0); + for (let i = 0; i < len1; i++) { + for (let j = Math.max(0, i - matchWindow); j < Math.min(len2, i + matchWindow + 1); j++) { + if (s1[i] === s2[j] && s2Matches[j] === 0) { + s1Matches[i] = 1; + s2Matches[j] = 1; + matches++; + break; + } + } + } + if (matches === 0) return 0; + let transpositions = 0; + let k = 0; + for (let i = 0; i < len1; i++) { + if (s1Matches[i] === 1) { + while (s2Matches[k] === 0) k++; + if (s1[i] !== s2[k++]) transpositions++; + } + } + transpositions /= 2; + return (matches / len1 + matches / len2 + (matches - transpositions) / matches) / 3; +} + +function jaroWinklerSimilarity(a: string, b: string): number { + let sim = jaroSimilarity(a, b); + if (sim > 0.7) { + // NOTE: string-comparison computes the common prefix on the raw inputs + // (pre-initParams), preserved here. + let prefix = 0; + for (let i = 0; i < Math.min(a.length, b.length) && a[i] === b[i]; i++) { + prefix++; + } + prefix = Math.min(4, prefix); + sim += 0.1 * prefix * (1 - sim); + } + return sim; +} + +function compareStrings( + actual: string, + expected: string, + similarityThreshold: number = 0.85, +): { similarity: number; meetsThreshold: boolean } { + const similarity = jaroWinklerSimilarity(normalizeString(actual), normalizeString(expected)); + return { + similarity, + meetsThreshold: similarity >= similarityThreshold, + }; +} + +export default defineBenchV4Task( { name: "extract_public_notices" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/sars/", { waitUntil: "load", }); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract ALL the public notice descriptions with their corresponding, GG number and publication date. Extract ALL notices from 2024 through 2020. Do not include the Notice number.", z.object({ public_notices: z.array( @@ -68,6 +145,8 @@ export default defineBenchTask( sessionUrl, }; } + // NOTE (preserved v3 quirk): compareStrings returns an object, which is + // always truthy — these `&&` chains never actually gate on similarity. const firstItemMatches = compareStrings( publicNotices[0].notice_description, @@ -151,13 +230,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_recipe.ts b/packages/evals/tasks/bench/extract/extract_recipe.ts index 145b16f89..393ecb134 100644 --- a/packages/evals/tasks/bench/extract/extract_recipe.ts +++ b/packages/evals/tasks/bench/extract/extract_recipe.ts @@ -1,11 +1,10 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_recipe" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/allrecipes-extract/", { @@ -13,8 +12,9 @@ export default defineBenchTask( }, ); + // NOTE: v3 passes a bare XPath here; ported verbatim on purpose. const selector = "/html/body/main/article/div[3]/div[3]/div[4]"; - const recipeDetails = await v3.extract( + const { data: recipeDetails } = await stagehand.extract( "Extract the title of the number of tablespoons of olive oil needed for the steak, and the number of teaspoons of lemon juice needed for the mushroom pan sauce.", z.object({ tablespoons_olive_oil: z @@ -86,13 +86,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_regulations_table.ts b/packages/evals/tasks/bench/extract/extract_regulations_table.ts index c2d5a5dc8..84b675e96 100644 --- a/packages/evals/tasks/bench/extract/extract_regulations_table.ts +++ b/packages/evals/tasks/bench/extract/extract_regulations_table.ts @@ -1,19 +1,19 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_regulations_table" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto( "https://browserbase.github.io/stagehand-eval-sites/sites/ncc-numbering-plan/", ); + // NOTE: v3 passes a bare XPath here; ported verbatim on purpose. const xpath = "/html/body/div[3]/main/div[2]/div[2]/div/div/div[2]/article/div[2]/div[1]/div/table"; - const allottees = await v3.extract( + const { data: allottees } = await stagehand.extract( "Extract ALL of the Allottees and their corresponding name, area, and area code.", z.object({ allottee_list: z.array( @@ -67,13 +67,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_repo_name.ts b/packages/evals/tasks/bench/extract/extract_repo_name.ts index 88e2f5294..140c06bd5 100644 --- a/packages/evals/tasks/bench/extract/extract_repo_name.ts +++ b/packages/evals/tasks/bench/extract/extract_repo_name.ts @@ -1,14 +1,19 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_repo_name" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://github.com/facebook/react"); - const { extraction } = await v3.extract( + // v3 used schemaless extract; v4 requires a schema. + // Single-word key to stay clear of the snake_case wire-casing bug (#14). + const { + data: { extraction }, + } = await stagehand.extract( "extract the title of the Github repository. Do not include the owner of the repository.", + z.object({ extraction: z.string() }), ); logger.log({ @@ -32,13 +37,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_resistor_info.ts b/packages/evals/tasks/bench/extract/extract_resistor_info.ts index 55b4a3751..83cd836b5 100644 --- a/packages/evals/tasks/bench/extract/extract_resistor_info.ts +++ b/packages/evals/tasks/bench/extract/extract_resistor_info.ts @@ -1,15 +1,27 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; -import { normalizeString } from "../../../utils.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +/** + * Inlined behavior-identical copy of `normalizeString` from stagehand + * packages/evals/utils.ts — v4 eval tasks may only import "zod" and + * "../../../framework/*.js". Pure computation, no behavior change. + */ +function normalizeString(str: string): string { + return str + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") + .replace(/\s*,\s*/g, ", ") + .trim(); +} + +export default defineBenchV4Task( { name: "extract_resistor_info" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/resistor/"); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract the manufacturer standard lead time, tolerance percentage, resistance, and operating temperature range of the resistor.", z.object({ manufacturer_standard_lead_time: z.string(), @@ -146,13 +158,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_rockauto.ts b/packages/evals/tasks/bench/extract/extract_rockauto.ts index 9cea2beb1..495cac952 100644 --- a/packages/evals/tasks/bench/extract/extract_rockauto.ts +++ b/packages/evals/tasks/bench/extract/extract_rockauto.ts @@ -1,14 +1,13 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_rockauto" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/rockauto/"); await new Promise((resolve) => setTimeout(resolve, 5000)); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "Extract the part number of all the coolant and antifreeze products in the 'economy' category. " + "Do not include the manufacturer name. Do not include products from the premium category.", z.object({ @@ -85,13 +84,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_single_link.ts b/packages/evals/tasks/bench/extract/extract_single_link.ts index 3b7a98cd7..e84ae1990 100644 --- a/packages/evals/tasks/bench/extract/extract_single_link.ts +++ b/packages/evals/tasks/bench/extract/extract_single_link.ts @@ -1,14 +1,13 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_single_link" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/geniusee/"); - const extraction = await v3.extract( + const { data: extraction } = await stagehand.extract( "extract the link to the 'contact us' page", z.object({ link: z.string().url(), @@ -36,13 +35,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_staff_members.ts b/packages/evals/tasks/bench/extract/extract_staff_members.ts index 7897cb6e1..ea6aa8a9c 100644 --- a/packages/evals/tasks/bench/extract/extract_staff_members.ts +++ b/packages/evals/tasks/bench/extract/extract_staff_members.ts @@ -1,14 +1,13 @@ import { z } from "zod"; -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_staff_members" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/panamcs/"); - const result = await v3.extract( + const { data: result } = await stagehand.extract( "extract a list of ALL the staff members on this page, with their name and their job title", z.object({ staff_members: z.array( @@ -130,13 +129,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/extract_zillow.ts b/packages/evals/tasks/bench/extract/extract_zillow.ts index 3166dd3b0..978983df3 100644 --- a/packages/evals/tasks/bench/extract/extract_zillow.ts +++ b/packages/evals/tasks/bench/extract/extract_zillow.ts @@ -1,14 +1,13 @@ import { z } from "zod"; -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "extract_zillow" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/zillow/"); - const real_estate_listings = await v3.extract( + const { data: real_estate_listings } = await stagehand.extract( "Extract EACH AND EVERY HOME PRICE AND ADDRESS ON THE PAGE. DO NOT MISS ANY OF THEM.", z.object({ listings: z.array( @@ -20,7 +19,8 @@ export default defineBenchTask( }), ); - await v3.close(); + // v3 closes mid-task here (and again in finally); preserved verbatim. + await stagehand.close(); const listings = real_estate_listings.listings; const expectedLength = 38; @@ -57,13 +57,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/extract/iframe_hn.ts b/packages/evals/tasks/bench/extract/iframe_hn.ts index 314a783be..b595967a0 100644 --- a/packages/evals/tasks/bench/extract/iframe_hn.ts +++ b/packages/evals/tasks/bench/extract/iframe_hn.ts @@ -1,14 +1,16 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; import { z } from "zod"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "iframe_hn" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/iframe-hn/"); - const result = await v3.extract( + // NOTE: the target content lives inside an iframe, but the v3 task + // relies entirely on extract() to see into it (no frames() API usage), + // so this ports 1:1 — iframe handling is the SDK's responsibility. + const { data: result } = await stagehand.extract( "extract the title of the first hackernews story", z.object({ story_title: z.string(), @@ -41,13 +43,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), logs: logger.getLogs(), debugUrl, sessionUrl, }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/ionwave_observe.ts b/packages/evals/tasks/bench/observe/ionwave_observe.ts index cb0a9cd1e..92ec23c70 100644 --- a/packages/evals/tasks/bench/observe/ionwave_observe.ts +++ b/packages/evals/tasks/bench/observe/ionwave_observe.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "ionwave_observe" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/ionwave/"); - const observations = await v3.observe(); + const { data: observations } = await stagehand.observe(); if (observations.length === 0) { return { @@ -35,7 +34,7 @@ export default defineBenchTask( } catch (error) { console.warn( `Failed to check observation with selector ${observation.selector}:`, - error.message, + error instanceof Error ? error.message : String(error), ); continue; } @@ -52,13 +51,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_amazon_add_to_cart.ts b/packages/evals/tasks/bench/observe/observe_amazon_add_to_cart.ts index 5d23c4685..d740020c8 100644 --- a/packages/evals/tasks/bench/observe/observe_amazon_add_to_cart.ts +++ b/packages/evals/tasks/bench/observe/observe_amazon_add_to_cart.ts @@ -1,29 +1,32 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_amazon_add_to_cart" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/amazon/"); - const observations1 = await v3.observe("Find and click the 'Add to Cart' button"); + const { data: observations1 } = await stagehand.observe( + "Find and click the 'Add to Cart' button", + ); // Example of using performPlaywrightMethod if you have the xpath if (observations1.length > 0) { const action1 = observations1[0]; - await v3.act(action1); + await stagehand.act(action1); } - const observations2 = await v3.observe("Find and click the 'Proceed to checkout' button"); + const { data: observations2 } = await stagehand.observe( + "Find and click the 'Proceed to checkout' button", + ); // Example of using performPlaywrightMethod if you have the xpath if (observations2.length > 0) { const action2 = observations2[0]; - await v3.act(action2); + await stagehand.act(action2); } - const currentUrl = page.url(); + const currentUrl = await page.url(); const expectedUrlPrefix = "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/sign-in.html"; @@ -37,13 +40,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_file_uploads.ts b/packages/evals/tasks/bench/observe/observe_file_uploads.ts index f34b247ce..ce148c12e 100644 --- a/packages/evals/tasks/bench/observe/observe_file_uploads.ts +++ b/packages/evals/tasks/bench/observe/observe_file_uploads.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_file_uploads" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/file-uploads-3/"); - const observations = await v3.observe("find the file upload element"); + const { data: observations } = await stagehand.observe("find the file upload element"); if (observations.length === 0) { return { @@ -22,10 +21,42 @@ export default defineBenchTask( const expectedLocator = `xpath=/html/body/input`; - const expectedBackendNodeId = await page.locator(expectedLocator).backendNodeId(); + // v3 compares backendNodeIds; the v4 Locator exposes no node identity + // so the same element-identity check is + // re-expressed in-page: resolve the observed selector and the expected + // selector and compare element references. + const foundMatch = await page.evaluate( + ({ + observedSelector, + expectedSelector, + }: { + observedSelector: string; + expectedSelector: string; + }) => { + const resolve = (selector: string): Element | null => { + const raw = selector.startsWith("xpath=") ? selector.slice("xpath=".length) : selector; + if (raw.startsWith("/") || raw.startsWith("(")) { + const result = document.evaluate( + raw, + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null, + ); + return result.singleNodeValue as Element | null; + } + return document.querySelector(raw); + }; - const actualBackendNodeId = await page.locator(observations[0].selector).backendNodeId(); - const foundMatch = expectedBackendNodeId === actualBackendNodeId; + const expected = resolve(expectedSelector); + const observed = resolve(observedSelector); + return expected !== null && expected === observed; + }, + { + observedSelector: observations[0].selector, + expectedSelector: expectedLocator, + }, + ); return { _success: foundMatch, @@ -37,14 +68,14 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), message: "returned selector does not resolve to same node as expected", debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_github.ts b/packages/evals/tasks/bench/observe/observe_github.ts index f57e192cf..22ee85f12 100644 --- a/packages/evals/tasks/bench/observe/observe_github.ts +++ b/packages/evals/tasks/bench/observe/observe_github.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_github" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/github/"); - const observations = await v3.observe( + const { data: observations } = await stagehand.observe( "find the scrollable element that holds the repos file tree.", ); @@ -33,35 +32,62 @@ export default defineBenchTask( `#repos-file-tree > div.Box-sc-g0xbh4-0.ReposFileTreePane-module__Box_5--tQNH_ > div > div > div > nav > ul`, ]; - // Precompute candidate backendNodeIds - const candidateIds = new Map(); - for (const sel of possibleLocators) { - try { - const id = await page.locator(sel).backendNodeId(); - candidateIds.set(sel, id); - } catch { - // ignore candidates that fail to resolve - } - } - + // v3 compares backendNodeIds; the v4 Locator exposes no node identity + // so the same element-identity check is + // re-expressed in-page: resolve the observed selector and each + // candidate selector and compare element references. Candidates that + // fail to resolve are ignored, as in v3. let foundMatch = false; let matchedLocator: string | null = null; for (const observation of observations) { try { - const obsId = await page.locator(observation.selector).backendNodeId(); - for (const [candSel, candId] of candidateIds) { - if (candId === obsId) { - foundMatch = true; - matchedLocator = candSel; - break; - } + const matched = await page.evaluate( + ({ + observedSelector, + candidateSelectors, + }: { + observedSelector: string; + candidateSelectors: string[]; + }) => { + const resolve = (selector: string): Element | null => { + const raw = selector.startsWith("xpath=") + ? selector.slice("xpath=".length) + : selector; + if (raw.startsWith("/") || raw.startsWith("(")) { + const result = document.evaluate( + raw, + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null, + ); + return result.singleNodeValue as Element | null; + } + return document.querySelector(raw); + }; + + const observed = resolve(observedSelector); + if (!observed) return null; + for (const candidate of candidateSelectors) { + if (resolve(candidate) === observed) return candidate; + } + return null; + }, + { + observedSelector: observation.selector, + candidateSelectors: possibleLocators, + }, + ); + if (matched) { + foundMatch = true; + matchedLocator = matched; + break; } - if (foundMatch) break; } catch (error) { console.warn( `Failed to check observation with selector ${observation.selector}:`, - error?.message ?? String(error), + error instanceof Error ? error.message : String(error), ); continue; } @@ -78,13 +104,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_iframes1.ts b/packages/evals/tasks/bench/observe/observe_iframes1.ts index e2b56b347..e828f26eb 100644 --- a/packages/evals/tasks/bench/observe/observe_iframes1.ts +++ b/packages/evals/tasks/bench/observe/observe_iframes1.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_iframes1" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/iframe-hn/"); - const observations = await v3.observe("find the main header of the page"); + const { data: observations } = await stagehand.observe("find the main header of the page"); if (observations.length === 0) { return { @@ -24,35 +23,64 @@ export default defineBenchTask( `body > header > h1`, ]; - // Precompute candidate backendNodeIds - const candidateIds = new Map(); - for (const sel of possibleLocators) { - try { - const id = await page.locator(sel).backendNodeId(); - candidateIds.set(sel, id); - } catch { - // ignore candidates that fail to resolve - } - } - + // v3 compares backendNodeIds; the v4 Locator exposes no node identity + // so the same element-identity check is + // re-expressed in-page. Both candidate selectors live in the main + // frame, so main-frame resolution preserves the v3 pass criterion: + // an observed selector that points inside the iframe never had a + // backendNodeId equal to either main-frame candidate in v3 (no match), + // and here it simply fails to resolve in the main document (no match). let foundMatch = false; let matchedLocator: string | null = null; for (const observation of observations) { try { - const obsId = await page.locator(observation.selector).backendNodeId(); - for (const [candSel, candId] of candidateIds) { - if (candId === obsId) { - foundMatch = true; - matchedLocator = candSel; - break; - } + const matched = await page.evaluate( + ({ + observedSelector, + candidateSelectors, + }: { + observedSelector: string; + candidateSelectors: string[]; + }) => { + const resolve = (selector: string): Element | null => { + const raw = selector.startsWith("xpath=") + ? selector.slice("xpath=".length) + : selector; + if (raw.startsWith("/") || raw.startsWith("(")) { + const result = document.evaluate( + raw, + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null, + ); + return result.singleNodeValue as Element | null; + } + return document.querySelector(raw); + }; + + const observed = resolve(observedSelector); + if (!observed) return null; + for (const candidate of candidateSelectors) { + if (resolve(candidate) === observed) return candidate; + } + return null; + }, + { + observedSelector: observation.selector, + candidateSelectors: possibleLocators, + }, + ); + if (matched) { + foundMatch = true; + matchedLocator = matched; + break; } - if (foundMatch) break; } catch (error) { console.warn( `Failed to check observation with selector ${observation.selector}:`, - error?.message ?? String(error), + error instanceof Error ? error.message : String(error), ); continue; } @@ -69,13 +97,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_iframes2.ts b/packages/evals/tasks/bench/observe/observe_iframes2.ts index 0a0214991..c8648b73c 100644 --- a/packages/evals/tasks/bench/observe/observe_iframes2.ts +++ b/packages/evals/tasks/bench/observe/observe_iframes2.ts @@ -1,21 +1,20 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; -import { Action } from "stagehand-v3"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; +import type { Action } from "@browserbasehq/stagehand"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_iframes2" }, - async ({ logger, debugUrl, sessionUrl, v3 }) => { + async ({ logger, debugUrl, sessionUrl, stagehand, page }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://iframetester.com/?url=https://shopify.com"); await new Promise((resolve) => setTimeout(resolve, 5000)); let observations: Action[]; try { - observations = await v3.observe("find the main header of the page"); + observations = (await stagehand.observe("find the main header of the page")).data; } catch (err) { return { _success: false, - message: err.message, + message: err instanceof Error ? err.message : String(err), debugUrl, sessionUrl, logs: logger.getLogs(), @@ -34,35 +33,65 @@ export default defineBenchTask( const possibleLocators = [`#iframe-window`, `body > header > h1`]; - // Precompute candidate backendNodeIds - const candidateIds = new Map(); - for (const sel of possibleLocators) { - try { - const id = await page.locator(sel).backendNodeId(); - candidateIds.set(sel, id); - } catch { - // ignore candidates that fail to resolve - } - } - + // v3 compares backendNodeIds; the v4 Locator exposes no node identity + // so the same element-identity check is + // re-expressed in-page. Both candidate selectors live in the main + // frame (the shopify iframe is cross-origin and unreachable from the + // main document either way): an observed selector that pierces into + // the iframe never had a backendNodeId equal to either main-frame + // candidate in v3 (no match), and here it simply fails to resolve in + // the main document (no match) — the pass criterion is preserved. let foundMatch = false; let matchedLocator: string | null = null; for (const observation of observations) { try { - const obsId = await page.locator(observation.selector).backendNodeId(); - for (const [candSel, candId] of candidateIds) { - if (candId === obsId) { - foundMatch = true; - matchedLocator = candSel; - break; - } + const matched = await page.evaluate( + ({ + observedSelector, + candidateSelectors, + }: { + observedSelector: string; + candidateSelectors: string[]; + }) => { + const resolve = (selector: string): Element | null => { + const raw = selector.startsWith("xpath=") + ? selector.slice("xpath=".length) + : selector; + if (raw.startsWith("/") || raw.startsWith("(")) { + const result = document.evaluate( + raw, + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null, + ); + return result.singleNodeValue as Element | null; + } + return document.querySelector(raw); + }; + + const observed = resolve(observedSelector); + if (!observed) return null; + for (const candidate of candidateSelectors) { + if (resolve(candidate) === observed) return candidate; + } + return null; + }, + { + observedSelector: observation.selector, + candidateSelectors: possibleLocators, + }, + ); + if (matched) { + foundMatch = true; + matchedLocator = matched; + break; } - if (foundMatch) break; } catch (error) { console.warn( `Failed to check observation with selector ${observation.selector}:`, - error?.message ?? String(error), + error instanceof Error ? error.message : String(error), ); continue; } @@ -79,13 +108,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_main_frame_element_ids.ts b/packages/evals/tasks/bench/observe/observe_main_frame_element_ids.ts index e6beea0e1..760c02416 100644 --- a/packages/evals/tasks/bench/observe/observe_main_frame_element_ids.ts +++ b/packages/evals/tasks/bench/observe/observe_main_frame_element_ids.ts @@ -1,4 +1,5 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; +import type { Action } from "@browserbasehq/stagehand"; const filler = Array.from( // Keep backend node IDs large enough to exercise Anthropic's bare-id failure mode. @@ -48,20 +49,23 @@ function buildHtml(): string { `)}`; } -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_main_frame_element_ids" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto(buildHtml()); - const results = []; + const results: Array<{ + instruction: string; + clicked: string | undefined; + observations: Action[]; + }> = []; for (const testCase of cases) { await page.evaluate(() => { delete document.body.dataset.clicked; }); - const observations = await v3.observe(testCase.instruction); + const { data: observations } = await stagehand.observe(testCase.instruction); if (observations.length === 0) { return { _success: false, @@ -74,8 +78,10 @@ export default defineBenchTask( }; } - await v3.act(observations[0]); - const clicked = await page.evaluate(() => document.body.dataset.clicked); + await stagehand.act(observations[0]); + const clicked = await page.evaluate( + () => document.body.dataset.clicked, + ); results.push({ instruction: testCase.instruction, clicked, @@ -112,7 +118,7 @@ export default defineBenchTask( logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_simple_google_search.ts b/packages/evals/tasks/bench/observe/observe_simple_google_search.ts index e8211bf25..bf6e2e100 100644 --- a/packages/evals/tasks/bench/observe/observe_simple_google_search.ts +++ b/packages/evals/tasks/bench/observe/observe_simple_google_search.ts @@ -1,28 +1,29 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_simple_google_search" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/google/"); - const observation1 = await v3.observe("Find the search bar and type 'OpenAI'"); + const { data: observation1 } = await stagehand.observe( + "Find the search bar and type 'OpenAI'", + ); if (observation1.length > 0) { const action1 = observation1[0]; - await v3.act(action1); + await stagehand.act(action1); } - const observation2 = await v3.observe("Press enter"); + const { data: observation2 } = await stagehand.observe("Press enter"); if (observation2.length > 0) { const action2 = observation2[0]; - await v3.act(action2); + await stagehand.act(action2); } await new Promise((resolve) => setTimeout(resolve, 3000)); const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/google/openai.html"; - const currentUrl = page.url(); + const currentUrl = await page.url(); return { _success: currentUrl.startsWith(expectedUrl), @@ -34,13 +35,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_taxes.ts b/packages/evals/tasks/bench/observe/observe_taxes.ts index 0341d495a..12bd8238d 100644 --- a/packages/evals/tasks/bench/observe/observe_taxes.ts +++ b/packages/evals/tasks/bench/observe/observe_taxes.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_taxes" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://file.1040.com/estimate/"); - const observations = await v3.observe( + const { data: observations } = await stagehand.observe( "Find all the form input elements under the 'Income' section", ); @@ -45,7 +44,7 @@ export default defineBenchTask( } catch (error) { console.warn( `Failed to check observation with selector ${observation.selector}:`, - error.message, + error instanceof Error ? error.message : String(error), ); continue; } @@ -62,13 +61,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_vantechjournal.ts b/packages/evals/tasks/bench/observe/observe_vantechjournal.ts index b8fdc9bf5..45a26896a 100644 --- a/packages/evals/tasks/bench/observe/observe_vantechjournal.ts +++ b/packages/evals/tasks/bench/observe/observe_vantechjournal.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_vantechjournal" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://vantechjournal.com/archive"); - const observations = await v3.observe("Find the 'load more' link"); + const { data: observations } = await stagehand.observe("Find the 'load more' link"); if (observations.length === 0) { return { @@ -24,17 +23,47 @@ export default defineBenchTask( "xpath=/html/body/div[2]/div/div/section/div/div/div[3]/a/span", ]; - const expectedIds: number[] = []; - for (const locator of expectedLocators) { - const node = page.locator(locator); - const id = await node.backendNodeId(); - if (id !== undefined && id !== null) expectedIds.push(id); - } - - const observedNode = page.locator(observations[0].selector); - const observedId = await observedNode.backendNodeId(); + // v3 compares backendNodeIds (first observation vs. each expected + // locator); the v4 Locator exposes no node identity + // so the same element-identity check is + // re-expressed in-page: resolve the observed selector and each + // expected selector and compare element references. Expected locators + // that fail to resolve are skipped, as in v3. + const foundMatch = await page.evaluate( + ({ + observedSelector, + expectedSelectors, + }: { + observedSelector: string; + expectedSelectors: string[]; + }) => { + const resolve = (selector: string): Element | null => { + const raw = selector.startsWith("xpath=") ? selector.slice("xpath=".length) : selector; + if (raw.startsWith("/") || raw.startsWith("(")) { + const result = document.evaluate( + raw, + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null, + ); + return result.singleNodeValue as Element | null; + } + return document.querySelector(raw); + }; - const foundMatch = expectedIds.includes(observedId); + const observed = resolve(observedSelector); + if (!observed) return false; + for (const expected of expectedSelectors) { + if (resolve(expected) === observed) return true; + } + return false; + }, + { + observedSelector: observations[0].selector, + expectedSelectors: expectedLocators, + }, + ); return { _success: foundMatch, @@ -53,7 +82,7 @@ export default defineBenchTask( logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/observe_yc_startup.ts b/packages/evals/tasks/bench/observe/observe_yc_startup.ts index 04a70322e..546ec883e 100644 --- a/packages/evals/tasks/bench/observe/observe_yc_startup.ts +++ b/packages/evals/tasks/bench/observe/observe_yc_startup.ts @@ -1,15 +1,14 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "observe_yc_startup" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://www.ycombinator.com/companies", { waitUntil: "networkidle", }); - const observations = await v3.observe( + const { data: observations } = await stagehand.observe( "Click the container element that holds links to each of the startup companies. The companies each have a name, a description, and a link to their website.", ); @@ -28,35 +27,61 @@ export default defineBenchTask( `div._section_18olp_165._results_18olp_345`, ]; - // Precompute candidate backendNodeIds - const candidateIds = new Map(); - for (const sel of possibleLocators) { - try { - const id = await page.locator(sel).backendNodeId(); - candidateIds.set(sel, id); - } catch { - // ignore candidates that fail to resolve - } - } - + // v3 compares backendNodeIds; the v4 Locator exposes no node identity + // so the same element-identity check is + // re-expressed in-page: resolve the observed selector and each + // candidate selector and compare element references. let foundMatch = false; let matchedLocator: string | null = null; for (const observation of observations) { try { - const obsId = await page.locator(observation.selector).backendNodeId(); - for (const [candSel, candId] of candidateIds) { - if (candId === obsId) { - foundMatch = true; - matchedLocator = candSel; - break; - } + const matched = await page.evaluate( + ({ + observedSelector, + candidateSelectors, + }: { + observedSelector: string; + candidateSelectors: string[]; + }) => { + const resolve = (selector: string): Element | null => { + const raw = selector.startsWith("xpath=") + ? selector.slice("xpath=".length) + : selector; + if (raw.startsWith("/") || raw.startsWith("(")) { + const result = document.evaluate( + raw, + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null, + ); + return result.singleNodeValue as Element | null; + } + return document.querySelector(raw); + }; + + const observed = resolve(observedSelector); + if (!observed) return null; + for (const candidate of candidateSelectors) { + if (resolve(candidate) === observed) return candidate; + } + return null; + }, + { + observedSelector: observation.selector, + candidateSelectors: possibleLocators, + }, + ); + if (matched) { + foundMatch = true; + matchedLocator = matched; + break; } - if (foundMatch) break; } catch (error) { console.warn( `Failed to check observation with selector ${observation.selector}:`, - error?.message ?? String(error), + error instanceof Error ? error.message : String(error), ); continue; } @@ -73,13 +98,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, ); diff --git a/packages/evals/tasks/bench/observe/panamcs.ts b/packages/evals/tasks/bench/observe/panamcs.ts index 90845def3..3c306cef9 100644 --- a/packages/evals/tasks/bench/observe/panamcs.ts +++ b/packages/evals/tasks/bench/observe/panamcs.ts @@ -1,13 +1,12 @@ -import { defineBenchTask } from "../../../framework/defineTask.js"; +import { defineBenchV4Task } from "../../../framework/defineTask.js"; -export default defineBenchTask( +export default defineBenchV4Task( { name: "panamcs" }, - async ({ debugUrl, sessionUrl, v3, logger }) => { + async ({ debugUrl, sessionUrl, stagehand, page, logger }) => { try { - const page = v3.context.pages()[0]; await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/panamcs/"); - const observations = await v3.observe("click the 'about us' link"); + const { data: observations } = await stagehand.observe("click the 'about us' link"); if (observations.length === 0) { return { @@ -35,7 +34,7 @@ export default defineBenchTask( } catch (error) { console.warn( `Failed to check observation with selector ${observation.selector}:`, - error.message, + error instanceof Error ? error.message : String(error), ); continue; } @@ -52,13 +51,13 @@ export default defineBenchTask( } catch (error) { return { _success: false, - error: error, + error: error instanceof Error ? error.message : String(error), debugUrl, sessionUrl, logs: logger.getLogs(), }; } finally { - await v3.close(); + await stagehand.close(); } }, );