evals: port the act/extract/observe bench suite to the v4 SDK

packages/evals/framework/defineTask.ts

@@ -11,6 +11,7 @@ import type {
   TaskDefinition,
   TaskMeta,
   TaskResult,
+  BenchV4TaskContext,
 } from "./types.js";
 
 /**
@@ -43,6 +44,31 @@ export function defineBenchTask(
   };
 }
 
+/**
+ * Define a bench tier task ported to the Stagehand v4 SDK.
+ * v4 bench tasks receive { stagehand, page, logger, input, ... } and return
+ * TaskResult. They live under tasks/bench/ and are selected via --sdk v4.
+ */
+export function defineBenchV4Task(
+  meta: BenchTaskMeta,
+  fn: (ctx: BenchV4TaskContext) => Promise<void | TaskResult>,
+): TaskDefinition {
+  return {
+    __taskDefinition: true,
+    meta,
+    // Fail fast with a clear message if a v3-context runner invokes a v4
+    // task — the v4 init/dispatch path lands with the harness change.
+    fn: (ctx) => {
+      if (!ctx || !("stagehand" in ctx) || ctx.stagehand === undefined) {
+        throw new Error(
+          `Task "${meta.name}" requires the v4 harness (--sdk v4); it was invoked with a v3 context.`,
+        );
+      }
+      return fn(ctx as unknown as BenchV4TaskContext);
+    },
+  };
+}
+
 /**
  * Generic defineTask — for cases where the tier is ambiguous at definition time.
  * Prefer defineCoreTask / defineBenchTask for better type inference.

packages/evals/framework/types.ts

@@ -7,7 +7,8 @@
  *
  * A third tier ("interpret") is planned but not yet implemented.
  */
-import type { AgentToolMode, AgentInstance, AvailableModel, LogLine, V3 } from "stagehand-v3";
+import type { AvailableModel, LogLine } from "stagehand-v3";
+import type { Page, Stagehand } from "@browserbasehq/stagehand";
 import type {
   CorePageHandle,
   CoreSession,
@@ -35,6 +36,13 @@ export interface TaskMeta {
 export interface BenchTaskMeta extends TaskMeta {
   /** Override the default model list for this specific task. */
   models?: string[];
+  /**
+   * Custom system prompt passed to the Stagehand instance at init time
+   * (v3 `V3Options.systemPrompt` / v4 `StagehandInitParams.systemPrompt`).
+   * Used by tasks that test instruction-following (e.g. combination/instructions).
+   * Additional instructions applied when initializing Stagehand for this task. 
+   */
+  systemPrompt?: string;
 }
 
 /** Context provided to core (tier 1) tasks. */
@@ -128,7 +136,7 @@ export interface TaskDefinition {
   /** User-provided metadata. */
   meta: TaskMeta | BenchTaskMeta;
   /** The task function. */
-  fn: (ctx: CoreTaskContext | BenchTaskContext) => Promise<void | TaskResult>;
+  fn: (ctx: CoreTaskContext | BenchTaskContext | BenchV4TaskContext) => Promise<void | TaskResult>;
   /** Which tier this task was defined for (set during discovery from directory). */
   tier?: Tier;
 }
@@ -162,3 +170,33 @@ export interface TaskRegistry {
   /** Lookup by category. */
   byCategory: Map<string, DiscoveredTask[]>;
 }
+
+/**
+ * Context type for bench tasks ported to the V4Stagehand v4 SDK.
+ *
+ * Deliberately mirrors BenchTaskContext (types.ts) so per-task diffs between
+ * the v3 and v4 suites stay 1:1, with the v3 surface swapped for the v4 one:
+ * `v3` → `stagehand`, Playwright `page` → v4 `V4Page`. Keeping `page` typed as
+ * the v4 V4Page makes any Playwright API usage in a ported task a type error.
+ */
+
+export interface BenchV4TaskContext {
+  /** V4Stagehand v4 client instance. */
+  stagehand: V4Stagehand;
+  /** v4 page object (RPC-backed — url()/title() are async). */
+  page: V4Page;
+  /** Eval logger. Note: the v4 SDK itself logs to the console, not here. */
+  logger: EvalLogger;
+  /** Full eval input (name, modelName, params). */
+  input: {
+    name: string;
+    modelName: AvailableModel;
+    params?: Record<string, unknown>;
+  };
+  /** Model used for this run. */
+  modelName: AvailableModel;
+  /** Debug URL (unavailable from the v4 SDK — always empty for now). */
+  debugUrl: string;
+  /** Session URL (Browserbase; constructed from the session ID). */
+  sessionUrl: string;
+}

packages/evals/tasks/bench/act/amazon_add_to_cart.ts

@@ -1,17 +1,16 @@
-import { defineBenchTask } from "../../../framework/defineTask.js";
+import { defineBenchV4Task } from "../../../framework/defineTask.js";
 
-export default defineBenchTask(
+export default defineBenchV4Task(
   { name: "amazon_add_to_cart" },
-  async ({ logger, debugUrl, sessionUrl, v3 }) => {
+  async ({ logger, debugUrl, sessionUrl, stagehand, page }) => {
     try {
-      const page = v3.context.pages()[0];
       await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/amazon/");
 
-      await v3.act("click the 'Add to Cart' button");
+      await stagehand.act("click the 'Add to Cart' button");
 
-      await v3.act("click the 'Proceed to checkout' button");
+      await stagehand.act("click the 'Proceed to checkout' button");
 
-      const currentUrl = page.url();
+      const currentUrl = await page.url();
       const expectedUrl =
         "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/sign-in.html";
 
@@ -27,13 +26,13 @@ export default defineBenchTask(
     } catch (error) {
       return {
         _success: false,
-        error: error,
+        error: error instanceof Error ? error.message : String(error),
         debugUrl,
         sessionUrl,
         logs: logger.getLogs(),
       };
     } finally {
-      await v3.close();
+      await stagehand.close();
     }
   },
 );

packages/evals/tasks/bench/act/bidnet.ts

@@ -1,32 +1,34 @@
-import { defineBenchTask } from "../../../framework/defineTask.js";
+import { defineBenchV4Task } from "../../../framework/defineTask.js";
 
-export default defineBenchTask({ name: "bidnet" }, async ({ logger, debugUrl, sessionUrl, v3 }) => {
-  try {
-    const page = v3.context.pages()[0];
-    await page.goto("https://www.bidnetdirect.com/");
+export default defineBenchV4Task(
+  { name: "bidnet" },
+  async ({ logger, debugUrl, sessionUrl, stagehand, page }) => {
+    try {
+      await page.goto("https://www.bidnetdirect.com/");
 
-    await v3.act('Click on the "Construction" keyword');
+      await stagehand.act('Click on the "Construction" keyword');
 
-    const expectedUrl =
-      "https://www.bidnetdirect.com/public/solicitations/open?keywords=Construction";
-    const currentUrl = page.url();
+      const expectedUrl =
+        "https://www.bidnetdirect.com/public/solicitations/open?keywords=Construction";
+      const currentUrl = await page.url();
 
-    return {
-      _success: currentUrl.startsWith(expectedUrl),
-      currentUrl,
-      debugUrl,
-      sessionUrl,
-      logs: logger.getLogs(),
-    };
-  } catch (error) {
-    return {
-      _success: false,
-      error: error,
-      debugUrl,
-      sessionUrl,
-      logs: logger.getLogs(),
-    };
-  } finally {
-    await v3.close();
-  }
-});
+      return {
+        _success: currentUrl.startsWith(expectedUrl),
+        currentUrl,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    } catch (error) {
+      return {
+        _success: false,
+        error: error instanceof Error ? error.message : String(error),
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    } finally {
+      await stagehand.close();
+    }
+  },
+);

packages/evals/tasks/bench/act/checkboxes.ts

@@ -1,15 +1,14 @@
-import { defineBenchTask } from "../../../framework/defineTask.js";
+import { defineBenchV4Task } from "../../../framework/defineTask.js";
 
-export default defineBenchTask(
+export default defineBenchV4Task(
   { name: "checkboxes" },
-  async ({ debugUrl, sessionUrl, v3, logger }) => {
+  async ({ debugUrl, sessionUrl, stagehand, page, logger }) => {
     try {
-      const page = v3.context.pages()[0];
       await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/checkboxes/");
 
-      await v3.act("click the 'baseball' option");
+      await stagehand.act("click the 'baseball' option");
 
-      await v3.act("click the 'netball' option");
+      await stagehand.act("click the 'netball' option");
 
       const baseballChecked = await page
         .locator('input[type="checkbox"][name="sports"][value="baseball"]')
@@ -34,7 +33,7 @@ export default defineBenchTask(
         logs: logger.getLogs(),
       };
     } finally {
-      await v3.close();
+      await stagehand.close();
     }
   },
 );

packages/evals/tasks/bench/act/csr_in_oopif.ts

@@ -1,21 +1,27 @@
-import { defineBenchTask } from "../../../framework/defineTask.js";
+import { z } from "zod";
+import { defineBenchV4Task } from "../../../framework/defineTask.js";
 
-export default defineBenchTask(
+export default defineBenchV4Task(
   { name: "csr_in_oopif" },
-  async ({ debugUrl, sessionUrl, v3, logger }) => {
+  async ({ debugUrl, sessionUrl, stagehand, page, logger }) => {
     // this eval is designed to test whether stagehand can successfully
     // click inside an CSR (closed mode shadow) root that is inside an
     // OOPIF (out of process iframe)
 
     try {
-      const page = v3.context.pages()[0];
       await page.goto(
         "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-root-in-oopif/",
       );
-      await v3.act("click the button");
+      await stagehand.act("click the button");
 
       await new Promise((resolve) => setTimeout(resolve, 1000));
-      const extraction = await v3.extract("extract the entire page text");
+
+      // v3 used schemaless extract; v4 requires a schema.
+      // Single-word key to stay clear of the snake_case wire-casing bug (#14).
+      const { data: extraction } = await stagehand.extract(
+        "extract the entire page text",
+        z.object({ extraction: z.string() }),
+      );
 
       const pageText = extraction.extraction;
 
@@ -38,13 +44,13 @@ export default defineBenchTask(
     } catch (error) {
       return {
         _success: false,
-        message: `error: ${error.message}`,
+        error: error instanceof Error ? error.message : String(error),
         debugUrl,
         sessionUrl,
         logs: logger.getLogs(),
       };
     } finally {
-      await v3.close();
+      await stagehand.close();
     }
   },
 );

packages/evals/tasks/bench/act/csr_in_spif.ts

@@ -1,20 +1,25 @@
-import { defineBenchTask } from "../../../framework/defineTask.js";
+import { z } from "zod";
+import { defineBenchV4Task } from "../../../framework/defineTask.js";
 
-export default defineBenchTask(
+export default defineBenchV4Task(
   { name: "csr_in_spif" },
-  async ({ debugUrl, sessionUrl, v3, logger }) => {
+  async ({ debugUrl, sessionUrl, stagehand, page, logger }) => {
     // this eval is designed to test whether stagehand can successfully
     // click inside an CSR (closed mode shadow) root that is inside an
     // SPIF (same process iframe)
 
     try {
-      const page = v3.context.pages()[0];
       await page.goto(
         "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-dom-in-spif/",
       );
-      await v3.act("click the button");
+      await stagehand.act("click the button");
 
-      const extraction = await v3.extract("extract the entire page text");
+      // v3 used schemaless extract; v4 requires a schema.
+      // Single-word key to stay clear of the snake_case wire-casing bug (#14).
+      const { data: extraction } = await stagehand.extract(
+        "extract the entire page text",
+        z.object({ extraction: z.string() }),
+      );
 
       const pageText = extraction.extraction;
 
@@ -37,13 +42,13 @@ export default defineBenchTask(
     } catch (error) {
       return {
         _success: false,
-        message: `error: ${error.message}`,
+        error: error instanceof Error ? error.message : String(error),
         debugUrl,
         sessionUrl,
         logs: logger.getLogs(),
       };
     } finally {
-      await v3.close();
+      await stagehand.close();
     }
   },
 );

packages/evals/tasks/bench/act/custom_dropdown.ts

@@ -1,8 +1,8 @@
-import { defineBenchTask } from "../../../framework/defineTask.js";
+import { defineBenchV4Task } from "../../../framework/defineTask.js";
 
-export default defineBenchTask(
+export default defineBenchV4Task(
   { name: "custom_dropdown" },
-  async ({ debugUrl, sessionUrl, v3, logger }) => {
+  async ({ debugUrl, sessionUrl, stagehand, page, logger }) => {
     /**
      * This eval is meant to test whether we do not incorrectly attempt
      * the selectOptionFromDropdown method (defined in actHandlerUtils.ts) on a
@@ -13,14 +13,12 @@ export default defineBenchTask(
      */
 
     try {
-      const page = v3.context.pages()[0];
       await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/expand-dropdown/");
 
-      await v3.act("choose Canada from the 'Select a Country' dropdown");
+      await stagehand.act("choose Canada from the 'Select a Country' dropdown");
 
-      // to test, we'll grab the full a11y tree, and make sure it contains 'Canada'
-      const extraction = await v3.extract();
-      const fullTree = extraction.pageText;
+      // read the rendered page text directly — no second model call
+      const fullTree = await page.locator("#chosenValue").innerText();
 
       if (fullTree.includes("Canada")) {
         return {
@@ -40,13 +38,13 @@ export default defineBenchTask(
     } catch (error) {
       return {
         _success: false,
-        message: `error attempting to select an option from the dropdown: ${error.message}`,
+        message: `error attempting to select an option from the dropdown: ${(error as Error).message}`,
         debugUrl,
         sessionUrl,
         logs: logger.getLogs(),
       };
     } finally {
-      await v3.close();
+      await stagehand.close();
     }
   },
 );