Validate observed Action target identity before act

packages/protocol/schemas.ts

@@ -1001,6 +1001,20 @@ export const LocalBrowserLaunchOptionsSchema = z
   .meta({ id: "LocalBrowserLaunchOptions" });
 
 /** Action object returned by observe and used by act */
+export const ActionTargetSchema = z
+  .strictObject({
+    frameOrdinal: z.number().int().nonnegative().meta({
+      description: "Stable frame ordinal assigned for the lifetime of the page",
+    }),
+    backendNodeId: z.number().int().positive().meta({
+      description: "CDP backend node ID captured when the action was observed",
+    }),
+  })
+  .meta({
+    id: "ActionTarget",
+    description: "Short-lived identity of the DOM node selected by an observed action",
+  });
+
 export const ActionSchema = z
   .strictObject({
     selector: z.string().meta({
@@ -1022,6 +1036,14 @@ export const ActionSchema = z
         description: "Arguments to pass to the method",
         example: ["Hello World"],
       }),
+    target: ActionTargetSchema.optional().meta({
+      description:
+        "Optional observed target identity. When present, act validates it before mutating the page.",
+    }),
+    argumentTargets: z.record(z.string().regex(/^\d+$/), ActionTargetSchema).optional().meta({
+      description:
+        "Optional observed identities for selector arguments, keyed by their argument index.",
+    }),
   })
   .meta({
     id: "Action",

packages/protocol/stagehand.v4.json

@@ -1665,12 +1665,47 @@
           "items": {
             "type": "string"
           }
+        },
+        "target": {
+          "description": "Optional observed target identity. When present, act validates it before mutating the page.",
+          "$ref": "#/$defs/ActionTarget"
+        },
+        "argument_targets": {
+          "description": "Optional observed identities for selector arguments, keyed by their argument index.",
+          "type": "object",
+          "propertyNames": {
+            "type": "string",
+            "pattern": "^\\d+$"
+          },
+          "additionalProperties": {
+            "$ref": "#/$defs/ActionTarget"
+          }
         }
       },
       "required": ["selector", "description"],
       "additionalProperties": false,
       "description": "Action object returned by observe and used by act"
     },
+    "ActionTarget": {
+      "type": "object",
+      "properties": {
+        "frame_ordinal": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991,
+          "description": "Stable frame ordinal assigned for the lifetime of the page"
+        },
+        "backend_node_id": {
+          "type": "integer",
+          "exclusiveMinimum": 0,
+          "maximum": 9007199254740991,
+          "description": "CDP backend node ID captured when the action was observed"
+        }
+      },
+      "required": ["frame_ordinal", "backend_node_id"],
+      "additionalProperties": false,
+      "description": "Short-lived identity of the DOM node selected by an observed action"
+    },
     "ActOptions": {
       "type": "object",
       "properties": {

packages/protocol/tests/protocol/object-model-protocol.test.ts

@@ -144,6 +144,36 @@ describe("Stagehand object-model protocol", () => {
     });
   });
 
+  it("accepts observed Action targets and rejects invalid identities", () => {
+    const action = {
+      selector: "xpath=/html/body/button",
+      description: "Submit button",
+      method: "click",
+      arguments: [],
+      target: { frameOrdinal: 0, backendNodeId: 12 },
+      argumentTargets: { "0": { frameOrdinal: 0, backendNodeId: 20 } },
+    };
+
+    expect(
+      StagehandMethods.stagehandAct.params.parse({ pageId: "target-1", input: action }),
+    ).toStrictEqual({ pageId: "target-1", input: action });
+    expect(() =>
+      StagehandMethods.stagehandAct.params.parse({
+        pageId: "target-1",
+        input: { ...action, target: { frameOrdinal: -1, backendNodeId: 0 } },
+      }),
+    ).toThrow();
+    expect(() =>
+      StagehandMethods.stagehandAct.params.parse({
+        pageId: "target-1",
+        input: {
+          ...action,
+          argumentTargets: { destination: { frameOrdinal: 0, backendNodeId: 20 } },
+        },
+      }),
+    ).toThrow();
+  });
+
   it("defines extraction with a page, instruction, JSON Schema, and optional call settings", () => {
     const params = StagehandMethods.stagehandExtract.params.parse({
       pageId: "target-1",

packages/protocol/types.ts

@@ -7,6 +7,7 @@ import type {
 } from "./schema-registry.js";
 import type {
   ActionSchema,
+  ActionTargetSchema,
   ActOptionsSchema,
   ActResultDataSchema,
   ActResultSchema,
@@ -196,7 +197,6 @@ import type {
   VertexModelProviderOptionsSchema,
   VertexProviderOptionsSchema,
 } from "./schemas.js";
-
 export type VariablePrimitive = z.infer<typeof VariablePrimitiveSchema>;
 export type VariableValue = z.infer<typeof VariableValueSchema>;
 export type Variables = z.infer<typeof VariablesSchema>;
@@ -251,6 +251,7 @@ export type LLMGenerateParams = z.infer<typeof LLMGenerateParamsSchema>;
 export type LLMGenerateResult = z.infer<typeof LLMGenerateResultSchema>;
 export type ClientModelReference = z.infer<typeof ClientModelReferenceSchema>;
 export type Action = z.infer<typeof ActionSchema>;
+export type ActionTarget = z.infer<typeof ActionTargetSchema>;
 export type ActOptions = z.infer<typeof ActOptionsSchema>;
 export type ActResultData = z.infer<typeof ActResultDataSchema>;
 export type ActResult = z.infer<typeof ActResultSchema>;

packages/sdk-python/src/stagehand/__init__.py

@@ -2,6 +2,7 @@
 
 from ._generated.models import (
     Action,
+    ActionTarget,
     ActResultData,
     Animations,
     BrowserbaseBrowserSettings,
@@ -55,6 +56,7 @@
 __all__ = [
     "ActResultData",
     "Action",
+    "ActionTarget",
     "Animations",
     "BrowserGetVersionResult",
     "BrowserClipboard",

packages/sdk-python/src/stagehand/_generated/models.py

@@ -14,6 +14,7 @@
     StrictFloat,
     StrictInt,
     StrictStr,
+    constr,
 )
 from stagehand._validation import (
     PageScreenshotOptionsValidation,
@@ -129,6 +130,25 @@ class Action(WireModel):
 
     Example: ['Hello World']
     """
+    target: Optional[ActionTarget] = None
+    """Optional observed target identity. When present, act validates it before mutating the page."""
+    argument_targets: Optional[dict[constr(pattern=r"^\d+$", strict=True), ActionTarget]] = (
+        None
+    )
+    """Optional observed identities for selector arguments, keyed by their argument index."""
+
+
+class ActionTarget(WireModel):
+    """Short-lived identity of the DOM node selected by an observed action"""
+
+    model_config = ConfigDict(
+        extra="forbid",
+        validate_by_name=True,
+    )
+    frame_ordinal: Annotated[StrictInt, Field(ge=0, le=9007199254740991)]
+    """Stable frame ordinal assigned for the lifetime of the page"""
+    backend_node_id: Annotated[StrictInt, Field(gt=0, le=9007199254740991)]
+    """CDP backend node ID captured when the action was observed"""
 
 
 class Animations(StrEnum):

packages/sdk-python/tests/test_stagehand.py

@@ -8,7 +8,14 @@
 import pytest
 from pydantic import BaseModel
 
-from stagehand import LLMGenerateInput, LLMGenerateOutput, Page, ProtocolLocator, Stagehand
+from stagehand import (
+    ActionTarget,
+    LLMGenerateInput,
+    LLMGenerateOutput,
+    Page,
+    ProtocolLocator,
+    Stagehand,
+)
 from stagehand._generated.models import (
     Action,
     ActResult,
@@ -243,7 +250,12 @@ async def connect(**_: object) -> RPCClient:
 async def test_stagehand_ai_methods_resolve_pages_and_validate_results(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    action = Action(selector="a", description="More information")
+    action = Action(
+        selector="a",
+        description="More information",
+        target=ActionTarget(frame_ordinal=0, backend_node_id=12),
+        argument_targets={"0": ActionTarget(frame_ordinal=0, backend_node_id=20)},
+    )
     act_result = ActResultData(
         success=True,
         message="Clicked the link",

packages/sdk-ts/src/index.ts

@@ -16,6 +16,7 @@ export type { InitScriptSource } from "./pageScripts.js";
 export { Stagehand } from "./stagehand.js";
 export type {
   Action,
+  ActionTarget,
   ActResultData,
   BrowserGetVersionResult,
   RuntimeLoopbackStatusResult,

packages/sdk-ts/tests/object-wrapper.test.ts

@@ -757,6 +757,8 @@ describe("Stagehand TS object wrapper", () => {
       description: "Submit button",
       method: "click",
       arguments: [],
+      target: { frameOrdinal: 0, backendNodeId: 12 },
+      argumentTargets: { "0": { frameOrdinal: 0, backendNodeId: 20 } },
     };
     client.queueResponse(StagehandMethods.stagehandObserve, {
       result: [observedAction],

packages/server/actionTarget.ts

@@ -0,0 +1,88 @@
+import type { Action, ActionTarget } from "../protocol/types.js";
+import { CachedActionRebindError } from "./errors.js";
+import type { EncodedId } from "./types/private/internal.js";
+import { trimTrailingTextNode } from "./utils.js";
+
+export class ActionTargetMismatchError extends Error {
+  constructor(
+    readonly expected: ActionTarget,
+    readonly actual: ActionTarget,
+  ) {
+    super(
+      `Observed action target changed: expected frame ${expected.frameOrdinal} node ${expected.backendNodeId}, ` +
+        `resolved frame ${actual.frameOrdinal} node ${actual.backendNodeId}`,
+    );
+    this.name = "ActionTargetMismatchError";
+  }
+}
+
+export function actionTargetFromEncodedId(elementId: EncodedId): ActionTarget {
+  const [frameOrdinal, backendNodeId] = elementId.split("-").map(Number);
+  return { frameOrdinal: frameOrdinal!, backendNodeId: backendNodeId! };
+}
+
+export function normalizeSnapshotRootXPath(xpath: string): string {
+  const trimmed = xpath.trim();
+  return trimmed === "/" ? "/html[1]" : trimmed;
+}
+
+/**
+ * Return the identity of the DOM element addressed by an emitted XPath.
+ * Snapshot inference may select a text node whose XPath is normalized to its
+ * parent element, so the selected encoded ID is not always the actionable ID.
+ */
+export function actionTargetForSnapshotSelector(
+  selector: string,
+  xpathMap: Record<string, string>,
+): ActionTarget | undefined {
+  const xpath = normalizeSnapshotRootXPath(selector.replace(/^xpath=/iu, ""));
+  const match =
+    Object.entries(xpathMap).find(([, candidateXpath]) => candidateXpath === xpath) ??
+    (xpath === "/html[1]"
+      ? Object.entries(xpathMap).find(([, candidateXpath]) => candidateXpath === "/html")
+      : undefined);
+  return match ? actionTargetFromEncodedId(match[0] as EncodedId) : undefined;
+}
+
+export function selectorAndTargetForSnapshotElement(
+  elementId: EncodedId,
+  xpathMap: Record<string, string>,
+): { selector: string; target?: ActionTarget } | undefined {
+  const rawXpath = trimTrailingTextNode(xpathMap[elementId]);
+  if (!rawXpath) return undefined;
+
+  const xpath = normalizeSnapshotRootXPath(rawXpath);
+  const selector = `xpath=${xpath}`;
+  const target = actionTargetForSnapshotSelector(selector, xpathMap);
+  return { selector, ...(target ? { target } : {}) };
+}
+
+/**
+ * Refresh short-lived target identities from the current snapshot while keeping
+ * cached selectors. Throws when a selector no longer resolves.
+ */
+export function rebindActionTargetsToSnapshot(
+  actions: Action[],
+  xpathMap: Record<string, string>,
+): Action[] {
+  return actions.map((action) => {
+    const target = actionTargetForSnapshotSelector(action.selector, xpathMap);
+    if (!target) {
+      throw new CachedActionRebindError("selector");
+    }
+
+    const destinationSelector = action.method === "dragAndDrop" ? action.arguments?.[0] : undefined;
+    const destinationTarget = destinationSelector
+      ? actionTargetForSnapshotSelector(destinationSelector, xpathMap)
+      : undefined;
+    if (destinationSelector && !destinationTarget) {
+      throw new CachedActionRebindError("argument");
+    }
+
+    return {
+      ...action,
+      target,
+      ...(destinationTarget ? { argumentTargets: { "0": destinationTarget } } : {}),
+    };
+  });
+}

packages/server/errors.ts

@@ -4,3 +4,14 @@ export class TimeoutError extends Error {
     this.name = "TimeoutError";
   }
 }
+
+export class CachedActionRebindError extends Error {
+  constructor(readonly kind: "selector" | "argument" = "selector") {
+    super(
+      kind === "argument"
+        ? "Cached action argument no longer resolves in the current page snapshot"
+        : "Cached action no longer resolves in the current page snapshot",
+    );
+    this.name = "CachedActionRebindError";
+  }
+}