fix(plan): address eligibility, benchmark state and desktop review findings

Author: Sg312

apps/desktop/src/main/computer-use/native-client.test.ts

@@ -62,6 +62,18 @@ describe('native computer use transport', () => {
     expect(results).toEqual([JSON.parse(status), JSON.parse(status)])
   })
 
+  it('accepts a maximum-sized frame followed by another valid frame', async () => {
+    const { client, reset } = helper(`let first; ${reader}
+      if (!first) { first = request; return; }
+      const a = JSON.stringify({id:first.id,result:${status}});
+      const b = JSON.stringify({id:request.id,result:${status}});
+      process.stdout.write(a.padEnd(16 * 1024 * 1024, ' ') + '\\n' + b + '\\n');
+    });`)
+    const results = await Promise.all([client.request('status', {}), client.request('status', {})])
+    expect(results).toEqual([JSON.parse(status), JSON.parse(status)])
+    expect(reset).not.toHaveBeenCalled()
+  })
+
   it('rejects malformed output and can start a fresh helper afterward', async () => {
     const { client, reset } = helper(`${reader}
       process.stdout.write(request.method==='bad'?'not json\\n':JSON.stringify({id:request.id,result:${status}})+'\\n');

apps/desktop/src/main/computer-use/native-client.ts

@@ -66,12 +66,12 @@ export class NativeComputerUseClient implements ComputerUseNativeClient {
     child.stdout.on('data', (chunk: Buffer) => {
       if (this.child !== child) return
       this.buffer = Buffer.concat([this.buffer, chunk])
-      if (this.buffer.length > MAX_FRAME_BYTES) {
-        this.stopWithError(new Error('Computer Use returned an oversized response.'))
-        return
-      }
       let newline = this.buffer.indexOf(10)
       while (newline >= 0) {
+        if (newline > MAX_FRAME_BYTES) {
+          this.stopWithError(new Error('Computer Use returned an oversized response.'))
+          return
+        }
         const line = this.buffer.subarray(0, newline).toString('utf8')
         this.buffer = this.buffer.subarray(newline + 1)
         try {
@@ -89,6 +89,8 @@ export class NativeComputerUseClient implements ComputerUseNativeClient {
         }
         newline = this.buffer.indexOf(10)
       }
+      if (this.buffer.length > MAX_FRAME_BYTES)
+        this.stopWithError(new Error('Computer Use returned an oversized response.'))
     })
     /** Native diagnostics must never copy app contents into application logs. */
     child.stderr.resume()

apps/desktop/src/main/index.ts

@@ -8,10 +8,10 @@ import {
   BrowserWindow,
   crashReporter,
   dialog,
+  globalShortcut,
   Notification,
   net,
   powerSaveBlocker,
-  globalShortcut,
   session,
   shell,
 } from 'electron'

apps/desktop/src/main/ipc.ts

@@ -9,9 +9,9 @@ import {
   isCurrentBrowserToolName,
 } from '@sim/browser-protocol'
 import {
-  type DesktopExecutorDevice,
   ComputerUseError,
   type ComputerUseToolFailure,
+  type DesktopExecutorDevice,
   type DesktopNotificationPayload,
   type DesktopServerChangeResult,
   type DesktopServerConfiguration,

apps/sim/app/api/copilot/confirm/route.test.ts

@@ -349,28 +349,36 @@ describe('Copilot Confirm API Route', () => {
     }
   )
 
-  it('completes a computer result only through its exact native claim', async () => {
-    getAsyncToolCall.mockResolvedValue({
-      ...existingRow,
-      toolName: 'computer',
-      claimedBy: 'desktop-computer',
-    })
-    const response = await POST(
-      createMockPostRequest({ toolCallId: 'tool-call-123', status: 'success', data: { ok: true } })
-    )
-    expect(response.status).toBe(200)
-    expect(completeClaimedAsyncToolCall).toHaveBeenCalledWith(
-      {
-        toolCallId: 'tool-call-123',
-        status: 'completed',
-        result: { __sealedClientToolCompletionV1: 'sealed-client-result' },
-        error: null,
-      },
-      'desktop-computer'
-    )
-    expect(completeAsyncToolCall).not.toHaveBeenCalled()
-    expect(publishToolConfirmation).toHaveBeenCalledOnce()
-  })
+  it.each([null, 'device-1'])(
+    'completes a native computer claim with background device %s',
+    async (desktopDeviceId) => {
+      getRunSegment.mockResolvedValue({ id: 'run-1', userId: 'user-1', desktopDeviceId })
+      getAsyncToolCall.mockResolvedValue({
+        ...existingRow,
+        toolName: 'computer',
+        claimedBy: 'desktop-computer',
+      })
+      const response = await POST(
+        createMockPostRequest({
+          toolCallId: 'tool-call-123',
+          status: 'success',
+          data: { ok: true },
+        })
+      )
+      expect(response.status).toBe(200)
+      expect(completeClaimedAsyncToolCall).toHaveBeenCalledWith(
+        {
+          toolCallId: 'tool-call-123',
+          status: 'completed',
+          result: { __sealedClientToolCompletionV1: 'sealed-client-result' },
+          error: null,
+        },
+        'desktop-computer'
+      )
+      expect(completeAsyncToolCall).not.toHaveBeenCalled()
+      expect(publishToolConfirmation).toHaveBeenCalledOnce()
+    }
+  )
 
   it('rejects background computer results without detaching the native claim', async () => {
     getAsyncToolCall.mockResolvedValue({

apps/sim/app/api/copilot/confirm/route.ts

@@ -12,6 +12,7 @@ import {
   type AsyncCompletionData,
   type AsyncConfirmationStatus,
   type AsyncTerminalStatus,
+  DESKTOP_TOOL_CLAIM_OWNER,
   getTerminalConfirmationStatus,
   isDeliveredAsyncStatus,
   isTerminalAsyncStatus,
@@ -42,7 +43,7 @@ import {
 import { isWorkflowToolName } from '@/lib/mothership/tools/client-executed-tools'
 import {
   getDesktopToolClaimOwner,
-  isDesktopToolCall,
+  isBackgroundDesktopToolCall,
   isNativeDesktopTool,
 } from '@/lib/mothership/tools/desktop-tools'
 import {
@@ -210,7 +211,10 @@ export const POST = withRouteHandler((req: NextRequest) => {
           return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
         }
 
-        if (run.desktopDeviceId && isDesktopToolCall(existing.toolName, toRecord(existing.args))) {
+        if (
+          run.desktopDeviceId &&
+          isBackgroundDesktopToolCall(existing.toolName, toRecord(existing.args))
+        ) {
           span.setAttribute(TraceAttr.CopilotConfirmOutcome, CopilotConfirmOutcome.Forbidden)
           return NextResponse.json(
             {
@@ -442,7 +446,12 @@ export const POST = withRouteHandler((req: NextRequest) => {
             ...(isPreclaimNativeTerminalOutcome
               ? { guard: { kind: 'pending' } as const }
               : existing.toolName === 'computer'
-                ? { guard: { kind: 'claimed', claimedBy: DESKTOP_TOOL_CLAIM_OWNER.computer } as const }
+                ? {
+                    guard: {
+                      kind: 'claimed',
+                      claimedBy: DESKTOP_TOOL_CLAIM_OWNER.computer,
+                    } as const,
+                  }
                 : {}),
           }
         )

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/route.ts

@@ -17,9 +17,7 @@ export const GET = defineInternalJsonRoute({
   contract: getBenchmarkContract,
   auth: internalSessionAuth,
   operation: benchmarkOperations.read,
-  rateLimit: internalRateLimits.none({
-    reason: 'The owner polls bounded benchmark artifacts under current source access',
-  }),
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-read' }),
   errorPolicy: internalOrchestrationErrorPolicy,
   beforeParse: async ({ principal }) => {
     await requireBenchmarkOperator(principal)

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-detail.tsx

@@ -40,7 +40,7 @@ interface BenchmarkEditorProps {
   busy: boolean
   saving: boolean
   stage: RunBenchmarkStageBody['stage'] | null
-  onUpdate: (body: UpdateBenchmarkBody) => void
+  onUpdate: (body: UpdateBenchmarkBody, onSaved: () => void) => void
   onRun: (stage: RunBenchmarkStageBody['stage'], runLabel?: string) => void
 }
 
@@ -53,7 +53,16 @@ function BenchmarkEditor({
   onUpdate,
   onRun,
 }: BenchmarkEditorProps) {
-  const [draft, setDraft] = useState(benchmark.artifacts)
+  const [edit, setEdit] = useState<{
+    version: number
+    artifacts: BenchmarkCase['artifacts']
+  } | null>(null)
+  const draft = edit?.artifacts ?? benchmark.artifacts
+  const changeDraft = (patch: Partial<BenchmarkCase['artifacts']>) =>
+    setEdit((current) => ({
+      version: current?.version ?? benchmark.version,
+      artifacts: { ...(current?.artifacts ?? benchmark.artifacts), ...patch },
+    }))
   const [runLabel, setRunLabel] = useState('')
   const { artifacts } = benchmark
   const referenceDirty =
@@ -76,31 +85,32 @@ function BenchmarkEditor({
 
   return (
     <div className='flex flex-col gap-8 divide-y divide-[var(--border)] [&>section+section]:pt-8'>
+      {edit && edit.version !== benchmark.version && (
+        <div role='alert' className='flex items-center gap-3 text-[var(--text-muted)] text-small'>
+          This benchmark changed on the server. Your edits are preserved; copy any text you need
+          before loading the latest version.
+          <Chip onClick={() => setEdit(null)}>Discard local edits</Chip>
+        </div>
+      )}
       <BenchmarkReference
         artifacts={draft}
         busy={busy}
         saving={saving}
         stage={stage}
         referenceDirty={referenceDirty}
         redactionDirty={redactionDirty}
-        onChange={(patch) => setDraft((current) => ({ ...current, ...patch }))}
+        onChange={changeDraft}
         onSave={() => {
           onUpdate(
-            referenceDirty
-              ? {
-                  version: benchmark.version,
-                  ...(draft.taskBrief !== artifacts.taskBrief
-                    ? { taskBrief: draft.taskBrief }
-                    : {}),
-                  ...(draft.referenceSpec !== artifacts.referenceSpec
-                    ? { referenceSpec: draft.referenceSpec }
-                    : {}),
-                }
-              : {
-                  version: benchmark.version,
-                  redactedSpec: draft.redactedSpec,
-                  blanks: draft.blanks,
-                }
+            {
+              version: edit?.version ?? benchmark.version,
+              ...(draft.taskBrief !== artifacts.taskBrief ? { taskBrief: draft.taskBrief } : {}),
+              ...(draft.referenceSpec !== artifacts.referenceSpec
+                ? { referenceSpec: draft.referenceSpec }
+                : {}),
+              ...(redactionDirty ? { redactedSpec: draft.redactedSpec, blanks: draft.blanks } : {}),
+            },
+            () => setEdit((current) => (current === edit ? null : current))
           )
         }}
         onRun={onRun}
@@ -122,8 +132,8 @@ function BenchmarkEditor({
       >
         {!canPlan && (
           <p className='text-[var(--text-muted)] text-small'>
-            Plan mode requires permission to create organization workspaces. An organization
-            administrator can update the selected user’s access.
+            The selected user needs Plan mode access and permission to create organization
+            workspaces.
           </p>
         )}
         {artifacts.generatedSpec ? (
@@ -310,15 +320,15 @@ export function BenchmarkDetail({
         <BenchmarkHistory organizationId={organizationId} benchmarkId={benchmarkId} />
       ) : (
         <BenchmarkEditor
-          key={`${benchmark.id}:${benchmark.version}`}
+          key={benchmark.id}
           benchmark={benchmark}
           canPlan={canPlan}
           busy={busy}
           saving={updateBenchmark.isPending}
           stage={stage}
-          onUpdate={(body) => {
+          onUpdate={(body, onSaved) => {
             runStage.reset()
-            updateBenchmark.mutate(body)
+            updateBenchmark.mutate(body, { onSuccess: onSaved })
           }}
           onRun={(nextStage, runLabel) => {
             updateBenchmark.reset()

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-history.tsx

@@ -133,10 +133,10 @@ export function BenchmarkHistory({ organizationId, benchmarkId }: BenchmarkHisto
           Loading saved results…
         </p>
       )}
-      {selected.data && (!baselineId || baseline.data) && (
+      {selected.data && (
         <BenchmarkRunComparison
           run={selected.data.run}
-          baseline={baseline.data?.run}
+          baseline={baseline.error ? undefined : baseline.data?.run}
           organizationId={organizationId}
         />
       )}