fix(plan): address eligibility, benchmark state and desktop review findings

Author: Sg312

apps/desktop/src/main/computer-use/native-client.test.ts

@@ -62,6 +62,18 @@ describe('native computer use transport', () => {
     expect(results).toEqual([JSON.parse(status), JSON.parse(status)])
   })
 
+  it('accepts a maximum-sized frame followed by another valid frame', async () => {
+    const { client, reset } = helper(`let first; ${reader}
+      if (!first) { first = request; return; }
+      const a = JSON.stringify({id:first.id,result:${status}});
+      const b = JSON.stringify({id:request.id,result:${status}});
+      process.stdout.write(a.padEnd(16 * 1024 * 1024, ' ') + '\\n' + b + '\\n');
+    });`)
+    const results = await Promise.all([client.request('status', {}), client.request('status', {})])
+    expect(results).toEqual([JSON.parse(status), JSON.parse(status)])
+    expect(reset).not.toHaveBeenCalled()
+  })
+
   it('rejects malformed output and can start a fresh helper afterward', async () => {
     const { client, reset } = helper(`${reader}
       process.stdout.write(request.method==='bad'?'not json\\n':JSON.stringify({id:request.id,result:${status}})+'\\n');

apps/desktop/src/main/computer-use/native-client.ts

@@ -66,12 +66,12 @@ export class NativeComputerUseClient implements ComputerUseNativeClient {
     child.stdout.on('data', (chunk: Buffer) => {
       if (this.child !== child) return
       this.buffer = Buffer.concat([this.buffer, chunk])
-      if (this.buffer.length > MAX_FRAME_BYTES) {
-        this.stopWithError(new Error('Computer Use returned an oversized response.'))
-        return
-      }
       let newline = this.buffer.indexOf(10)
       while (newline >= 0) {
+        if (newline > MAX_FRAME_BYTES) {
+          this.stopWithError(new Error('Computer Use returned an oversized response.'))
+          return
+        }
         const line = this.buffer.subarray(0, newline).toString('utf8')
         this.buffer = this.buffer.subarray(newline + 1)
         try {
@@ -89,6 +89,8 @@ export class NativeComputerUseClient implements ComputerUseNativeClient {
         }
         newline = this.buffer.indexOf(10)
       }
+      if (this.buffer.length > MAX_FRAME_BYTES)
+        this.stopWithError(new Error('Computer Use returned an oversized response.'))
     })
     /** Native diagnostics must never copy app contents into application logs. */
     child.stderr.resume()

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/route.ts

@@ -17,9 +17,7 @@ export const GET = defineInternalJsonRoute({
   contract: getBenchmarkContract,
   auth: internalSessionAuth,
   operation: benchmarkOperations.read,
-  rateLimit: internalRateLimits.none({
-    reason: 'The owner polls bounded benchmark artifacts under current source access',
-  }),
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-read' }),
   errorPolicy: internalOrchestrationErrorPolicy,
   beforeParse: async ({ principal }) => {
     await requireBenchmarkOperator(principal)

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-detail.tsx

@@ -40,7 +40,7 @@ interface BenchmarkEditorProps {
   busy: boolean
   saving: boolean
   stage: RunBenchmarkStageBody['stage'] | null
-  onUpdate: (body: UpdateBenchmarkBody) => void
+  onUpdate: (body: UpdateBenchmarkBody, onSaved: () => void) => void
   onRun: (stage: RunBenchmarkStageBody['stage'], runLabel?: string) => void
 }
 
@@ -53,7 +53,16 @@ function BenchmarkEditor({
   onUpdate,
   onRun,
 }: BenchmarkEditorProps) {
-  const [draft, setDraft] = useState(benchmark.artifacts)
+  const [edit, setEdit] = useState<{
+    version: number
+    artifacts: BenchmarkCase['artifacts']
+  } | null>(null)
+  const draft = edit?.artifacts ?? benchmark.artifacts
+  const changeDraft = (patch: Partial<BenchmarkCase['artifacts']>) =>
+    setEdit((current) => ({
+      version: current?.version ?? benchmark.version,
+      artifacts: { ...(current?.artifacts ?? benchmark.artifacts), ...patch },
+    }))
   const [runLabel, setRunLabel] = useState('')
   const { artifacts } = benchmark
   const referenceDirty =
@@ -76,31 +85,32 @@ function BenchmarkEditor({
 
   return (
     <div className='flex flex-col gap-8 divide-y divide-[var(--border)] [&>section+section]:pt-8'>
+      {edit && edit.version !== benchmark.version && (
+        <div role='alert' className='flex items-center gap-3 text-[var(--text-muted)] text-small'>
+          This benchmark changed on the server. Your edits are preserved; copy any text you need
+          before loading the latest version.
+          <Chip onClick={() => setEdit(null)}>Discard local edits</Chip>
+        </div>
+      )}
       <BenchmarkReference
         artifacts={draft}
         busy={busy}
         saving={saving}
         stage={stage}
         referenceDirty={referenceDirty}
         redactionDirty={redactionDirty}
-        onChange={(patch) => setDraft((current) => ({ ...current, ...patch }))}
+        onChange={changeDraft}
         onSave={() => {
           onUpdate(
-            referenceDirty
-              ? {
-                  version: benchmark.version,
-                  ...(draft.taskBrief !== artifacts.taskBrief
-                    ? { taskBrief: draft.taskBrief }
-                    : {}),
-                  ...(draft.referenceSpec !== artifacts.referenceSpec
-                    ? { referenceSpec: draft.referenceSpec }
-                    : {}),
-                }
-              : {
-                  version: benchmark.version,
-                  redactedSpec: draft.redactedSpec,
-                  blanks: draft.blanks,
-                }
+            {
+              version: edit?.version ?? benchmark.version,
+              ...(draft.taskBrief !== artifacts.taskBrief ? { taskBrief: draft.taskBrief } : {}),
+              ...(draft.referenceSpec !== artifacts.referenceSpec
+                ? { referenceSpec: draft.referenceSpec }
+                : {}),
+              ...(redactionDirty ? { redactedSpec: draft.redactedSpec, blanks: draft.blanks } : {}),
+            },
+            () => setEdit((current) => (current === edit ? null : current))
           )
         }}
         onRun={onRun}
@@ -122,8 +132,8 @@ function BenchmarkEditor({
       >
         {!canPlan && (
           <p className='text-[var(--text-muted)] text-small'>
-            Plan mode requires permission to create organization workspaces. An organization
-            administrator can update the selected user’s access.
+            The selected user needs Plan mode access and permission to create organization
+            workspaces.
           </p>
         )}
         {artifacts.generatedSpec ? (
@@ -310,15 +320,15 @@ export function BenchmarkDetail({
         <BenchmarkHistory organizationId={organizationId} benchmarkId={benchmarkId} />
       ) : (
         <BenchmarkEditor
-          key={`${benchmark.id}:${benchmark.version}`}
+          key={benchmark.id}
           benchmark={benchmark}
           canPlan={canPlan}
           busy={busy}
           saving={updateBenchmark.isPending}
           stage={stage}
-          onUpdate={(body) => {
+          onUpdate={(body, onSaved) => {
             runStage.reset()
-            updateBenchmark.mutate(body)
+            updateBenchmark.mutate(body, { onSuccess: onSaved })
           }}
           onRun={(nextStage, runLabel) => {
             updateBenchmark.reset()

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-history.tsx

@@ -133,10 +133,10 @@ export function BenchmarkHistory({ organizationId, benchmarkId }: BenchmarkHisto
           Loading saved results…
         </p>
       )}
-      {selected.data && (!baselineId || baseline.data) && (
+      {selected.data && (
         <BenchmarkRunComparison
           run={selected.data.run}
-          baseline={baseline.data?.run}
+          baseline={baseline.error ? undefined : baseline.data?.run}
           organizationId={organizationId}
         />
       )}

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-reference.tsx

@@ -1,8 +1,8 @@
 'use client'
 
 import { useRef, useState } from 'react'
-import { Chip, ChipInput, ChipTextarea, toast } from '@sim/emcn'
-import { Code, Plus, Trash, Upload } from '@sim/emcn/icons'
+import { Chip, ChipTextarea, toast } from '@sim/emcn'
+import { Code, Upload } from '@sim/emcn/icons'
 import { getErrorMessage } from '@sim/utils/errors'
 import type { BenchmarkCase, RunBenchmarkStageBody } from '@/lib/api/contracts/benchmarks'
 import { BenchmarkJson } from '@/app/o/[organizationId]/benchmark/components/benchmark-json'
@@ -119,7 +119,10 @@ export function BenchmarkReference({
           <Chip
             leftIcon={Code}
             disabled={busy || referenceDirty || !referenceSpec.trim()}
-            onClick={() => setJsonOpen(true)}
+            onClick={() => {
+              onChange({})
+              setJsonOpen(true)
+            }}
           >
             Edit JSON
           </Chip>
@@ -148,71 +151,28 @@ export function BenchmarkReference({
           <ChipTextarea
             aria-label='Redacted reference spec'
             value={redactedSpec}
-            onChange={(event) => onChange({ redactedSpec: event.target.value })}
+            readOnly
             rows={10}
             resizable
           />
           <p className='text-[var(--text-muted)] text-small'>
             Each marker uses [[BLANK:id]], and its expected answer must restore the exact original
             passage. Keep discoverable details and remove answers revealed elsewhere in the text.
           </p>
-          {blanks.map((blank, index) => (
-            <div key={index} className='flex items-start gap-2'>
-              <div className='grid min-w-0 flex-1 gap-2 sm:grid-cols-[140px_1fr]'>
-                <ChipInput
-                  aria-label={`Blank ${index + 1} ID`}
-                  value={blank.id}
-                  maxLength={64}
-                  onChange={(event) =>
-                    onChange({
-                      redactedSpec: redactedSpec.replaceAll(
-                        `[[BLANK:${blank.id}]]`,
-                        `[[BLANK:${event.target.value}]]`
-                      ),
-                      blanks: blanks.map((item, position) =>
-                        position === index ? { ...item, id: event.target.value } : item
-                      ),
-                    })
-                  }
-                />
-                <ChipTextarea
-                  aria-label={`Expected answer for blank ${index + 1}`}
-                  value={blank.answer}
-                  rows={2}
-                  resizable
-                  onChange={(event) =>
-                    onChange({
-                      blanks: blanks.map((item, position) =>
-                        position === index ? { ...item, answer: event.target.value } : item
-                      ),
-                    })
-                  }
-                />
+          <dl className='flex flex-col gap-3'>
+            {blanks.map((blank) => (
+              <div key={blank.id} className='flex flex-col gap-1'>
+                <dt className='text-[var(--text-body)] text-small'>{blank.id}</dt>
+                <dd className='whitespace-pre-wrap text-[var(--text-muted)] text-small'>
+                  {blank.answer}
+                </dd>
               </div>
-              <Chip
-                leftIcon={Trash}
-                aria-label={`Remove blank ${index + 1}`}
-                onClick={() =>
-                  onChange({
-                    redactedSpec: redactedSpec.replaceAll(`[[BLANK:${blank.id}]]`, blank.answer),
-                    blanks: blanks.filter((_, position) => position !== index),
-                  })
-                }
-              />
-            </div>
-          ))}
-          <div>
-            <Chip
-              leftIcon={Plus}
-              onClick={() => {
-                let suffix = blanks.length + 1
-                while (blanks.some((blank) => blank.id === `detail_${suffix}`)) suffix += 1
-                onChange({ blanks: [...blanks, { id: `detail_${suffix}`, answer: '' }] })
-              }}
-            >
-              Add blank
-            </Chip>
-          </div>
+            ))}
+          </dl>
+          <p className='text-[var(--text-muted)] text-small'>
+            Use Edit JSON to change markers and answers together. Changes are checked against the
+            original reference before applying.
+          </p>
         </fieldset>
       )}
       {dirty && (

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-results.tsx

@@ -44,12 +44,12 @@ export function BenchmarkResults({
                 </span>
               )}
               {!result && answer?.evidenceError && (
-                <div>
+                <dl className='text-small'>
                   <dt className='text-[var(--text-muted)]'>Evidence needs review</dt>
                   <dd className='mt-1 whitespace-pre-wrap break-words text-[var(--text-body)]'>
                     {answer.evidenceError}
                   </dd>
-                </div>
+                </dl>
               )}
             </div>
             <dl className='flex flex-col gap-3 text-small'>

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-step.tsx

@@ -28,7 +28,16 @@ export function BenchmarkStep({
             pending && 'text-[var(--text-muted)]'
           )}
         >
-          {pending ? <Loader aria-label='Running' className='size-[14px] animate-spin' /> : number}
+          {pending ? (
+            <>
+              <Loader animate className='size-[14px]' />
+              <span role='status' className='sr-only'>
+                Running {title}
+              </span>
+            </>
+          ) : (
+            number
+          )}
         </span>
         <div className='min-w-0 flex-1'>
           <h2 id={`benchmark-step-${number}`} className='text-[var(--text-primary)] text-base'>

apps/sim/app/o/[organizationId]/benchmark/components/create-benchmark.tsx

@@ -3,6 +3,7 @@
 import { useState } from 'react'
 import { Chip, ChipCombobox, ChipInput, ChipTextarea } from '@sim/emcn'
 import { useBenchmarkWorkspaces, useCreateBenchmark } from '@/hooks/queries/benchmarks'
+import { useDebounce } from '@/hooks/use-debounce'
 
 interface CreateBenchmarkProps {
   organizationId: string
@@ -11,14 +12,26 @@ interface CreateBenchmarkProps {
 }
 
 export function CreateBenchmark({ organizationId, runAsUserId, onCreated }: CreateBenchmarkProps) {
-  const [sourceWorkspaceId, setSourceWorkspaceId] = useState('')
+  const [sourceWorkspace, setSourceWorkspace] = useState<{ value: string; label: string } | null>(
+    null
+  )
+  const [workspaceSearch, setWorkspaceSearch] = useState('')
   const [name, setName] = useState('')
   const [taskBrief, setTaskBrief] = useState('')
-  const workspaces = useBenchmarkWorkspaces(organizationId, runAsUserId)
+  const sourceWorkspaceId = sourceWorkspace?.value ?? ''
+  const workspaces = useBenchmarkWorkspaces(
+    organizationId,
+    runAsUserId,
+    useDebounce(workspaceSearch, 250)
+  )
   const createBenchmark = useCreateBenchmark(organizationId)
-  const options = (workspaces.data?.pages.flatMap((page) => page.workspaces) ?? []).map(
+  const loadedOptions = (workspaces.data?.pages.flatMap((page) => page.workspaces) ?? []).map(
     (workspace) => ({ value: workspace.id, label: workspace.name })
   )
+  const options =
+    sourceWorkspace && !loadedOptions.some((option) => option.value === sourceWorkspace.value)
+      ? [sourceWorkspace, ...loadedOptions]
+      : loadedOptions
 
   return (
     <form
@@ -41,9 +54,12 @@ export function CreateBenchmark({ organizationId, runAsUserId, onCreated }: Crea
           aria-label='Source workspace'
           options={options}
           value={sourceWorkspaceId}
-          onChange={setSourceWorkspaceId}
+          onChange={(value) =>
+            setSourceWorkspace(options.find((option) => option.value === value) ?? null)
+          }
           placeholder='Select a workspace'
           searchable
+          onSearchChange={setWorkspaceSearch}
           isLoading={workspaces.isLoading}
           error={workspaces.error?.message}
           disabled={createBenchmark.isPending}

apps/sim/app/o/[organizationId]/components/organization-sidebar/organization-sidebar.tsx

@@ -102,7 +102,7 @@ export const OrganizationSidebar = memo(function OrganizationSidebar({
     organization.id,
     searchAccess.memberScoped,
     canUseHome,
-    benchmarkEnabled && mothershipAvailable
+    benchmarkEnabled
   )
   const settingsPath = organizationRoutes(organization.id).settings
   const isSettings = pathname === settingsPath || pathname?.startsWith(`${settingsPath}/`)