fix(voice): hold interrupted speech until recognition resolves

src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts

@@ -810,6 +810,89 @@ describe("native assistant speech stream", () => {
     }
   });
 
+  it("holds an interruption past VAD idle until delayed final transcript arrives", async () => {
+    vi.useFakeTimers();
+    try {
+      startNativeAssistantSpeech("session-1", vi.fn());
+      useChatStore
+        .getState()
+        .setMessages("session-1", [
+          assistant([{ type: "text", text: "Interrupted reply." }]),
+        ]);
+      await vi.runAllTimersAsync();
+      await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled());
+      const firstStreamId = mocks.start.mock.calls[0]?.[0] as string;
+      emit("started");
+
+      useVoiceConversationStore.setState({ userSpeaking: true });
+      await vi.runAllTimersAsync();
+      expect(mocks.stop).toHaveBeenCalled();
+      mocks.streamHandler?.({
+        streamId: firstStreamId,
+        state: "interrupted",
+        error: null,
+        delivery: { segments: [] },
+      });
+
+      useVoiceConversationStore.setState({ userSpeaking: false });
+      await vi.advanceTimersByTimeAsync(300);
+      expect(mocks.start).toHaveBeenCalledTimes(1);
+
+      finalizeVoiceTranscript("delayed-final");
+      useChatStore
+        .getState()
+        .setMessages("session-1", [
+          assistant(
+            [{ type: "text", text: "Interrupted reply." }],
+            "completed",
+            "assistant-1",
+          ),
+          voiceUser("delayed-final"),
+        ]);
+      await vi.runAllTimersAsync();
+
+      expect(mocks.start).toHaveBeenCalledTimes(1);
+      expect(takeVoicePlaybackNotices("session-1")).toContain(
+        "Original text: Interrupted reply.",
+      );
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("resumes a no-result interruption after the recognition segment timeout", async () => {
+    vi.useFakeTimers();
+    try {
+      startNativeAssistantSpeech("session-1", vi.fn());
+      useChatStore
+        .getState()
+        .setMessages("session-1", [
+          assistant(
+            [{ type: "text", text: "False alarm reply." }],
+            "completed",
+          ),
+        ]);
+      useVoiceConversationStore.setState({ userSpeaking: true });
+      await vi.runAllTimersAsync();
+      expect(mocks.start).not.toHaveBeenCalled();
+
+      useVoiceConversationStore.setState({ userSpeaking: false });
+      await vi.advanceTimersByTimeAsync(300);
+      expect(mocks.start).not.toHaveBeenCalled();
+
+      await vi.advanceTimersByTimeAsync(200);
+      await vi.runAllTimersAsync();
+      expect(mocks.start).toHaveBeenCalledTimes(1);
+      expect(mocks.append).toHaveBeenCalledWith(
+        mocks.start.mock.calls[0]?.[0],
+        "False alarm reply.",
+      );
+      expect(takeVoicePlaybackNotices("session-1")).toBeNull();
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   it("ignores a late started event after interruption is requested", async () => {
     let resolveStop: ((stopped: boolean) => void) | undefined;
     startNativeAssistantSpeech("session-1", vi.fn());
@@ -2431,7 +2514,7 @@ describe("native assistant speech stream", () => {
         },
       });
       useVoiceConversationStore.setState({ userSpeaking: false });
-      await vi.advanceTimersByTimeAsync(250);
+      await vi.advanceTimersByTimeAsync(500);
       await vi.runAllTimersAsync();
 
       const secondStreamId = mocks.start.mock.calls[1]?.[0] as string;
@@ -2467,7 +2550,7 @@ describe("native assistant speech stream", () => {
         },
       });
       useVoiceConversationStore.setState({ userSpeaking: false });
-      await vi.advanceTimersByTimeAsync(250);
+      await vi.advanceTimersByTimeAsync(500);
       await vi.runAllTimersAsync();
 
       const thirdStreamId = mocks.start.mock.calls[2]?.[0] as string;
@@ -2535,7 +2618,7 @@ describe("native assistant speech stream", () => {
         ]);
 
       useVoiceConversationStore.setState({ userSpeaking: false });
-      await vi.advanceTimersByTimeAsync(250);
+      await vi.advanceTimersByTimeAsync(500);
       await vi.runAllTimersAsync();
 
       const resumedStreamId = mocks.start.mock.calls[1]?.[0] as string;
@@ -2681,7 +2764,7 @@ describe("native assistant speech stream", () => {
         refreshedSiriVoice,
       );
       useVoiceConversationStore.setState({ userSpeaking: false });
-      await vi.advanceTimersByTimeAsync(250);
+      await vi.advanceTimersByTimeAsync(500);
       await vi.runAllTimersAsync();
 
       expect(mocks.siriStart).toHaveBeenCalledTimes(2);
@@ -2874,6 +2957,43 @@ describe("native assistant speech stream", () => {
     });
   });
 
+  it("preserves the recognition deadline across repeated VAD edges", async () => {
+    vi.useFakeTimers();
+    try {
+      startNativeAssistantSpeech("session-1", vi.fn());
+      useChatStore
+        .getState()
+        .setMessages("session-1", [
+          assistant([{ type: "text", text: "Interrupted reply." }]),
+        ]);
+      await vi.runAllTimersAsync();
+      const firstStreamId = mocks.start.mock.calls[0]?.[0] as string;
+
+      useVoiceConversationStore.setState({ userSpeaking: true });
+      mocks.streamHandler?.({
+        streamId: firstStreamId,
+        state: "interrupted",
+        error: null,
+        delivery: { segments: [] },
+      });
+      useVoiceConversationStore.setState({ userSpeaking: false });
+
+      await vi.advanceTimersByTimeAsync(300);
+      expect(mocks.start).toHaveBeenCalledTimes(1);
+
+      useVoiceConversationStore.setState({ userSpeaking: true });
+      useVoiceConversationStore.setState({ userSpeaking: false });
+      await vi.advanceTimersByTimeAsync(199);
+      expect(mocks.start).toHaveBeenCalledTimes(1);
+
+      await vi.advanceTimersByTimeAsync(1);
+      expect(mocks.start).toHaveBeenCalledTimes(2);
+      await vi.runAllTimersAsync();
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   it("never starts a held reply when a newer finalized voice transcript arrives", async () => {
     useVoiceConversationStore.setState({ userSpeaking: true });
     startNativeAssistantSpeech("session-1", vi.fn());

src/features/voice-conversation/lib/nativeAssistantSpeech.ts

@@ -134,6 +134,7 @@ function boundedDeliveryText(
 }
 const MALFORMED_VOICE_TRANSCRIPT_KEY = "\0malformed-voice-transcript";
 const USER_IDLE_TRANSCRIPT_SETTLE_MS = 250;
+const USER_RECOGNITION_SEGMENT_TIMEOUT_MS = 500;
 
 function voiceTranscriptKeyForMessage(
   sessionId: string,
@@ -963,7 +964,8 @@ export function startNativeAssistantSpeech(
   let resumableInterruption: ResumableInterruption | null = null;
   let heldReleaseReady = false;
   let interruptionReleaseReady = false;
-  let idleSettling = false;
+  let pendingUserRecognitionSegment = false;
+  let recognitionSegmentTimer: number | null = null;
   let heldReleaseTimer: number | null = null;
 
   const cacheCausalTranscriptKeys = (
@@ -1268,9 +1270,15 @@ export function startNativeAssistantSpeech(
       return;
     }
     const messages = useChatStore.getState().messagesBySession[sessionId] ?? [];
-    if (heldSpeech || voice.userSpeaking || idleSettling) {
-      // Text can keep streaming during the idle settling turn. Refresh the
-      // held snapshot before a finalized voice message can invalidate it.
+    if (
+      heldSpeech ||
+      voice.userSpeaking ||
+      pendingUserRecognitionSegment ||
+      heldReleaseTimer !== null
+    ) {
+      // Text can keep streaming while recognition resolves the user's
+      // interruption segment. Refresh the held snapshot before a finalized
+      // voice message can invalidate it.
       holdAssistantChanges(messages);
     }
     const finalizedTranscriptKey = voice.latestFinalizedTranscriptKey;
@@ -1291,7 +1299,7 @@ export function startNativeAssistantSpeech(
       interruptActiveUtterance(true, "userSpeaking");
     }
 
-    if (voice.userSpeaking || idleSettling) return;
+    if (voice.userSpeaking || pendingUserRecognitionSegment) return;
     if (heldSpeech && !heldReleaseReady) return;
     if (resumableInterruption) return;
     if (activeUtterance?.interruptionRequested) return;
@@ -1526,6 +1534,19 @@ export function startNativeAssistantSpeech(
   }
   let wasUserSpeaking = initialVoice.userSpeaking;
   let wasMicrophoneMuted = initialVoice.microphoneMuted;
+  let latestObservedFinalizedTranscriptKey =
+    useVoiceConversationStore.getState().latestFinalizedTranscriptKey;
+  const resolvePendingRecognitionSegment = (releaseSpeech = true) => {
+    if (recognitionSegmentTimer !== null) {
+      window.clearTimeout(recognitionSegmentTimer);
+      recognitionSegmentTimer = null;
+    }
+    pendingUserRecognitionSegment = false;
+    if (!releaseSpeech) return;
+    heldReleaseReady = heldSpeech !== null;
+    interruptionReleaseReady = true;
+    releaseResumableInterruption();
+  };
   const unsubscribeVoice = useVoiceConversationStore.subscribe((voice) => {
     const runningForSession =
       voice.status.lifecycle === "running" &&
@@ -1546,34 +1567,90 @@ export function startNativeAssistantSpeech(
     const becameUserSpeaking = voice.userSpeaking && !wasUserSpeaking;
     const becameUserIdle = !voice.userSpeaking && wasUserSpeaking;
     const becameMicrophoneMuted = voice.microphoneMuted && !wasMicrophoneMuted;
+    const finalizedTranscriptChanged =
+      voice.latestFinalizedTranscriptKey !==
+      latestObservedFinalizedTranscriptKey;
+    const hasInterruptedPlaybackHold =
+      activeUtterance?.interruptionRequested || resumableInterruption !== null;
     wasUserSpeaking = voice.userSpeaking;
     wasMicrophoneMuted = voice.microphoneMuted;
+    latestObservedFinalizedTranscriptKey = voice.latestFinalizedTranscriptKey;
     if (activeGeneration !== generation) return;
     if (becameMicrophoneMuted) {
-      if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer);
-      heldReleaseTimer = null;
-      idleSettling = false;
+      resolvePendingRecognitionSegment(false);
       interruptionReleaseReady = false;
       discardHeldAndResumableSpeech();
       inspect();
       return;
     }
+    if (finalizedTranscriptChanged && !pendingUserRecognitionSegment) {
+      if (heldReleaseTimer !== null) {
+        window.clearTimeout(heldReleaseTimer);
+        heldReleaseTimer = null;
+      }
+      holdAssistantChanges(
+        useChatStore.getState().messagesBySession[sessionId] ?? [],
+      );
+      discardInvalidHeldSpeech(voice.latestFinalizedTranscriptKey);
+      inspect();
+      return;
+    }
     if (becameUserSpeaking) {
-      if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer);
-      heldReleaseTimer = null;
-      idleSettling = false;
+      if (heldReleaseTimer !== null) {
+        window.clearTimeout(heldReleaseTimer);
+        heldReleaseTimer = null;
+      }
+      const interrupted = interruptActiveUtterance(true, "userSpeaking");
+      if (interrupted && recognitionSegmentTimer !== null) {
+        window.clearTimeout(recognitionSegmentTimer);
+        recognitionSegmentTimer = null;
+      }
+      pendingUserRecognitionSegment ||= interrupted;
       heldReleaseReady = false;
       interruptionReleaseReady = false;
-      interruptActiveUtterance(true, "userSpeaking");
+      inspect();
+      return;
+    }
+    if (finalizedTranscriptChanged && pendingUserRecognitionSegment) {
+      holdAssistantChanges(
+        useChatStore.getState().messagesBySession[sessionId] ?? [],
+      );
+      const hadResumableInterruption = resumableInterruption !== null;
+      resolvePendingRecognitionSegment(false);
+      if (hadResumableInterruption) {
+        discardResumableInterruption();
+        discardInvalidHeldSpeech(voice.latestFinalizedTranscriptKey);
+      } else {
+        discardHeldAndResumableSpeech();
+      }
       inspect();
       return;
     }
     if (becameUserIdle) {
       if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer);
-      idleSettling = true;
-      // VAD can report silence shortly before the recognizer commits its final
-      // transcript. Give that transcript a bounded opportunity to invalidate
-      // causally stale speech before releasing the held reply.
+      if (pendingUserRecognitionSegment || hasInterruptedPlaybackHold) {
+        pendingUserRecognitionSegment = true;
+        // VAD silence does not imply recognition is idle. Keep interrupted
+        // playback held until a final transcript arrives or the unresolved
+        // user-recognition segment hits a conservative bound.
+        recognitionSegmentTimer ??= window.setTimeout(() => {
+          recognitionSegmentTimer = null;
+          const current = useVoiceConversationStore.getState();
+          if (
+            activeGeneration !== generation ||
+            current.userSpeaking ||
+            current.status.lifecycle !== "running" ||
+            current.status.sessionId !== sessionId ||
+            !pendingUserRecognitionSegment
+          ) {
+            return;
+          }
+          resolvePendingRecognitionSegment(true);
+          inspect();
+        }, USER_RECOGNITION_SEGMENT_TIMEOUT_MS);
+        inspect();
+        return;
+      }
       heldReleaseTimer = window.setTimeout(() => {
         heldReleaseTimer = null;
         const current = useVoiceConversationStore.getState();
@@ -1585,7 +1662,6 @@ export function startNativeAssistantSpeech(
         ) {
           return;
         }
-        idleSettling = false;
         heldReleaseReady = heldSpeech !== null;
         interruptionReleaseReady = true;
         releaseResumableInterruption();
@@ -1596,8 +1672,15 @@ export function startNativeAssistantSpeech(
     inspect();
   });
   stopVoiceSubscription = () => {
-    if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer);
+    if (heldReleaseTimer !== null) {
+      window.clearTimeout(heldReleaseTimer);
+    }
+    if (recognitionSegmentTimer !== null) {
+      window.clearTimeout(recognitionSegmentTimer);
+    }
     heldReleaseTimer = null;
+    recognitionSegmentTimer = null;
+    pendingUserRecognitionSegment = false;
     discardHeldAndResumableSpeech();
     unsubscribeVoice();
   };