diff --git a/changelog/+nova-sonic-server-interruption.fixed.md b/changelog/+nova-sonic-server-interruption.fixed.md new file mode 100644 index 000000000..072d30728 --- /dev/null +++ b/changelog/+nova-sonic-server-interruption.fixed.md @@ -0,0 +1 @@ +- Fixed AWS Nova Sonic not surfacing server-side interruption. When the user interrupted the bot mid-response, the `INTERRUPTED` stop reason was acknowledged internally but no `InterruptionFrame` was emitted, so `BaseOutputTransport` kept draining its audio buffer and the bot kept talking past the interruption. Nova Sonic now broadcasts `InterruptionFrame` on both `INTERRUPTED` paths (text-stage and audio-stage). This was previously masked by enabling local VAD on the user aggregator, which generated `UserStartedSpeakingFrame` and triggered the aggregator-side interruption path; the fix makes the behavior correct without local VAD as a workaround. diff --git a/changelog/+ultravox-server-interruption.fixed.md b/changelog/+ultravox-server-interruption.fixed.md new file mode 100644 index 000000000..2e4fc0acd --- /dev/null +++ b/changelog/+ultravox-server-interruption.fixed.md @@ -0,0 +1 @@ +- Fixed Ultravox Realtime not surfacing server-side interruption. The server sends a `playback_clear_buffer` message when the user interrupts the bot mid-speech, instructing clients to drop buffered output audio; this was previously unhandled, so `BaseOutputTransport` kept playing the buffered audio and the bot kept talking past the interruption. Ultravox now broadcasts `InterruptionFrame` on `playback_clear_buffer`. This was previously masked by enabling local VAD on the user aggregator, which generated `UserStartedSpeakingFrame` and triggered the aggregator-side interruption path; the fix makes the behavior correct without local VAD as a workaround. diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 2e6aab9ca..44e2b01a7 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -1434,7 +1434,14 @@ class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]): else: if self._assistant_is_responding: # TEXT INTERRUPTED before audio started means no AUDIO - # contentEnd will arrive — end the response here. + # contentEnd will arrive — end the response here. Emit + # InterruptionFrame upstream so the assistant aggregator + # marks the message interrupted=True, and downstream so + # BaseOutputTransport can clear any audio it had already + # buffered. Must fire before _report_assistant_response_ended + # so the aggregator handles InterruptionFrame before + # LLMFullResponseEndFrame closes the turn. + await self.broadcast_interruption() self._assistant_is_responding = False await self._report_assistant_response_ended() # Session continuation: TEXT INTERRUPTED is a completion @@ -1447,6 +1454,18 @@ class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]): if stop_reason in ("END_TURN", "INTERRUPTED"): # END_TURN: normal completion. INTERRUPTED: user interrupted # mid-audio. Both mean no more audio for this turn. + if stop_reason == "INTERRUPTED": + # Emit InterruptionFrame upstream so the assistant + # aggregator marks the message interrupted=True, and + # downstream so BaseOutputTransport clears the audio + # buffer (without this the bot keeps talking past the + # interruption while the buffer drains, since Nova + # Sonic doesn't surface server-side interruption any + # other way). Must fire before + # _report_assistant_response_ended so the aggregator + # handles InterruptionFrame before LLMFullResponseEndFrame + # closes the turn. + await self.broadcast_interruption() self._assistant_is_responding = False await self._report_assistant_response_ended() elif content.role == Role.USER: diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index 91aa66486..d0f7b3a15 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -604,6 +604,18 @@ class UltravoxRealtimeLLMService(LLMService): case "state": if self._bot_responding and data.get("state") != "speaking": await self._handle_response_end() + case "playback_clear_buffer": + # Server signals that the user interrupted the bot + # mid-speech and any buffered output audio should be + # dropped. Broadcast InterruptionFrame so the assistant + # aggregator records the message interrupted=True + # (upstream) and BaseOutputTransport clears its audio + # buffer (downstream). The subsequent "state" message + # transitioning off "speaking" is what closes the + # response via _handle_response_end; firing the + # interruption first ensures the aggregator handles + # InterruptionFrame before LLMFullResponseEndFrame. + await self.broadcast_interruption() case "client_tool_invocation": await self._handle_tool_invocation( data.get("toolName"), data.get("invocationId"), data.get("parameters")