From 20d9bf4af678ddc991ceef2c68d3f40f00f09cdc Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 20 May 2026 15:50:32 -0400 Subject: [PATCH] Document user-turn-frame behavior in realtime service docstrings Each realtime LLM service docstring now states whether the service emits UserStartedSpeakingFrame / UserStoppedSpeakingFrame from server-side turn signals, and what that implies for the rest of the pipeline. For the services that don't (Gemini Live, AWS Nova Sonic, Ultravox), the docstring spells out which downstream processors won't activate (RTVI client speech events, TurnTrackingObserver, AudioBufferProcessor turn recording, UserIdleController, user mute strategies, voicemail detector), points at realtime_service_mode for correct context-write semantics, and notes the option of wiring local VAD plus the caveat that locally- generated turn boundaries are a heuristic that may not match the provider's server-side turn decisions. For the services that do (OpenAI Realtime, Inworld, Grok/xAI Realtime), the docstring confirms turn frames are emitted from server VAD and points at realtime_service_mode. --- src/pipecat/services/aws/nova_sonic/llm.py | 11 +++++++++++ src/pipecat/services/google/gemini_live/llm.py | 12 ++++++++++++ src/pipecat/services/inworld/realtime/llm.py | 5 +++++ src/pipecat/services/openai/realtime/llm.py | 9 +++++++++ src/pipecat/services/ultravox/llm.py | 11 +++++++++++ src/pipecat/services/xai/realtime/llm.py | 5 +++++ 6 files changed, 53 insertions(+) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 44e2b01a7..afd47a3fa 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -241,6 +241,17 @@ class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]): Provides bidirectional audio streaming, real-time transcription, text generation, and function calling capabilities using AWS Nova Sonic model. + + Does NOT emit ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame``, + so pipeline processors that depend on those frames — RTVI client + speech events, ``TurnTrackingObserver``, ``AudioBufferProcessor`` turn + recording, ``UserIdleController``, user mute strategies, voicemail + detector — won't activate with the default server-VAD-only setup. Pair + with ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are correct anyway. To produce the turn frames + locally, wire ``vad_analyzer=SileroVADAnalyzer()`` (or similar) into + ``LLMUserAggregatorParams``; locally-generated turn boundaries are a + heuristic and may not match Nova Sonic's server-side turn decisions. """ Settings = AWSNovaSonicLLMSettings diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 80d722d29..6d3c4fa0e 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -361,6 +361,18 @@ class GeminiLiveLLMService(LLMService[GeminiLLMAdapter]): This service enables real-time conversations with Gemini, supporting both text and audio modalities. It handles voice transcription, streaming audio responses, and tool usage. + + Does NOT emit ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` + (the API exposes an ``interrupted`` event but no turn-start/-end), so + pipeline processors that depend on those frames — RTVI client speech + events, ``TurnTrackingObserver``, ``AudioBufferProcessor`` turn + recording, ``UserIdleController``, user mute strategies, voicemail + detector — won't activate with the default server-VAD-only setup. Pair + with ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are correct anyway. To produce the turn frames + locally, see ``examples/realtime/realtime-gemini-live-local-vad.py``; + note that locally-generated turn boundaries are a heuristic and may + not match Gemini Live's server-side turn decisions. """ Settings = GeminiLiveLLMSettings diff --git a/src/pipecat/services/inworld/realtime/llm.py b/src/pipecat/services/inworld/realtime/llm.py index 352e669e4..c151ee2ee 100644 --- a/src/pipecat/services/inworld/realtime/llm.py +++ b/src/pipecat/services/inworld/realtime/llm.py @@ -201,6 +201,11 @@ class InworldRealtimeLLMService(LLMService[InworldRealtimeLLMAdapter]): Supports function calling, conversation management, and real-time transcription. + Emits ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` from + Inworld's server-side VAD events. Pair with + ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are decoupled from those frames. + Example:: llm = InworldRealtimeLLMService( diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 9ce78ab89..4ef98aa35 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -204,6 +204,15 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]): Implements the OpenAI Realtime API with WebSocket communication for low-latency bidirectional audio and text interactions. Supports function calling, conversation management, and real-time transcription. + + Emits ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` from + OpenAI's server-side VAD events, so pipeline processors that depend on + those frames (RTVI client speech events, ``TurnTrackingObserver``, + ``AudioBufferProcessor`` turn recording, ``UserIdleController``, user + mute strategies, voicemail detector) work out of the box. Pair with + ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are decoupled from those frames; see the + ``examples/realtime/realtime-openai.py`` example. """ Settings = OpenAIRealtimeLLMSettings diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index d0f7b3a15..8cae4fbe5 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -174,6 +174,17 @@ class UltravoxRealtimeLLMService(LLMService): Note: Ultravox is an audio-native model, so voice transcriptions are not used by the model and may not always align with its understanding of user input. + + Does NOT emit ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame``, + so pipeline processors that depend on those frames — RTVI client + speech events, ``TurnTrackingObserver``, ``AudioBufferProcessor`` turn + recording, ``UserIdleController``, user mute strategies, voicemail + detector — won't activate with the default server-VAD-only setup. Pair + with ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are correct anyway. To produce the turn frames + locally, wire ``vad_analyzer=SileroVADAnalyzer()`` (or similar) into + ``LLMUserAggregatorParams``; locally-generated turn boundaries are a + heuristic and may not match Ultravox's server-side turn decisions. """ Settings = UltravoxRealtimeLLMSettings diff --git a/src/pipecat/services/xai/realtime/llm.py b/src/pipecat/services/xai/realtime/llm.py index cbfdbaf61..0b9562fa9 100644 --- a/src/pipecat/services/xai/realtime/llm.py +++ b/src/pipecat/services/xai/realtime/llm.py @@ -195,6 +195,11 @@ class GrokRealtimeLLMService(LLMService[GrokRealtimeLLMAdapter]): - Built-in tools (web_search, x_search, file_search) - Custom function calling - Server-side VAD (Voice Activity Detection) + + Emits ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` from + Grok's server-side VAD events. Pair with + ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are decoupled from those frames. """ Settings = GrokRealtimeLLMSettings