diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 44e2b01a7..afd47a3fa 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -241,6 +241,17 @@ class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]): Provides bidirectional audio streaming, real-time transcription, text generation, and function calling capabilities using AWS Nova Sonic model. + + Does NOT emit ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame``, + so pipeline processors that depend on those frames — RTVI client + speech events, ``TurnTrackingObserver``, ``AudioBufferProcessor`` turn + recording, ``UserIdleController``, user mute strategies, voicemail + detector — won't activate with the default server-VAD-only setup. Pair + with ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are correct anyway. To produce the turn frames + locally, wire ``vad_analyzer=SileroVADAnalyzer()`` (or similar) into + ``LLMUserAggregatorParams``; locally-generated turn boundaries are a + heuristic and may not match Nova Sonic's server-side turn decisions. """ Settings = AWSNovaSonicLLMSettings diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 80d722d29..6d3c4fa0e 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -361,6 +361,18 @@ class GeminiLiveLLMService(LLMService[GeminiLLMAdapter]): This service enables real-time conversations with Gemini, supporting both text and audio modalities. It handles voice transcription, streaming audio responses, and tool usage. + + Does NOT emit ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` + (the API exposes an ``interrupted`` event but no turn-start/-end), so + pipeline processors that depend on those frames — RTVI client speech + events, ``TurnTrackingObserver``, ``AudioBufferProcessor`` turn + recording, ``UserIdleController``, user mute strategies, voicemail + detector — won't activate with the default server-VAD-only setup. Pair + with ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are correct anyway. To produce the turn frames + locally, see ``examples/realtime/realtime-gemini-live-local-vad.py``; + note that locally-generated turn boundaries are a heuristic and may + not match Gemini Live's server-side turn decisions. """ Settings = GeminiLiveLLMSettings diff --git a/src/pipecat/services/inworld/realtime/llm.py b/src/pipecat/services/inworld/realtime/llm.py index 352e669e4..c151ee2ee 100644 --- a/src/pipecat/services/inworld/realtime/llm.py +++ b/src/pipecat/services/inworld/realtime/llm.py @@ -201,6 +201,11 @@ class InworldRealtimeLLMService(LLMService[InworldRealtimeLLMAdapter]): Supports function calling, conversation management, and real-time transcription. + Emits ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` from + Inworld's server-side VAD events. Pair with + ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are decoupled from those frames. + Example:: llm = InworldRealtimeLLMService( diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 9ce78ab89..4ef98aa35 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -204,6 +204,15 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]): Implements the OpenAI Realtime API with WebSocket communication for low-latency bidirectional audio and text interactions. Supports function calling, conversation management, and real-time transcription. + + Emits ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` from + OpenAI's server-side VAD events, so pipeline processors that depend on + those frames (RTVI client speech events, ``TurnTrackingObserver``, + ``AudioBufferProcessor`` turn recording, ``UserIdleController``, user + mute strategies, voicemail detector) work out of the box. Pair with + ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are decoupled from those frames; see the + ``examples/realtime/realtime-openai.py`` example. """ Settings = OpenAIRealtimeLLMSettings diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index d0f7b3a15..8cae4fbe5 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -174,6 +174,17 @@ class UltravoxRealtimeLLMService(LLMService): Note: Ultravox is an audio-native model, so voice transcriptions are not used by the model and may not always align with its understanding of user input. + + Does NOT emit ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame``, + so pipeline processors that depend on those frames — RTVI client + speech events, ``TurnTrackingObserver``, ``AudioBufferProcessor`` turn + recording, ``UserIdleController``, user mute strategies, voicemail + detector — won't activate with the default server-VAD-only setup. Pair + with ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are correct anyway. To produce the turn frames + locally, wire ``vad_analyzer=SileroVADAnalyzer()`` (or similar) into + ``LLMUserAggregatorParams``; locally-generated turn boundaries are a + heuristic and may not match Ultravox's server-side turn decisions. """ Settings = UltravoxRealtimeLLMSettings diff --git a/src/pipecat/services/xai/realtime/llm.py b/src/pipecat/services/xai/realtime/llm.py index cbfdbaf61..0b9562fa9 100644 --- a/src/pipecat/services/xai/realtime/llm.py +++ b/src/pipecat/services/xai/realtime/llm.py @@ -195,6 +195,11 @@ class GrokRealtimeLLMService(LLMService[GrokRealtimeLLMAdapter]): - Built-in tools (web_search, x_search, file_search) - Custom function calling - Server-side VAD (Voice Activity Detection) + + Emits ``UserStartedSpeakingFrame`` / ``UserStoppedSpeakingFrame`` from + Grok's server-side VAD events. Pair with + ``LLMContextAggregatorPair(..., realtime_service_mode=RealtimeServiceModeConfig())`` + so context writes are decoupled from those frames. """ Settings = GrokRealtimeLLMSettings