wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3276 from pipecat-ai/mb/grok-realtime-cleanup

Browse Source 
GrokRealtimeLLMService cleanup
This commit is contained in:
Mark Backman
2025-12-22 18:13:23 -05:00
committed by GitHub
parent ae9ee33af9 5ad8e5436d
commit 49b53d72a9
5 changed files with 120 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
README.md
View File

@@ -71,19 +71,19 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout
## 🧩 Available services ## 🧩 Available services
| Category | Services | | Category | Services |
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) |
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), Ultravox, | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), Ultravox, |
| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local |
| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) | | Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) |
| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | | Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) |
| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | | Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) |
| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | | Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) |
| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | | Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) |
| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | | Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) |
📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services)

9
examples/foundational/51-grok-realtime.py
View File

@@ -25,7 +25,6 @@ Usage:
python 50-grok-realtime.py --transport daily python 50-grok-realtime.py --transport daily
""" """
import asyncio
import os import os
from datetime import datetime from datetime import datetime
@@ -37,7 +36,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
# Note: Grok has built-in server-side VAD, so we don't need local VAD # Note: Grok has built-in server-side VAD, so we don't need local VAD
# from pipecat.audio.vad.silero import SileroVADAnalyzer # from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame, TranscriptionMessage from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage
from pipecat.observers.loggers.transcription_log_observer import ( from pipecat.observers.loggers.transcription_log_observer import (
TranscriptionLogObserver, TranscriptionLogObserver,
) )
@@ -53,7 +52,6 @@ from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport from pipecat.runner.utils import create_transport
from pipecat.services.grok.realtime.events import ( from pipecat.services.grok.realtime.events import (
SessionProperties, SessionProperties,
TurnDetection,
WebSearchTool, WebSearchTool,
XSearchTool, XSearchTool,
) )
@@ -173,11 +171,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
# Configure Grok session properties # Configure Grok session properties
session_properties = SessionProperties( session_properties = SessionProperties(
# Voice options: Ara (warm, friendly), Rex (confident), Sal (smooth), # Voice options: Ara, Rex, Sal, Eve, Leo
# Eve (energetic), Leo (authoritative)
voice="Ara", voice="Ara",
# Enable server-side VAD for automatic turn detection
turn_detection=TurnDetection(type="server_vad"),
# System instructions # System instructions
instructions="""You are a helpful and friendly AI assistant powered by Grok. instructions="""You are a helpful and friendly AI assistant powered by Grok.

6
scripts/evals/run-release-evals.py
View File

@@ -226,6 +226,11 @@ TESTS_50 = [
("50-ultravox-realtime.py", EVAL_ORDER), ("50-ultravox-realtime.py", EVAL_ORDER),
] ]
TESTS_51 = [
("51-grok-realtime.py", EVAL_WEATHER),
]
TESTS = [ TESTS = [
*TESTS_07, *TESTS_07,
*TESTS_12, *TESTS_12,
@@ -240,6 +245,7 @@ TESTS = [
*TESTS_44, *TESTS_44,
*TESTS_49, *TESTS_49,
*TESTS_50, *TESTS_50,
*TESTS_51,
] ]

29
src/pipecat/services/grok/realtime/events.py
View File

@@ -205,7 +205,9 @@ class SessionProperties(BaseModel):
Parameters: Parameters:
instructions: System instructions for the assistant. instructions: System instructions for the assistant.
voice: The voice the model uses to respond. Options: Ara, Rex, Sal, Eve, Leo. voice: The voice the model uses to respond. Options: Ara, Rex, Sal, Eve, Leo.
turn_detection: Configuration for turn detection, or None for manual. Defaults to "Ara".
turn_detection: Configuration for turn detection. Defaults to server-side VAD.
Set to None for manual turn detection.
audio: Configuration for input and output audio. audio: Configuration for input and output audio.
tools: Available tools for the assistant (web_search, x_search, file_search, function). tools: Available tools for the assistant (web_search, x_search, file_search, function).
""" """
@@ -215,7 +217,9 @@ class SessionProperties(BaseModel):
instructions: Optional[str] = None instructions: Optional[str] = None
voice: Optional[GrokVoice] = "Ara" voice: Optional[GrokVoice] = "Ara"
turn_detection: Optional[TurnDetection] = None turn_detection: Optional[TurnDetection] = Field(
default_factory=lambda: TurnDetection(type="server_vad")
)
audio: Optional[AudioConfiguration] = None audio: Optional[AudioConfiguration] = None
# Tools can be ToolsSchema when provided by user, or list of dicts for API # Tools can be ToolsSchema when provided by user, or list of dicts for API
tools: Optional[ToolsSchema | List[GrokTool]] = None tools: Optional[ToolsSchema | List[GrokTool]] = None
@@ -629,6 +633,26 @@ class ResponseAudioDone(ServerEvent):
item_id: str item_id: str
class ResponseFunctionCallArgumentsDelta(ServerEvent):
"""Event containing incremental function call arguments.
Parameters:
type: Event type, always "response.function_call_arguments.delta".
response_id: ID of the response.
item_id: ID of the conversation item.
call_id: ID of the function call.
delta: Incremental function arguments as JSON.
previous_item_id: ID of the previous item, if any.
"""
type: Literal["response.function_call_arguments.delta"]
response_id: Optional[str] = None
item_id: Optional[str] = None
call_id: str
delta: str
previous_item_id: Optional[str] = None
class ResponseFunctionCallArgumentsDone(ServerEvent): class ResponseFunctionCallArgumentsDone(ServerEvent):
"""Event indicating function call arguments are complete. """Event indicating function call arguments are complete.
@@ -820,6 +844,7 @@ _server_event_types = {
"response.output_audio_transcript.done": ResponseAudioTranscriptDone, "response.output_audio_transcript.done": ResponseAudioTranscriptDone,
"response.output_audio.delta": ResponseAudioDelta, "response.output_audio.delta": ResponseAudioDelta,
"response.output_audio.done": ResponseAudioDone, "response.output_audio.done": ResponseAudioDone,
"response.function_call_arguments.delta": ResponseFunctionCallArgumentsDelta,
"response.function_call_arguments.done": ResponseFunctionCallArgumentsDone, "response.function_call_arguments.done": ResponseFunctionCallArgumentsDone,
"response.done": ResponseDone, "response.done": ResponseDone,
} }

102
src/pipecat/services/grok/realtime/llm.py
View File

@@ -27,14 +27,12 @@ from pipecat.frames.frames import (
EndFrame, EndFrame,
Frame, Frame,
InputAudioRawFrame, InputAudioRawFrame,
InterimTranscriptionFrame,
InterruptionFrame, InterruptionFrame,
LLMContextFrame, LLMContextFrame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
LLMFullResponseStartFrame, LLMFullResponseStartFrame,
LLMMessagesAppendFrame, LLMMessagesAppendFrame,
LLMSetToolsFrame, LLMSetToolsFrame,
LLMTextFrame,
LLMUpdateSettingsFrame, LLMUpdateSettingsFrame,
StartFrame, StartFrame,
TranscriptionFrame, TranscriptionFrame,
@@ -57,7 +55,6 @@ from pipecat.processors.aggregators.llm_response_universal import (
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601 from pipecat.utils.time import time_now_iso8601
from . import events from . import events
@@ -110,50 +107,36 @@ class GrokRealtimeLLMService(LLMService):
self, self,
*, *,
api_key: str, api_key: str,
voice: events.GrokVoice = "Ara",
base_url: str = "wss://api.x.ai/v1/realtime", base_url: str = "wss://api.x.ai/v1/realtime",
session_properties: Optional[events.SessionProperties] = None, session_properties: Optional[events.SessionProperties] = None,
start_audio_paused: bool = False, start_audio_paused: bool = False,
sample_rate: int = 24000,
**kwargs, **kwargs,
): ):
"""Initialize the Grok Realtime Voice Agent LLM service. """Initialize the Grok Realtime Voice Agent LLM service.
Args: Args:
api_key: xAI API key for authentication. api_key: xAI API key for authentication.
voice: Voice to use for responses. Options: Ara, Rex, Sal, Eve, Leo.
Defaults to "Ara".
base_url: WebSocket base URL for the realtime API. base_url: WebSocket base URL for the realtime API.
Defaults to "wss://api.x.ai/v1/realtime". Defaults to "wss://api.x.ai/v1/realtime".
session_properties: Configuration properties for the realtime session. session_properties: Configuration properties for the realtime session.
If None, uses default SessionProperties with the specified voice. If None, uses default SessionProperties with voice "Ara".
To set a different voice, configure it in session_properties:
session_properties = events.SessionProperties(voice="Rex")
Available voices: Ara, Rex, Sal, Eve, Leo.
start_audio_paused: Whether to start with audio input paused. Defaults to False. start_audio_paused: Whether to start with audio input paused. Defaults to False.
sample_rate: Audio sample rate in Hz. Supported: 8000, 16000, 21050, 24000,
32000, 44100, 48000. Defaults to 24000.
**kwargs: Additional arguments passed to parent LLMService. **kwargs: Additional arguments passed to parent LLMService.
""" """
super().__init__(base_url=base_url, **kwargs) super().__init__(base_url=base_url, **kwargs)
self.api_key = api_key self.api_key = api_key
self.base_url = base_url self.base_url = base_url
self._sample_rate = sample_rate
self._voice = voice
# Initialize session_properties with voice and audio config # Initialize session_properties
if session_properties: self._session_properties: events.SessionProperties = (
self._session_properties = session_properties session_properties or events.SessionProperties()
# Ensure voice is set )
if not self._session_properties.voice:
self._session_properties.voice = voice
else:
self._session_properties = events.SessionProperties(
voice=voice,
turn_detection=events.TurnDetection(type="server_vad"),
audio=events.AudioConfiguration(
input=events.AudioInput(format=events.PCMAudioFormat(rate=sample_rate)),
output=events.AudioOutput(format=events.PCMAudioFormat(rate=sample_rate)),
),
)
self._audio_input_paused = start_audio_paused self._audio_input_paused = start_audio_paused
self._websocket = None self._websocket = None
@@ -192,6 +175,50 @@ class GrokRealtimeLLMService(LLMService):
""" """
self._audio_input_paused = paused self._audio_input_paused = paused
def _get_configured_sample_rate(self, direction: str) -> Optional[int]:
"""Get manually configured sample rate for input or output.
Args:
direction: Either "input" or "output".
Returns:
Configured sample rate or None if not manually configured.
For PCMU/PCMA formats, returns 8000 Hz (G.711 standard).
"""
if not self._session_properties.audio:
return None
audio_config = (
self._session_properties.audio.input
if direction == "input"
else self._session_properties.audio.output
)
if audio_config and audio_config.format:
# PCM format has configurable rate
if hasattr(audio_config.format, "rate"):
return audio_config.format.rate
# PCMU/PCMA formats are fixed at 8000 Hz (G.711 standard)
elif audio_config.format.type in ("audio/pcmu", "audio/pcma"):
return 8000
return None
def _get_output_sample_rate(self) -> int:
"""Get the output sample rate from session properties.
Returns:
Output sample rate in Hz.
Note:
This assumes start() has been called, which guarantees
session_properties.audio.output exists.
"""
rate = self._get_configured_sample_rate("output")
if rate is None:
raise RuntimeError("Output sample rate not configured.")
return rate
def _is_turn_detection_enabled(self) -> bool: def _is_turn_detection_enabled(self) -> bool:
"""Check if server-side VAD is enabled.""" """Check if server-side VAD is enabled."""
if self._session_properties.turn_detection: if self._session_properties.turn_detection:
@@ -230,7 +257,7 @@ class GrokRealtimeLLMService(LLMService):
) -> int: ) -> int:
"""Calculate audio duration in milliseconds based on PCM audio parameters.""" """Calculate audio duration in milliseconds based on PCM audio parameters."""
if sample_rate is None: if sample_rate is None:
sample_rate = self._sample_rate sample_rate = self._get_output_sample_rate()
samples = total_bytes / bytes_per_sample samples = total_bytes / bytes_per_sample
duration_seconds = samples / sample_rate duration_seconds = samples / sample_rate
return int(duration_seconds * 1000) return int(duration_seconds * 1000)
@@ -260,6 +287,23 @@ class GrokRealtimeLLMService(LLMService):
frame: The start frame triggering service initialization. frame: The start frame triggering service initialization.
""" """
await super().start(frame) await super().start(frame)
# Ensure audio configuration exists with both input and output
if not self._session_properties.audio:
self._session_properties.audio = events.AudioConfiguration()
# Fill in missing input configuration
if not self._session_properties.audio.input:
self._session_properties.audio.input = events.AudioInput(
format=events.PCMAudioFormat(rate=frame.audio_in_sample_rate)
)
# Fill in missing output configuration
if not self._session_properties.audio.output:
self._session_properties.audio.output = events.AudioOutput(
format=events.PCMAudioFormat(rate=frame.audio_out_sample_rate)
)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
@@ -501,7 +545,7 @@ class GrokRealtimeLLMService(LLMService):
frame = TTSAudioRawFrame( frame = TTSAudioRawFrame(
audio=audio, audio=audio,
sample_rate=self._sample_rate, sample_rate=self._get_output_sample_rate(),
num_channels=1, num_channels=1,
) )
await self.push_frame(frame) await self.push_frame(frame)