From 02b97035f87b8177231cf510eb5357ce549abdfe Mon Sep 17 00:00:00 2001 From: Nicholas Zhao Date: Fri, 13 Mar 2026 14:55:54 -0700 Subject: [PATCH 1/3] Add xAI TTS service --- README.md | 28 ++-- pyproject.toml | 1 + src/pipecat/services/xai/__init__.py | 13 ++ src/pipecat/services/xai/tts.py | 213 +++++++++++++++++++++++++++ tests/test_xai_tts.py | 91 ++++++++++++ 5 files changed, 332 insertions(+), 14 deletions(-) create mode 100644 src/pipecat/services/xai/__init__.py create mode 100644 src/pipecat/services/xai/tts.py create mode 100644 tests/test_xai_tts.py diff --git a/README.md b/README.md index 842863548..8d2322200 100644 --- a/README.md +++ b/README.md @@ -85,20 +85,20 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout ## 🧩 Available services -| Category | Services | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | -| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/server/services/tts/smallest), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | -| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | -| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | -| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | -| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | -| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | -| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | -| Community | [Browse community integrations →](https://docs.pipecat.ai/server/services/community-integrations) | +| Category | Services | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | +| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/server/services/tts/smallest), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), xAI, [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | +| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | +| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | +| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | +| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | +| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | +| Community | [Browse community integrations →](https://docs.pipecat.ai/server/services/community-integrations) | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) diff --git a/pyproject.toml b/pyproject.toml index cba80aee4..e1f2dcf16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,7 @@ webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ] websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<1" ] websockets-base = [ "websockets>=13.1,<16.0" ] whisper = [ "faster-whisper~=1.2.1" ] +xai = [] [dependency-groups] dev = [ diff --git a/src/pipecat/services/xai/__init__.py b/src/pipecat/services/xai/__init__.py new file mode 100644 index 000000000..5433f9431 --- /dev/null +++ b/src/pipecat/services/xai/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import sys + +from pipecat.services import DeprecatedModuleProxy + +from .tts import * + +sys.modules[__name__] = DeprecatedModuleProxy(globals(), "xai", "xai.tts") diff --git a/src/pipecat/services/xai/tts.py b/src/pipecat/services/xai/tts.py new file mode 100644 index 000000000..37a3db80a --- /dev/null +++ b/src/pipecat/services/xai/tts.py @@ -0,0 +1,213 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""xAI text-to-speech service implementation. + +Uses xAI's HTTP TTS endpoint documented at: +https://docs.x.ai/developers/model-capabilities/audio/text-to-speech +""" + +from dataclasses import dataclass +from typing import AsyncGenerator, Optional + +import aiohttp +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame +from pipecat.services.settings import TTSSettings +from pipecat.services.tts_service import TTSService +from pipecat.transcriptions.language import Language +from pipecat.utils.tracing.service_decorators import traced_tts + + +@dataclass +class XAITTSSettings(TTSSettings): + """Settings for XAITTSService.""" + + pass + + +class XAITTSService(TTSService): + """xAI HTTP text-to-speech service. + + The service requests raw PCM audio so emitted ``TTSAudioRawFrame`` objects + match Pipecat's downstream expectations without extra decoding. + """ + + Settings = XAITTSSettings + _settings: Settings + + XAI_DEFAULT_SAMPLE_RATE = 24000 + XAI_PCM_CODEC = "pcm" + + class InputParams(BaseModel): + """Input parameters for xAI TTS configuration. + + .. deprecated:: 0.0.105 + Use ``settings=XAITTSService.Settings(...)`` instead. + + Parameters: + language: Language for speech synthesis. + """ + + language: Optional[Language] = None + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.x.ai/v1/tts", + voice: Optional[str] = None, + language: Optional[str | Language] = None, + sample_rate: Optional[int] = None, + aiohttp_session: Optional[aiohttp.ClientSession] = None, + params: Optional[InputParams] = None, + settings: Optional[Settings] = None, + **kwargs, + ): + """Initialize the xAI TTS service. + + Args: + api_key: xAI API key for authentication. + base_url: xAI TTS endpoint. Defaults to ``https://api.x.ai/v1/tts``. + voice: Voice identifier. Defaults to ``"eve"``. + + .. deprecated:: 0.0.105 + Use ``settings=XAITTSService.Settings(voice=...)`` instead. + + language: BCP-47 or base language code (for example ``"en"`` or ``"pt-BR"``). + Defaults to ``"en"``. + + .. deprecated:: 0.0.105 + Use ``settings=XAITTSService.Settings(language=...)`` instead. + + sample_rate: Output sample rate for PCM audio. Defaults to 24000 Hz. + aiohttp_session: Optional shared aiohttp session. + params: Deprecated input parameters object. + settings: Runtime-updatable settings. When provided alongside deprecated + parameters, ``settings`` values take precedence. + **kwargs: Additional keyword arguments passed to ``TTSService``. + """ + default_settings = self.Settings( + model=None, + voice="eve", + language="en", + ) + + if voice is not None: + self._warn_init_param_moved_to_settings("voice", "voice") + default_settings.voice = voice + if language is not None: + self._warn_init_param_moved_to_settings("language", "language") + default_settings.language = ( + self.language_to_service_language(language) + if isinstance(language, Language) + else language + ) + + if params is not None: + self._warn_init_param_moved_to_settings("params") + if not settings and params.language is not None: + default_settings.language = self.language_to_service_language(params.language) + + if settings is not None: + default_settings.apply_update(settings) + + super().__init__( + pause_frame_processing=True, + push_start_frame=True, + push_stop_frames=True, + sample_rate=sample_rate or self.XAI_DEFAULT_SAMPLE_RATE, + settings=default_settings, + **kwargs, + ) + + self._api_key = api_key + self._base_url = base_url + self._session = aiohttp_session + self._session_owner = aiohttp_session is None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics.""" + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to xAI's language format.""" + return str(language) + + async def start(self, frame): + """Start the xAI TTS service.""" + await super().start(frame) + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession() + self._session_owner = True + + async def stop(self, frame): + """Stop the xAI TTS service.""" + await super().stop(frame) + await self._close_session() + + async def cancel(self, frame): + """Cancel the xAI TTS service.""" + await super().cancel(frame) + await self._close_session() + + async def _close_session(self): + if self._session_owner and self._session and not self._session.closed: + await self._session.close() + if self._session_owner: + self._session = None + + @traced_tts + async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using xAI's TTS API.""" + logger.debug(f"{self}: Generating TTS [{text}]") + + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession() + self._session_owner = True + + payload = { + "text": text, + "voice_id": self._settings.voice, + "output_format": { + "codec": self.XAI_PCM_CODEC, + "sample_rate": self.sample_rate, + }, + } + if self._settings.language: + payload["language"] = str(self._settings.language) + + headers = { + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + } + + measuring_ttfb = True + try: + async with self._session.post(self._base_url, json=payload, headers=headers) as response: + if response.status != 200: + error = await response.text(errors="ignore") + logger.error( + f"{self} error getting audio (status: {response.status}, error: {error})" + ) + yield ErrorFrame( + error=f"Error getting audio (status: {response.status}, error: {error})" + ) + return + + await self.start_tts_usage_metrics(text) + + async for chunk in response.content.iter_chunked(self.chunk_size): + if not chunk: + continue + if measuring_ttfb: + await self.stop_ttfb_metrics() + measuring_ttfb = False + yield TTSAudioRawFrame(chunk, self.sample_rate, 1, context_id=context_id) + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/tests/test_xai_tts.py b/tests/test_xai_tts.py new file mode 100644 index 000000000..b4c1513f6 --- /dev/null +++ b/tests/test_xai_tts.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Tests for XAITTSService.""" + +import asyncio +import unittest + +import aiohttp +import pytest +from aiohttp import web + +from pipecat.frames.frames import ( + AggregatedTextFrame, + TTSAudioRawFrame, + TTSSpeakFrame, + TTSStartedFrame, + TTSStoppedFrame, + TTSTextFrame, +) +from pipecat.services.xai.tts import XAITTSService +from pipecat.tests.utils import run_test + + +@pytest.mark.asyncio +async def test_run_xai_tts_success(aiohttp_client): + """xAI TTS should send the documented request body and emit PCM frames.""" + + request_bodies = [] + + async def handler(request): + request_bodies.append(await request.json()) + + response = web.StreamResponse( + status=200, + reason="OK", + headers={"Content-Type": "audio/pcm"}, + ) + await response.prepare(request) + await response.write(b"\x00\x01\x02\x03" * 1024) + await asyncio.sleep(0.01) + await response.write(b"\x04\x05\x06\x07" * 1024) + await response.write_eof() + return response + + app = web.Application() + app.router.add_post("/v1/tts", handler) + client = await aiohttp_client(app) + base_url = str(client.make_url("/v1/tts")) + + async with aiohttp.ClientSession() as session: + tts_service = XAITTSService( + api_key="test-key", + base_url=base_url, + aiohttp_session=session, + sample_rate=24000, + ) + + down_frames, _ = await run_test( + tts_service, + frames_to_send=[TTSSpeakFrame(text="Hello from xAI.")], + ) + + frame_types = [type(frame) for frame in down_frames] + assert AggregatedTextFrame in frame_types + assert TTSStartedFrame in frame_types + assert TTSStoppedFrame in frame_types + assert TTSTextFrame in frame_types + + audio_frames = [frame for frame in down_frames if isinstance(frame, TTSAudioRawFrame)] + assert audio_frames + assert all(frame.sample_rate == 24000 for frame in audio_frames) + assert all(frame.num_channels == 1 for frame in audio_frames) + + assert len(request_bodies) == 1 + assert request_bodies[0] == { + "text": "Hello from xAI.", + "voice_id": "eve", + "language": "en", + "output_format": { + "codec": "pcm", + "sample_rate": 24000, + }, + } + + +if __name__ == "__main__": + unittest.main() From bbd14de9c579021d8dde853dc7e704d81d0acc43 Mon Sep 17 00:00:00 2001 From: Nicholas Zhao Date: Tue, 24 Mar 2026 22:21:17 -0700 Subject: [PATCH 2/3] Address PR review: rename to XAIHttpTTSService, add language map, clean up API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename XAITTSService → XAIHttpTTSService and XAITTSSettings → XAIHttpTTSSettings - Add language_to_xai_language() with explicit LANGUAGE_MAP using resolve_language() - Remove deprecated InputParams, params, voice, language init params - Remove XAI_DEFAULT_SAMPLE_RATE and XAI_PCM_CODEC constants; add encoding param - Set sample_rate=None default (picked up from PipelineParams or user) - Use Language.EN enum instead of string "en" for default language - Add changelog/4031.added.md - Add 07e-interruptible-xai.py foundational example - Update 14g-function-calling-grok.py to use XAIHttpTTSService - Register 07e in run-release-evals.py --- changelog/4031.added.md | 1 + .../foundational/07e-interruptible-xai.py | 129 ++++++++++++++++ .../foundational/14g-function-calling-grok.py | 142 +++++++++--------- src/pipecat/services/xai/tts.py | 135 +++++++++-------- tests/test_xai_tts.py | 6 +- 5 files changed, 276 insertions(+), 137 deletions(-) create mode 100644 changelog/4031.added.md create mode 100644 examples/foundational/07e-interruptible-xai.py diff --git a/changelog/4031.added.md b/changelog/4031.added.md new file mode 100644 index 000000000..0c228a24a --- /dev/null +++ b/changelog/4031.added.md @@ -0,0 +1 @@ +- Added `XAIHttpTTSService` for text-to-speech using xAI's HTTP TTS API. diff --git a/examples/foundational/07e-interruptible-xai.py b/examples/foundational/07e-interruptible-xai.py new file mode 100644 index 000000000..762c27e7c --- /dev/null +++ b/examples/foundational/07e-interruptible-xai.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.grok.llm import GrokLLMService +from pipecat.services.xai.tts import XAIHttpTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We use lambdas to defer transport parameter creation until the transport +# type is selected at runtime. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = XAIHttpTTSService( + api_key=os.getenv("GROK_API_KEY"), + aiohttp_session=session, + settings=XAIHttpTTSService.Settings( + voice="eve", + ), + ) + + llm = GrokLLMService( + api_key=os.getenv("GROK_API_KEY"), + settings=GrokLLMService.Settings( + system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", + ), + ) + + context = LLMContext() + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + user_aggregator, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + assistant_aggregator, # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + audio_out_sample_rate=8000, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + context.add_message( + {"role": "user", "content": "Please introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 4de1f6528..34ec82b5f 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -7,6 +7,7 @@ import os +import aiohttp from dotenv import load_dotenv from loguru import logger @@ -24,10 +25,10 @@ from pipecat.processors.aggregators.llm_response_universal import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.grok.llm import GrokLLMService from pipecat.services.llm_service import FunctionCallParams +from pipecat.services.xai.tts import XAIHttpTTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -60,83 +61,88 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + async with aiohttp.ClientSession() as session: + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - settings=CartesiaTTSService.Settings( - voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ), - ) + tts = XAIHttpTTSService( + api_key=os.getenv("GROK_API_KEY"), + aiohttp_session=session, + settings=XAIHttpTTSService.Settings( + voice="eve", + ), + ) - llm = GrokLLMService( - api_key=os.getenv("GROK_API_KEY"), - settings=GrokLLMService.Settings( - system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", - ), - ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + llm = GrokLLMService( + api_key=os.getenv("GROK_API_KEY"), + settings=GrokLLMService.Settings( + system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", + ), + ) + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - context = LLMContext(tools=tools) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), - ) + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + context = LLMContext(tools=tools) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) - pipeline = Pipeline( - [ - transport.input(), - stt, - user_aggregator, - llm, - tts, - transport.output(), - assistant_aggregator, - ] - ) + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + context.add_message( + {"role": "user", "content": "Please introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - await runner.run(task) + await runner.run(task) async def bot(runner_args: RunnerArguments): diff --git a/src/pipecat/services/xai/tts.py b/src/pipecat/services/xai/tts.py index 37a3db80a..a7026b2ae 100644 --- a/src/pipecat/services/xai/tts.py +++ b/src/pipecat/services/xai/tts.py @@ -15,57 +15,77 @@ from typing import AsyncGenerator, Optional import aiohttp from loguru import logger -from pydantic import BaseModel from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame from pipecat.services.settings import TTSSettings from pipecat.services.tts_service import TTSService -from pipecat.transcriptions.language import Language +from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts +def language_to_xai_language(language: Language) -> Optional[str]: + """Convert a Language enum to xAI language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding xAI language code, or None if not supported. + """ + LANGUAGE_MAP = { + Language.AR: "ar-EG", + Language.AR_EG: "ar-EG", + Language.AR_SA: "ar-SA", + Language.AR_AE: "ar-AE", + Language.BN: "bn", + Language.DE: "de", + Language.EN: "en", + Language.ES: "es-ES", + Language.ES_ES: "es-ES", + Language.ES_MX: "es-MX", + Language.FR: "fr", + Language.HI: "hi", + Language.ID: "id", + Language.IT: "it", + Language.JA: "ja", + Language.KO: "ko", + Language.PT: "pt-PT", + Language.PT_BR: "pt-BR", + Language.PT_PT: "pt-PT", + Language.RU: "ru", + Language.TR: "tr", + Language.VI: "vi", + Language.ZH: "zh", + } + + return resolve_language(language, LANGUAGE_MAP, use_base_code=True) + + @dataclass -class XAITTSSettings(TTSSettings): - """Settings for XAITTSService.""" +class XAIHttpTTSSettings(TTSSettings): + """Settings for XAIHttpTTSService.""" pass -class XAITTSService(TTSService): +class XAIHttpTTSService(TTSService): """xAI HTTP text-to-speech service. The service requests raw PCM audio so emitted ``TTSAudioRawFrame`` objects match Pipecat's downstream expectations without extra decoding. """ - Settings = XAITTSSettings + Settings = XAIHttpTTSSettings _settings: Settings - XAI_DEFAULT_SAMPLE_RATE = 24000 - XAI_PCM_CODEC = "pcm" - - class InputParams(BaseModel): - """Input parameters for xAI TTS configuration. - - .. deprecated:: 0.0.105 - Use ``settings=XAITTSService.Settings(...)`` instead. - - Parameters: - language: Language for speech synthesis. - """ - - language: Optional[Language] = None - def __init__( self, *, api_key: str, base_url: str = "https://api.x.ai/v1/tts", - voice: Optional[str] = None, - language: Optional[str | Language] = None, sample_rate: Optional[int] = None, + encoding: Optional[str] = "pcm", aiohttp_session: Optional[aiohttp.ClientSession] = None, - params: Optional[InputParams] = None, settings: Optional[Settings] = None, **kwargs, ): @@ -74,54 +94,25 @@ class XAITTSService(TTSService): Args: api_key: xAI API key for authentication. base_url: xAI TTS endpoint. Defaults to ``https://api.x.ai/v1/tts``. - voice: Voice identifier. Defaults to ``"eve"``. - - .. deprecated:: 0.0.105 - Use ``settings=XAITTSService.Settings(voice=...)`` instead. - - language: BCP-47 or base language code (for example ``"en"`` or ``"pt-BR"``). - Defaults to ``"en"``. - - .. deprecated:: 0.0.105 - Use ``settings=XAITTSService.Settings(language=...)`` instead. - - sample_rate: Output sample rate for PCM audio. Defaults to 24000 Hz. + sample_rate: Audio sample rate. If None, uses default. + encoding: Output encoding format. Defaults to "pcm". aiohttp_session: Optional shared aiohttp session. - params: Deprecated input parameters object. - settings: Runtime-updatable settings. When provided alongside deprecated - parameters, ``settings`` values take precedence. + settings: Runtime-updatable settings. **kwargs: Additional keyword arguments passed to ``TTSService``. """ default_settings = self.Settings( model=None, voice="eve", - language="en", + language=Language.EN, ) - if voice is not None: - self._warn_init_param_moved_to_settings("voice", "voice") - default_settings.voice = voice - if language is not None: - self._warn_init_param_moved_to_settings("language", "language") - default_settings.language = ( - self.language_to_service_language(language) - if isinstance(language, Language) - else language - ) - - if params is not None: - self._warn_init_param_moved_to_settings("params") - if not settings and params.language is not None: - default_settings.language = self.language_to_service_language(params.language) - if settings is not None: default_settings.apply_update(settings) super().__init__( - pause_frame_processing=True, + sample_rate=sample_rate, push_start_frame=True, push_stop_frames=True, - sample_rate=sample_rate or self.XAI_DEFAULT_SAMPLE_RATE, settings=default_settings, **kwargs, ) @@ -130,14 +121,22 @@ class XAITTSService(TTSService): self._base_url = base_url self._session = aiohttp_session self._session_owner = aiohttp_session is None + self._encoding = encoding def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics.""" return True def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert a Language enum to xAI's language format.""" - return str(language) + """Convert a Language enum to xAI language format. + + Args: + language: The language to convert. + + Returns: + The xAI-specific language code, or None if not supported. + """ + return language_to_xai_language(language) async def start(self, frame): """Start the xAI TTS service.""" @@ -175,7 +174,7 @@ class XAITTSService(TTSService): "text": text, "voice_id": self._settings.voice, "output_format": { - "codec": self.XAI_PCM_CODEC, + "codec": self._encoding, "sample_rate": self.sample_rate, }, } @@ -189,12 +188,11 @@ class XAITTSService(TTSService): measuring_ttfb = True try: - async with self._session.post(self._base_url, json=payload, headers=headers) as response: + async with self._session.post( + self._base_url, json=payload, headers=headers + ) as response: if response.status != 200: error = await response.text(errors="ignore") - logger.error( - f"{self} error getting audio (status: {response.status}, error: {error})" - ) yield ErrorFrame( error=f"Error getting audio (status: {response.status}, error: {error})" ) @@ -208,6 +206,11 @@ class XAITTSService(TTSService): if measuring_ttfb: await self.stop_ttfb_metrics() measuring_ttfb = False - yield TTSAudioRawFrame(chunk, self.sample_rate, 1, context_id=context_id) + yield TTSAudioRawFrame( + chunk, + self.sample_rate, + 1, + context_id=context_id, + ) except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/tests/test_xai_tts.py b/tests/test_xai_tts.py index b4c1513f6..aab984567 100644 --- a/tests/test_xai_tts.py +++ b/tests/test_xai_tts.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Tests for XAITTSService.""" +"""Tests for XAIHttpTTSService.""" import asyncio import unittest @@ -21,7 +21,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, TTSTextFrame, ) -from pipecat.services.xai.tts import XAITTSService +from pipecat.services.xai.tts import XAIHttpTTSService from pipecat.tests.utils import run_test @@ -52,7 +52,7 @@ async def test_run_xai_tts_success(aiohttp_client): base_url = str(client.make_url("/v1/tts")) async with aiohttp.ClientSession() as session: - tts_service = XAITTSService( + tts_service = XAIHttpTTSService( api_key="test-key", base_url=base_url, aiohttp_session=session, From adc003d6c76ffd7e15d8ff5a91718f2c6daed01f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 25 Mar 2026 10:53:07 -0400 Subject: [PATCH 3/3] Code review cleanup --- README.md | 28 ++-- .../07e-interruptible-anthropic.py | 125 ------------------ .../foundational/07e-interruptible-xai.py | 3 +- .../foundational/14g-function-calling-grok.py | 2 +- scripts/evals/run-release-evals.py | 2 +- src/pipecat/services/xai/__init__.py | 13 -- src/pipecat/services/xai/tts.py | 4 +- uv.lock | 2 +- 8 files changed, 20 insertions(+), 159 deletions(-) delete mode 100644 examples/foundational/07e-interruptible-anthropic.py diff --git a/README.md b/README.md index 8d2322200..7f087b9b8 100644 --- a/README.md +++ b/README.md @@ -85,20 +85,20 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout ## 🧩 Available services -| Category | Services | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | -| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/server/services/tts/smallest), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), xAI, [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | -| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | -| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | -| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | -| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | -| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | -| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | -| Community | [Browse community integrations →](https://docs.pipecat.ai/server/services/community-integrations) | +| Category | Services | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | +| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/server/services/tts/smallest), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [xAI](https://docs.pipecat.ai/server/services/tts/xai), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | +| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | +| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | +| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | +| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | +| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | +| Community | [Browse community integrations →](https://docs.pipecat.ai/server/services/community-integrations) | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) diff --git a/examples/foundational/07e-interruptible-anthropic.py b/examples/foundational/07e-interruptible-anthropic.py deleted file mode 100644 index fbb497d9b..000000000 --- a/examples/foundational/07e-interruptible-anthropic.py +++ /dev/null @@ -1,125 +0,0 @@ -# -# Copyright (c) 2024-2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - -# We use lambdas to defer transport parameter creation until the transport -# type is selected at runtime. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - settings=CartesiaTTSService.Settings( - voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ), - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - settings=AnthropicLLMService.Settings( - system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", - ), - ) - - context = LLMContext() - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), - ) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - user_aggregator, # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - assistant_aggregator, # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - context.add_message( - {"role": "developer", "content": "Please introduce yourself to the user."} - ) - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/07e-interruptible-xai.py b/examples/foundational/07e-interruptible-xai.py index 762c27e7c..b12c469ef 100644 --- a/examples/foundational/07e-interruptible-xai.py +++ b/examples/foundational/07e-interruptible-xai.py @@ -93,7 +93,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): params=PipelineParams( enable_metrics=True, enable_usage_metrics=True, - audio_out_sample_rate=8000, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, ) @@ -103,7 +102,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Client connected") # Kick off the conversation. context.add_message( - {"role": "user", "content": "Please introduce yourself to the user."} + {"role": "developer", "content": "Please introduce yourself to the user."} ) await task.queue_frames([LLMRunFrame()]) diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 34ec82b5f..148e8f127 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -131,7 +131,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Client connected") # Kick off the conversation. context.add_message( - {"role": "user", "content": "Please introduce yourself to the user."} + {"role": "developer", "content": "Please introduce yourself to the user."} ) await task.queue_frames([LLMRunFrame()]) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 4d113844e..829f03538 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -108,7 +108,7 @@ TESTS_07 = [ ("07c-interruptible-deepgram-http.py", EVAL_SIMPLE_MATH), ("07d-interruptible-elevenlabs.py", EVAL_SIMPLE_MATH), ("07d-interruptible-elevenlabs-http.py", EVAL_SIMPLE_MATH), - ("07e-interruptible-anthropic.py", EVAL_SIMPLE_MATH), + ("07e-interruptible-xai.py", EVAL_SIMPLE_MATH), ("07f-interruptible-azure.py", EVAL_SIMPLE_MATH), ("07f-interruptible-azure-http.py", EVAL_SIMPLE_MATH), ("07g-interruptible-openai.py", EVAL_SIMPLE_MATH), diff --git a/src/pipecat/services/xai/__init__.py b/src/pipecat/services/xai/__init__.py index 5433f9431..e69de29bb 100644 --- a/src/pipecat/services/xai/__init__.py +++ b/src/pipecat/services/xai/__init__.py @@ -1,13 +0,0 @@ -# -# Copyright (c) 2024-2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import sys - -from pipecat.services import DeprecatedModuleProxy - -from .tts import * - -sys.modules[__name__] = DeprecatedModuleProxy(globals(), "xai", "xai.tts") diff --git a/src/pipecat/services/xai/tts.py b/src/pipecat/services/xai/tts.py index a7026b2ae..c580ce912 100644 --- a/src/pipecat/services/xai/tts.py +++ b/src/pipecat/services/xai/tts.py @@ -62,7 +62,7 @@ def language_to_xai_language(language: Language) -> Optional[str]: @dataclass -class XAIHttpTTSSettings(TTSSettings): +class XAITTSSettings(TTSSettings): """Settings for XAIHttpTTSService.""" pass @@ -75,7 +75,7 @@ class XAIHttpTTSService(TTSService): match Pipecat's downstream expectations without extra decoding. """ - Settings = XAIHttpTTSSettings + Settings = XAITTSSettings _settings: Settings def __init__( diff --git a/uv.lock b/uv.lock index f49eaf616..2693d32e5 100644 --- a/uv.lock +++ b/uv.lock @@ -4966,7 +4966,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1,<1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "camb", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "kokoro", "krisp", "langchain", "lemonslice", "livekit", "lmnt", "local", "local-smart-turn", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "noisereduce", "novita", "nvidia", "openai", "rnnoise", "openpipe", "openrouter", "perplexity", "piper", "qwen", "remote-smart-turn", "resembleai", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "smallest", "sentry", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "camb", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "kokoro", "krisp", "langchain", "lemonslice", "livekit", "lmnt", "local", "local-smart-turn", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "noisereduce", "novita", "nvidia", "openai", "rnnoise", "openpipe", "openrouter", "perplexity", "piper", "qwen", "remote-smart-turn", "resembleai", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "smallest", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper", "xai"] [package.metadata.requires-dev] dev = [