From 077610184dd55b71836c41dc31eb0dcea48ff89c Mon Sep 17 00:00:00 2001 From: Blaine Kasten Date: Tue, 17 Mar 2026 07:24:02 -0500 Subject: [PATCH] Add together STT and TTS services --- .../07e-interruptible-together.py | 125 ++++++ pyproject.toml | 2 +- src/pipecat/services/stt_latency.py | 1 + src/pipecat/services/together/__init__.py | 2 + src/pipecat/services/together/stt.py | 365 +++++++++++++++ src/pipecat/services/together/tts.py | 423 ++++++++++++++++++ uv.lock | 10 + 7 files changed, 927 insertions(+), 1 deletion(-) create mode 100644 examples/foundational/07e-interruptible-together.py create mode 100644 src/pipecat/services/together/stt.py create mode 100644 src/pipecat/services/together/tts.py diff --git a/examples/foundational/07e-interruptible-together.py b/examples/foundational/07e-interruptible-together.py new file mode 100644 index 000000000..1420e9263 --- /dev/null +++ b/examples/foundational/07e-interruptible-together.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.together.llm import TogetherLLMService +from pipecat.services.together.stt import TogetherSTTService +from pipecat.services.together.tts import TogetherTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +# We use lambdas to defer transport parameter creation until the transport +# type is selected at runtime. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = TogetherSTTService(api_key=os.getenv("TOGETHER_API_KEY")) + + tts = TogetherTTSService( + api_key=os.getenv("TOGETHER_API_KEY"), + voice="tara", + ) + + llm = TogetherLLMService(api_key=os.getenv("TOGETHER_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + user_aggregator, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + assistant_aggregator, # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/pyproject.toml b/pyproject.toml index 91afcc794..c4275eb09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,7 +118,7 @@ soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-voice[smart]~=0.2.8" ] strands = [ "strands-agents>=1.9.1,<2" ] tavus=[] -together = [] +together = [ "pipecat-ai[websockets-base]" ] tracing = [ "opentelemetry-sdk>=1.33.0,<2", "opentelemetry-api>=1.33.0,<2", "opentelemetry-instrumentation>=0.54b0,<1" ] ultravox = [ "pipecat-ai[websockets-base]" ] webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ] diff --git a/src/pipecat/services/stt_latency.py b/src/pipecat/services/stt_latency.py index 351e041a6..5dbccc348 100644 --- a/src/pipecat/services/stt_latency.py +++ b/src/pipecat/services/stt_latency.py @@ -46,6 +46,7 @@ SAMBANOVA_TTFS_P99: float = 2.20 SARVAM_TTFS_P99: float = 1.17 SONIOX_TTFS_P99: float = 0.35 SPEECHMATICS_TTFS_P99: float = 0.74 +TOGETHER_TTFS_P99: float = 1.00 # These services run locally and should be replaced with measured values NVIDIA_TTFS_P99: float = DEFAULT_TTFS_P99 diff --git a/src/pipecat/services/together/__init__.py b/src/pipecat/services/together/__init__.py index 8b78d89c1..2cc39a8d1 100644 --- a/src/pipecat/services/together/__init__.py +++ b/src/pipecat/services/together/__init__.py @@ -9,5 +9,7 @@ import sys from pipecat.services import DeprecatedModuleProxy from .llm import * +from .stt import * +from .tts import * sys.modules[__name__] = DeprecatedModuleProxy(globals(), "together", "together.llm") diff --git a/src/pipecat/services/together/stt.py b/src/pipecat/services/together/stt.py new file mode 100644 index 000000000..c1ee056f7 --- /dev/null +++ b/src/pipecat/services/together/stt.py @@ -0,0 +1,365 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Together AI speech-to-text service implementation.""" + +import base64 +import json +from dataclasses import dataclass +from typing import AsyncGenerator, Optional + +from loguru import logger + +from pipecat.services.settings import STTSettings +from pipecat.services.stt_latency import TOGETHER_TTFS_P99 + +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Together, you need to `pip install pipecat-ai[together]`.") + raise Exception(f"Missing module: {e}") + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterimTranscriptionFrame, + StartFrame, + TranscriptionFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.stt_service import WebsocketSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + + +@dataclass +class TogetherSTTSettings(STTSettings): + """Settings for the Together AI STT service. + + Parameters: + model: Together AI transcription model to use. + language: Language of the audio input. + """ + + model: str = "openai/whisper-large-v3" + language: Language = Language.EN + + +class TogetherSTTService(WebsocketSTTService): + """Together AI speech-to-text service. + + Provides real-time speech recognition using Together AI's WebSocket API + with OpenAI-compatible speech-to-text endpoints. + """ + + _settings: TogetherSTTSettings + + def __init__( + self, + *, + api_key: str, + model: str = "openai/whisper-large-v3", + language: Language = Language.EN, + sample_rate: int = 16000, + base_url: str = "wss://api.together.xyz/v1", + ttfs_p99_latency: float = TOGETHER_TTFS_P99, + **kwargs, + ): + """Initialize the Together AI STT service. + + Args: + api_key: Together AI API key for authentication. + model: Together AI transcription model. Defaults to "openai/whisper-large-v3". + language: Language of the audio input. Defaults to English. + sample_rate: Audio sample rate (default: 16000). Together AI requires 16kHz input. + base_url: The URL of the Together AI WebSocket API. + ttfs_p99_latency: P99 latency from speech end to final transcript in seconds. + Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark + **kwargs: Additional arguments passed to the parent WebsocketSTTService. + """ + super().__init__( + sample_rate=sample_rate, + ttfs_p99_latency=ttfs_p99_latency, + settings=TogetherSTTSettings( + model=model, + language=language, + ), + **kwargs, + ) + + self._api_key = api_key + self._base_url = base_url + self._receive_task = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Together STT service supports metrics generation. + """ + return True + + async def start(self, frame: StartFrame): + """Start the Together AI STT service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the Together AI STT service. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the Together AI STT service. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Send audio data to Together AI for transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: None (transcription results come via WebSocket callbacks). + """ + await self._send_audio(audio) + yield None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Together AI-specific handling. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, VADUserStartedSpeakingFrame): + await self.start_processing_metrics() + elif isinstance(frame, VADUserStoppedSpeakingFrame): + await self._commit_audio_buffer() + + # ------------------------------------------------------------------ + # WebSocket connection management + # ------------------------------------------------------------------ + + async def _connect(self): + """Connect to the transcription endpoint and start receiving.""" + await super()._connect() + await self._connect_websocket() + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Disconnect and clean up background tasks.""" + await super()._disconnect() + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=1.0) + self._receive_task = None + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Establish the WebSocket connection to the Together AI endpoint.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + url = ( + f"{self._base_url}/realtime?intent=transcription" + f"&model={self._settings.model}" + f"&input_audio_format=pcm_s16le_16000" + ) + headers = { + "Authorization": f"Bearer {self._api_key}", + "OpenAI-Beta": "realtime=v1", + } + + self._websocket = await websocket_connect(url, additional_headers=headers) + await self._call_event_handler("on_connected") + except Exception as e: + await self.push_error( + error_msg=f"Error connecting to Together AI STT: {e}", + exception=e, + ) + self._websocket = None + + async def _disconnect_websocket(self): + """Close the WebSocket connection.""" + try: + if self._websocket: + await self._websocket.close() + except Exception as e: + await self.push_error( + error_msg=f"Error disconnecting: {e}", + exception=e, + ) + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + # ------------------------------------------------------------------ + # Client events + # ------------------------------------------------------------------ + + async def _send_audio(self, audio: bytes): + """Send audio data via ``input_audio_buffer.append``. + + Args: + audio: Raw audio bytes at the pipeline sample rate. + """ + try: + if not self._disconnecting and self._websocket: + payload = base64.b64encode(audio).decode("utf-8") + await self._websocket.send( + json.dumps({"type": "input_audio_buffer.append", "audio": payload}) + ) + except Exception as e: + if self._disconnecting or not self._websocket: + return + await self.push_error( + error_msg=f"Error sending audio: {e}", + exception=e, + ) + + async def _commit_audio_buffer(self): + """Commit the current audio buffer for transcription.""" + try: + if not self._disconnecting and self._websocket: + await self._websocket.send(json.dumps({"type": "input_audio_buffer.commit"})) + except Exception as e: + if self._disconnecting or not self._websocket: + return + await self.push_error( + error_msg=f"Error committing audio buffer: {e}", + exception=e, + ) + + # ------------------------------------------------------------------ + # Server event handling + # ------------------------------------------------------------------ + + async def _receive_messages(self): + """Receive and dispatch server events from the transcription session. + + Called by ``WebsocketService._receive_task_handler`` which wraps + this method with automatic reconnection on connection errors. + """ + async for message in self._websocket: + try: + evt = json.loads(message) + except json.JSONDecodeError: + logger.warning(f"{self} failed to parse WebSocket message") + continue + + evt_type = evt.get("type", "") + + if evt_type == "session.created": + logger.debug(f"{self} session created") + elif evt_type == "conversation.item.input_audio_transcription.delta": + await self._handle_transcription_delta(evt) + elif evt_type == "conversation.item.input_audio_transcription.completed": + await self._handle_transcription_completed(evt) + elif evt_type == "conversation.item.input_audio_transcription.failed": + await self._handle_transcription_failed(evt) + elif evt_type == "input_audio_buffer.committed": + logger.trace(f"Audio buffer committed: item_id={evt.get('item_id', '')}") + elif evt_type == "error": + await self._handle_error(evt) + else: + logger.trace(f"{self} unhandled event: {evt_type}") + + async def _handle_transcription_delta(self, evt: dict): + """Handle incremental transcription text. + + Args: + evt: The delta event from the server. + """ + delta = evt.get("transcript", "") + if delta.strip(): + await self.push_frame( + InterimTranscriptionFrame( + delta, + self._user_id, + time_now_iso8601(), + result=evt, + ) + ) + + async def _handle_transcription_completed(self, evt: dict): + """Handle a completed transcription for a speech segment. + + Args: + evt: The completed event containing the full transcript. + """ + transcript = evt.get("transcript", "").strip() + if transcript: + await self.push_frame( + TranscriptionFrame( + transcript, + self._user_id, + time_now_iso8601(), + result=evt, + ) + ) + await self._handle_transcription_trace(transcript, True, self._settings.language) + await self.stop_processing_metrics() + + @traced_stt + async def _handle_transcription_trace( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Record transcription result for tracing. + + Args: + transcript: The transcribed text. + is_final: Whether this is a final transcription result. + language: Optional language of the transcription. + """ + pass + + async def _handle_transcription_failed(self, evt: dict): + """Handle a transcription failure for a speech segment. + + Args: + evt: The failed event containing error details. + """ + error_info = evt.get("error", {}) + await self.push_error(error_msg=f"Transcription failed: {error_info}") + + async def _handle_error(self, evt: dict): + """Handle a fatal error from the transcription session. + + Raises an exception so that ``WebsocketService`` can decide + whether to attempt reconnection. + + Args: + evt: The error event. + """ + error_info = evt.get("error", {}) + error_msg = error_info.get("message", "Unknown error") + error_code = error_info.get("code", "") + msg = f"Together AI STT error [{error_code}]: {error_msg}" + await self.push_error(error_msg=msg) + raise Exception(msg) diff --git a/src/pipecat/services/together/tts.py b/src/pipecat/services/together/tts.py new file mode 100644 index 000000000..e3daa3fa0 --- /dev/null +++ b/src/pipecat/services/together/tts.py @@ -0,0 +1,423 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Together AI text-to-speech service implementation.""" + +import base64 +import json +from dataclasses import dataclass +from typing import AsyncGenerator, Optional + +from loguru import logger + +from pipecat.services.settings import TTSSettings + +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Together, you need to `pip install pipecat-ai[together]`.") + raise Exception(f"Missing module: {e}") + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterruptionFrame, + StartFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.tts_service import WebsocketTTSService +from pipecat.transcriptions.language import Language +from pipecat.utils.tracing.service_decorators import traced_tts + + +@dataclass +class TogetherTTSSettings(TTSSettings): + """Settings for the Together AI TTS service. + + Parameters: + model: Together AI TTS model to use. + voice: Voice to use for synthesis. + language: Language of the text input. + max_partial_length: Maximum partial text length for streaming. + """ + + model: str = "canopylabs/orpheus-3b-0.1-ft" + language: Optional[Language] = Language.EN + voice: Optional[str] = "tara" + max_partial_length: Optional[int] = None + + +class TogetherTTSService(WebsocketTTSService): + """Together AI TTS service with WebSocket streaming. + + Provides text-to-speech using Together AI's realtime WebSocket API. + Supports streaming synthesis with configurable voice and model options. + """ + + _settings: TogetherTTSSettings + + def __init__( + self, + *, + api_key: str, + model: str = "canopylabs/orpheus-3b-0.1-ft", + voice: str = "tara", + language: Optional[Language] = Language.EN, + max_partial_length: Optional[int] = None, + url: str = "wss://api.together.ai/v1/audio/speech/websocket", + sample_rate: Optional[int] = 24000, + **kwargs, + ): + """Initialize the Together AI TTS service. + + Args: + api_key: Together AI API key for authentication. + model: Together AI TTS model. Defaults to "canopylabs/orpheus-3b-0.1-ft". + voice: Voice to use for synthesis. Defaults to "tara". + language: Language of the text input. Defaults to English. + max_partial_length: Maximum partial text length for streaming. + url: WebSocket URL for Together AI TTS API. + sample_rate: Audio sample rate (default: 24000). + **kwargs: Additional arguments passed to the parent service. + """ + super().__init__( + sample_rate=sample_rate, + settings=TogetherTTSSettings( + model=model, + voice=voice, + language=language, + max_partial_length=max_partial_length, + ), + **kwargs, + ) + + self._api_key = api_key + self._url = url + self._session_id = None + self._receive_task = None + self._context_id: Optional[str] = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Together TTS service supports metrics generation. + """ + return True + + def _build_websocket_url(self) -> str: + """Build the WebSocket URL with query parameters.""" + url = f"{self._url}?model={self._settings.model}&voice={self._settings.voice}" + if self._settings.max_partial_length is not None: + url += f"&max_partial_length={self._settings.max_partial_length}" + return url + + async def start(self, frame: StartFrame): + """Start the Together AI TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the Together AI TTS service. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the Together AI TTS service. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + # ------------------------------------------------------------------ + # WebSocket connection management + # ------------------------------------------------------------------ + + async def _connect(self): + """Connect to the transcription endpoint and start receiving.""" + await super()._connect() + await self._connect_websocket() + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Disconnect and clean up background tasks.""" + await super()._disconnect() + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=1.0) + self._receive_task = None + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Establish the WebSocket connection to the Together AI TTS endpoint.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + ws_url = self._build_websocket_url() + logger.debug(f"Connecting to Together AI TTS: {ws_url}") + + headers = {"Authorization": f"Bearer {self._api_key}"} + self._websocket = await websocket_connect(ws_url, additional_headers=headers) + await self._call_event_handler("on_connected") + + # Ensure voice is set on the server side + try: + voice_update_msg = { + "type": "tts_session.updated", + "session": {"voice": self._settings.voice}, + } + await self._websocket.send(json.dumps(voice_update_msg)) + logger.debug(f"Sent initial voice setting to WebSocket: {self._settings.voice}") + except Exception as e: + logger.error(f"Error sending initial voice setting: {e}") + + logger.debug("Connected to Together AI TTS") + + except Exception as e: + await self.push_error( + error_msg=f"Error connecting to Together AI TTS: {e}", + exception=e, + ) + self._websocket = None + + async def _disconnect_websocket(self): + """Close the WebSocket connection.""" + try: + if self._websocket: + await self._websocket.close() + except Exception as e: + await self.push_error( + error_msg=f"Error disconnecting: {e}", + exception=e, + ) + finally: + self._websocket = None + self._session_id = None + await self._call_event_handler("on_disconnected") + + # ------------------------------------------------------------------ + # Client events + # ------------------------------------------------------------------ + + async def _ws_send(self, message: dict): + """Send a JSON message over the WebSocket. + + Args: + message: The message dict to serialize and send. + """ + try: + if not self._disconnecting and self._websocket: + await self._websocket.send(json.dumps(message)) + except Exception as e: + if self._disconnecting or not self._websocket: + return + await self.push_error( + error_msg=f"Error sending message: {e}", + exception=e, + ) + + async def flush_audio(self): + """Flush any pending audio and finalize the current context.""" + logger.trace(f"{self}: flushing audio") + await self._ws_send({"type": "input_text_buffer.commit"}) + + # ------------------------------------------------------------------ + # Server event handling + # ------------------------------------------------------------------ + + async def _receive_messages(self): + """Receive and dispatch server events from the TTS session. + + Called by ``WebsocketService._receive_task_handler`` which wraps + this method with automatic reconnection on connection errors. + """ + async for message in self._websocket: + if not isinstance(message, str): + continue + + try: + evt = json.loads(message) + except json.JSONDecodeError: + logger.warning(f"{self} failed to parse WebSocket message") + continue + + evt_type = evt.get("type", "") + + if evt_type == "session.created": + await self._handle_session_created(evt) + elif evt_type == "session.updated": + await self._handle_session_updated(evt) + elif evt_type == "conversation.item.input_text.received": + text = evt.get("text", "") + logger.debug(f"{self} text received: {text[:50]}{'...' if len(text) > 50 else ''}") + elif evt_type == "conversation.item.audio_output.delta": + await self._handle_audio_delta(evt) + elif evt_type == "conversation.item.audio_output.done": + await self._handle_audio_done(evt) + elif evt_type == "conversation.item.tts.failed": + await self._handle_tts_failed(evt) + elif evt_type == "error": + await self._handle_error(evt) + else: + logger.trace(f"{self} unhandled event: {evt_type}") + + async def _handle_session_created(self, evt: dict): + """Handle ``session.created`` event. + + Args: + evt: The session created event from the server. + """ + session = evt.get("session", {}) + self._session_id = session.get("id") + logger.debug(f"{self} session created: {self._session_id}") + + async def _handle_session_updated(self, evt: dict): + """Handle ``session.updated`` event. + + Args: + evt: The session updated event from the server. + """ + session = evt.get("session", {}) + if "voice" in session: + updated_voice = session.get("voice") + logger.debug(f"{self} voice updated to: {updated_voice}") + + async def _handle_audio_delta(self, evt: dict): + """Handle an audio output delta containing a chunk of synthesized audio. + + Args: + evt: The delta event from the server. + """ + delta = evt.get("delta") + if delta: + try: + await self.stop_ttfb_metrics() + audio_chunk = base64.b64decode(delta) + frame = TTSAudioRawFrame( + audio=audio_chunk, + sample_rate=self.sample_rate, + num_channels=1, + context_id=self._context_id, + ) + await self.push_frame(frame) + except Exception as e: + logger.error(f"{self} error processing audio delta: {e}") + + async def _handle_audio_done(self, evt: dict): + """Handle audio output completion for a speech segment. + + Args: + evt: The done event from the server. + """ + item_id = evt.get("item_id") + logger.debug(f"{self} audio generation complete for: {item_id}") + await self.push_frame(TTSStoppedFrame(context_id=self._context_id)) + + async def _handle_tts_failed(self, evt: dict): + """Handle a TTS failure. + + Args: + evt: The failed event containing error details. + """ + error = evt.get("error", {}) + await self.push_error(error_msg=f"TTS error: {error}") + await self.push_frame(TTSStoppedFrame(context_id=self._context_id)) + + async def _handle_error(self, evt: dict): + """Handle a fatal error from the TTS session. + + Raises an exception so that ``WebsocketService`` can decide + whether to attempt reconnection. + + Args: + evt: The error event. + """ + error = evt.get("error", {}) + error_msg = error.get("message", "Unknown error") + error_code = error.get("code", "") + msg = f"Together AI TTS error [{error_code}]: {error_msg}" + await self.push_error(error_msg=msg) + raise Exception(msg) + + # ------------------------------------------------------------------ + # Interruption handling + # ------------------------------------------------------------------ + + async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): + """Handle interruption by canceling current generation. + + Args: + frame: The interruption frame. + direction: Frame processing direction. + """ + await super()._handle_interruption(frame, direction) + await self.stop_all_metrics() + await self._ws_send({"type": "input_text_buffer.clear"}) + + # ------------------------------------------------------------------ + # TTS generation + # ------------------------------------------------------------------ + + @traced_tts + async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Together AI's streaming API. + + Args: + text: The text to synthesize into speech. + context_id: The context ID for tracking audio frames. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + logger.debug(f"{self}: Generating TTS [{text}]") + + try: + if not self._websocket or self._websocket.state is not State.OPEN: + await self._connect() + if not self._websocket or self._websocket.state is not State.OPEN: + logger.error(f"{self} failed to connect to WebSocket") + yield TTSStoppedFrame(context_id=context_id) + return + + self._context_id = context_id + + await self.start_ttfb_metrics() + yield TTSStartedFrame(context_id=context_id) + + try: + await self._ws_send({"type": "input_text_buffer.append", "text": text}) + await self._ws_send({"type": "input_text_buffer.commit"}) + await self.start_tts_usage_metrics(text) + except Exception as e: + logger.error(f"{self} error sending message: {e}") + yield TTSStoppedFrame(context_id=context_id) + await self._disconnect() + await self._connect() + return + + yield None + + except Exception as e: + logger.error(f"{self} exception: {e}") + await self.push_error(error_msg=f"Error generating TTS: {e}", exception=e) + yield TTSStoppedFrame(context_id=context_id) diff --git a/uv.lock b/uv.lock index 144b30e66..d91b223f7 100644 --- a/uv.lock +++ b/uv.lock @@ -2126,6 +2126,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/3f/9859f655d11901e7b2996c6e3d33e0caa9a1d4572c3bc61ed0faa64b2f4c/greenlet-3.3.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9bc885b89709d901859cf95179ec9f6bb67a3d2bb1f0e88456461bd4b7f8fd0d", size = 277747, upload-time = "2026-02-20T20:16:21.325Z" }, { url = "https://files.pythonhosted.org/packages/fb/07/cb284a8b5c6498dbd7cba35d31380bb123d7dceaa7907f606c8ff5993cbf/greenlet-3.3.2-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b568183cf65b94919be4438dc28416b234b678c608cafac8874dfeeb2a9bbe13", size = 579202, upload-time = "2026-02-20T20:47:28.955Z" }, { url = "https://files.pythonhosted.org/packages/ed/45/67922992b3a152f726163b19f890a85129a992f39607a2a53155de3448b8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:527fec58dc9f90efd594b9b700662ed3fb2493c2122067ac9c740d98080a620e", size = 590620, upload-time = "2026-02-20T20:55:55.581Z" }, + { url = "https://files.pythonhosted.org/packages/03/5f/6e2a7d80c353587751ef3d44bb947f0565ec008a2e0927821c007e96d3a7/greenlet-3.3.2-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508c7f01f1791fbc8e011bd508f6794cb95397fdb198a46cb6635eb5b78d85a7", size = 602132, upload-time = "2026-02-20T21:02:43.261Z" }, { url = "https://files.pythonhosted.org/packages/ad/55/9f1ebb5a825215fadcc0f7d5073f6e79e3007e3282b14b22d6aba7ca6cb8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad0c8917dd42a819fe77e6bdfcb84e3379c0de956469301d9fd36427a1ca501f", size = 591729, upload-time = "2026-02-20T20:20:58.395Z" }, { url = "https://files.pythonhosted.org/packages/24/b4/21f5455773d37f94b866eb3cf5caed88d6cea6dd2c6e1f9c34f463cba3ec/greenlet-3.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97245cc10e5515dbc8c3104b2928f7f02b6813002770cfaffaf9a6e0fc2b94ef", size = 1551946, upload-time = "2026-02-20T20:49:31.102Z" }, { url = "https://files.pythonhosted.org/packages/00/68/91f061a926abead128fe1a87f0b453ccf07368666bd59ffa46016627a930/greenlet-3.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8c1fdd7d1b309ff0da81d60a9688a8bd044ac4e18b250320a96fc68d31c209ca", size = 1618494, upload-time = "2026-02-20T20:21:06.541Z" }, @@ -2133,6 +2134,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/47/16400cb42d18d7a6bb46f0626852c1718612e35dcb0dffa16bbaffdf5dd2/greenlet-3.3.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:c56692189a7d1c7606cb794be0a8381470d95c57ce5be03fb3d0ef57c7853b86", size = 278890, upload-time = "2026-02-20T20:19:39.263Z" }, { url = "https://files.pythonhosted.org/packages/a3/90/42762b77a5b6aa96cd8c0e80612663d39211e8ae8a6cd47c7f1249a66262/greenlet-3.3.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ebd458fa8285960f382841da585e02201b53a5ec2bac6b156fc623b5ce4499f", size = 581120, upload-time = "2026-02-20T20:47:30.161Z" }, { url = "https://files.pythonhosted.org/packages/bf/6f/f3d64f4fa0a9c7b5c5b3c810ff1df614540d5aa7d519261b53fba55d4df9/greenlet-3.3.2-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a443358b33c4ec7b05b79a7c8b466f5d275025e750298be7340f8fc63dff2a55", size = 594363, upload-time = "2026-02-20T20:55:56.965Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8b/1430a04657735a3f23116c2e0d5eb10220928846e4537a938a41b350bed6/greenlet-3.3.2-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4375a58e49522698d3e70cc0b801c19433021b5c37686f7ce9c65b0d5c8677d2", size = 605046, upload-time = "2026-02-20T21:02:45.234Z" }, { url = "https://files.pythonhosted.org/packages/72/83/3e06a52aca8128bdd4dcd67e932b809e76a96ab8c232a8b025b2850264c5/greenlet-3.3.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e2cd90d413acbf5e77ae41e5d3c9b3ac1d011a756d7284d7f3f2b806bbd6358", size = 594156, upload-time = "2026-02-20T20:20:59.955Z" }, { url = "https://files.pythonhosted.org/packages/70/79/0de5e62b873e08fe3cef7dbe84e5c4bc0e8ed0c7ff131bccb8405cd107c8/greenlet-3.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:442b6057453c8cb29b4fb36a2ac689382fc71112273726e2423f7f17dc73bf99", size = 1554649, upload-time = "2026-02-20T20:49:32.293Z" }, { url = "https://files.pythonhosted.org/packages/5a/00/32d30dee8389dc36d42170a9c66217757289e2afb0de59a3565260f38373/greenlet-3.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45abe8eb6339518180d5a7fa47fa01945414d7cca5ecb745346fc6a87d2750be", size = 1619472, upload-time = "2026-02-20T20:21:07.966Z" }, @@ -2141,6 +2143,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd", size = 280358, upload-time = "2026-02-20T20:17:43.971Z" }, { url = "https://files.pythonhosted.org/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd", size = 601217, upload-time = "2026-02-20T20:47:31.462Z" }, { url = "https://files.pythonhosted.org/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac", size = 611792, upload-time = "2026-02-20T20:55:58.423Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb", size = 626250, upload-time = "2026-02-20T21:02:46.596Z" }, { url = "https://files.pythonhosted.org/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070", size = 613875, upload-time = "2026-02-20T20:21:01.102Z" }, { url = "https://files.pythonhosted.org/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79", size = 1571467, upload-time = "2026-02-20T20:49:33.495Z" }, { url = "https://files.pythonhosted.org/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395", size = 1640001, upload-time = "2026-02-20T20:21:09.154Z" }, @@ -2149,6 +2152,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/48/f8b875fa7dea7dd9b33245e37f065af59df6a25af2f9561efa8d822fde51/greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4", size = 279120, upload-time = "2026-02-20T20:19:01.9Z" }, { url = "https://files.pythonhosted.org/packages/49/8d/9771d03e7a8b1ee456511961e1b97a6d77ae1dea4a34a5b98eee706689d3/greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986", size = 603238, upload-time = "2026-02-20T20:47:32.873Z" }, { url = "https://files.pythonhosted.org/packages/59/0e/4223c2bbb63cd5c97f28ffb2a8aee71bdfb30b323c35d409450f51b91e3e/greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92", size = 614219, upload-time = "2026-02-20T20:55:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/94/2b/4d012a69759ac9d77210b8bfb128bc621125f5b20fc398bce3940d036b1c/greenlet-3.3.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccd21bb86944ca9be6d967cf7691e658e43417782bce90b5d2faeda0ff78a7dd", size = 628268, upload-time = "2026-02-20T21:02:48.024Z" }, { url = "https://files.pythonhosted.org/packages/7a/34/259b28ea7a2a0c904b11cd36c79b8cef8019b26ee5dbe24e73b469dea347/greenlet-3.3.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6997d360a4e6a4e936c0f9625b1c20416b8a0ea18a8e19cabbefc712e7397ab", size = 616774, upload-time = "2026-02-20T20:21:02.454Z" }, { url = "https://files.pythonhosted.org/packages/0a/03/996c2d1689d486a6e199cb0f1cf9e4aa940c500e01bdf201299d7d61fa69/greenlet-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64970c33a50551c7c50491671265d8954046cb6e8e2999aacdd60e439b70418a", size = 1571277, upload-time = "2026-02-20T20:49:34.795Z" }, { url = "https://files.pythonhosted.org/packages/d9/c4/2570fc07f34a39f2caf0bf9f24b0a1a0a47bc2e8e465b2c2424821389dfc/greenlet-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a9172f5bf6bd88e6ba5a84e0a68afeac9dc7b6b412b245dd64f52d83c81e55b", size = 1640455, upload-time = "2026-02-20T20:21:10.261Z" }, @@ -2157,6 +2161,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/ae/8bffcbd373b57a5992cd077cbe8858fff39110480a9d50697091faea6f39/greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab", size = 279650, upload-time = "2026-02-20T20:18:00.783Z" }, { url = "https://files.pythonhosted.org/packages/d1/c0/45f93f348fa49abf32ac8439938726c480bd96b2a3c6f4d949ec0124b69f/greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082", size = 650295, upload-time = "2026-02-20T20:47:34.036Z" }, { url = "https://files.pythonhosted.org/packages/b3/de/dd7589b3f2b8372069ab3e4763ea5329940fc7ad9dcd3e272a37516d7c9b/greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9", size = 662163, upload-time = "2026-02-20T20:56:01.295Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ac/85804f74f1ccea31ba518dcc8ee6f14c79f73fe36fa1beba38930806df09/greenlet-3.3.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9", size = 675371, upload-time = "2026-02-20T21:02:49.664Z" }, { url = "https://files.pythonhosted.org/packages/d2/d8/09bfa816572a4d83bccd6750df1926f79158b1c36c5f73786e26dbe4ee38/greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506", size = 664160, upload-time = "2026-02-20T20:21:04.015Z" }, { url = "https://files.pythonhosted.org/packages/48/cf/56832f0c8255d27f6c35d41b5ec91168d74ec721d85f01a12131eec6b93c/greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce", size = 1619181, upload-time = "2026-02-20T20:49:36.052Z" }, { url = "https://files.pythonhosted.org/packages/0a/23/b90b60a4aabb4cec0796e55f25ffbfb579a907c3898cd2905c8918acaa16/greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5", size = 1687713, upload-time = "2026-02-20T20:21:11.684Z" }, @@ -2165,6 +2170,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/6d/8f2ef704e614bcf58ed43cfb8d87afa1c285e98194ab2cfad351bf04f81e/greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54", size = 286617, upload-time = "2026-02-20T20:19:29.856Z" }, { url = "https://files.pythonhosted.org/packages/5e/0d/93894161d307c6ea237a43988f27eba0947b360b99ac5239ad3fe09f0b47/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4", size = 655189, upload-time = "2026-02-20T20:47:35.742Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/d2d506ebd8abcb57386ec4f7ba20f4030cbe56eae541bc6fd6ef399c0b41/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff", size = 658225, upload-time = "2026-02-20T20:56:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/d1/67/8197b7e7e602150938049d8e7f30de1660cfb87e4c8ee349b42b67bdb2e1/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:59b3e2c40f6706b05a9cd299c836c6aa2378cabe25d021acd80f13abf81181cf", size = 666581, upload-time = "2026-02-20T21:02:51.526Z" }, { url = "https://files.pythonhosted.org/packages/8e/30/3a09155fbf728673a1dea713572d2d31159f824a37c22da82127056c44e4/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4", size = 657907, upload-time = "2026-02-20T20:21:05.259Z" }, { url = "https://files.pythonhosted.org/packages/f3/fd/d05a4b7acd0154ed758797f0a43b4c0962a843bedfe980115e842c5b2d08/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727", size = 1618857, upload-time = "2026-02-20T20:49:37.309Z" }, { url = "https://files.pythonhosted.org/packages/6f/e1/50ee92a5db521de8f35075b5eff060dd43d39ebd46c2181a2042f7070385/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e", size = 1680010, upload-time = "2026-02-20T20:21:13.427Z" }, @@ -4737,6 +4743,9 @@ speechmatics = [ strands = [ { name = "strands-agents" }, ] +together = [ + { name = "websockets" }, +] tracing = [ { name = "opentelemetry-api" }, { name = "opentelemetry-instrumentation" }, @@ -4862,6 +4871,7 @@ requires-dist = [ { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'together'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'ultravox'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, { name = "pipecat-ai-krisp", marker = "extra == 'krisp'", specifier = "~=0.4.0" },