From 099814d74a84066cc49678e07e0b36aaf8f69b68 Mon Sep 17 00:00:00 2001 From: Harshita Jain Date: Fri, 20 Mar 2026 09:30:16 -0700 Subject: [PATCH 1/3] Add Smallest AI TTS service integration Adds SmallestTTSService, a WebSocket-based TTS service using Smallest AI's Lightning v3.1 model. Follows current Pipecat service conventions: - SmallestTTSSettings dataclass with runtime-updatable settings (voice, language, speed, etc.) - Reconnects on model change; keepalive every 30s to prevent idle timeout - TTS settings default to None so the API applies its own defaults - Model enum: SmallestTTSModel.LIGHTNING_V3_1 Includes a foundational example (07zl-interruptible-smallest.py) using Deepgram STT + Smallest TTS + OpenAI LLM. STT integration will follow in a separate PR once the hallucination/finalize behaviour is resolved. Made-with: Cursor --- .../07zl-interruptible-smallest.py | 122 +++++ pyproject.toml | 1 + src/pipecat/services/smallest/__init__.py | 1 + src/pipecat/services/smallest/tts.py | 424 ++++++++++++++++++ uv.lock | 2 +- 5 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 examples/foundational/07zl-interruptible-smallest.py create mode 100644 src/pipecat/services/smallest/__init__.py create mode 100644 src/pipecat/services/smallest/tts.py diff --git a/examples/foundational/07zl-interruptible-smallest.py b/examples/foundational/07zl-interruptible-smallest.py new file mode 100644 index 000000000..8ffa48245 --- /dev/null +++ b/examples/foundational/07zl-interruptible-smallest.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.smallest.tts import SmallestTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + ) + + tts = SmallestTTSService( + api_key=os.getenv("SMALLEST_API_KEY"), + settings=SmallestTTSService.Settings( + voice="sophia", + ), + ) + + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + settings=OpenAILLMService.Settings( + system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + ), + ) + + context = LLMContext() + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + context.add_message({"role": "user", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/pyproject.toml b/pyproject.toml index 0416d6367..57fe00105 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0. sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] sarvam = [ "sarvamai==0.1.26", "pipecat-ai[websockets-base]" ] +smallest = [ "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] silero = [] simli = [ "simli-ai~=2.0.1"] diff --git a/src/pipecat/services/smallest/__init__.py b/src/pipecat/services/smallest/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/src/pipecat/services/smallest/__init__.py @@ -0,0 +1 @@ + diff --git a/src/pipecat/services/smallest/tts.py b/src/pipecat/services/smallest/tts.py new file mode 100644 index 000000000..ea2d03c66 --- /dev/null +++ b/src/pipecat/services/smallest/tts.py @@ -0,0 +1,424 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Smallest AI text-to-speech service implementation. + +This module provides a WebSocket-based integration with Smallest AI's +Waves API for real-time text-to-speech synthesis. +""" + +import asyncio +import base64 +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, AsyncGenerator, Optional + +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + StartFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven +from pipecat.services.tts_service import InterruptibleTTSService +from pipecat.transcriptions.language import Language +from pipecat.utils.tracing.service_decorators import traced_tts + +try: + from websockets.asyncio.client import connect as websocket_connect + from websockets.protocol import State +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Smallest, you need to `pip install pipecat-ai[smallest]`.") + raise Exception(f"Missing module: {e}") + + +class SmallestTTSModel(str, Enum): + """Available Smallest AI TTS models.""" + + LIGHTNING_V2 = "lightning-v2" + LIGHTNING_V3_1 = "lightning-v3.1" + + +def language_to_smallest_tts_language(language: Language) -> Optional[str]: + """Convert a Language enum to a Smallest TTS language string. + + Args: + language: The Language enum value to convert. + + Returns: + The Smallest language code string, or None if unsupported. + """ + BASE_LANGUAGES = { + Language.AR: "ar", + Language.BN: "bn", + Language.DE: "de", + Language.EN: "en", + Language.ES: "es", + Language.FR: "fr", + Language.GU: "gu", + Language.HE: "he", + Language.HI: "hi", + Language.IT: "it", + Language.KN: "kn", + Language.MR: "mr", + Language.NL: "nl", + Language.PL: "pl", + Language.RU: "ru", + Language.TA: "ta", + } + + result = BASE_LANGUAGES.get(language) + + if not result: + lang_str = str(language.value) + base_code = lang_str.split("-")[0].lower() + result = base_code if base_code in BASE_LANGUAGES.values() else None + + return result + + +@dataclass +class SmallestTTSSettings(TTSSettings): + """Settings for SmallestTTSService. + + Parameters: + speed: Speech speed multiplier. + consistency: Consistency level for voice generation (0-1). + similarity: Similarity level for voice generation (0-1). + enhancement: Enhancement level for voice generation (0-2). + """ + + speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + consistency: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + similarity: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + enhancement: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + + +class SmallestTTSService(InterruptibleTTSService): + """Smallest AI real-time text-to-speech service using WebSocket streaming. + + Provides real-time text-to-speech synthesis using Smallest AI's WebSocket API. + Supports streaming audio generation with configurable voice parameters and + language settings. Handles interruptions by reconnecting the WebSocket. + + Example:: + + tts = SmallestTTSService( + api_key="your-api-key", + settings=SmallestTTSService.Settings( + voice="sophia", + language="en", + speed=1.0, + ), + ) + """ + + Settings = SmallestTTSSettings + _settings: Settings + + def __init__( + self, + *, + api_key: str, + base_url: str = "wss://waves-api.smallest.ai", + sample_rate: Optional[int] = None, + settings: Optional[Settings] = None, + **kwargs, + ): + """Initialize the Smallest AI WebSocket TTS service. + + Args: + api_key: Smallest AI API key for authentication. + base_url: Base WebSocket URL for the Smallest API. + sample_rate: Audio sample rate in Hz. If None, uses default. + settings: Runtime-updatable settings for the TTS service. + **kwargs: Additional arguments passed to parent InterruptibleTTSService. + """ + default_settings = self.Settings( + model=SmallestTTSModel.LIGHTNING_V3_1.value, + voice="sophia", + language=language_to_smallest_tts_language(Language.EN), + speed=None, + consistency=None, + similarity=None, + enhancement=None, + ) + + if settings is not None: + default_settings.apply_update(settings) + + super().__init__( + aggregate_sentences=True, + push_text_frames=True, + pause_frame_processing=True, + sample_rate=sample_rate, + settings=default_settings, + **kwargs, + ) + + self._api_key = api_key + self._base_url = base_url.rstrip("/") + self._receive_task = None + self._keepalive_task = None + self._context_id: Optional[str] = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Smallest service supports metrics generation. + """ + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Smallest service language format. + + Args: + language: The language to convert. + + Returns: + The Smallest-specific language code, or None if not supported. + """ + return language_to_smallest_tts_language(language) + + def _build_msg(self, text: str) -> dict: + """Build a WebSocket message for the Smallest API. + + Args: + text: The text to synthesize. + + Returns: + Dictionary with the API message payload. + """ + msg = { + "text": text, + "voice_id": self._settings.voice, + "language": self._settings.language, + "sample_rate": self.sample_rate, + } + + if self._settings.speed is not None: + msg["speed"] = self._settings.speed + if self._settings.consistency is not None: + msg["consistency"] = self._settings.consistency + if self._settings.similarity is not None: + msg["similarity"] = self._settings.similarity + if self._settings.enhancement is not None: + msg["enhancement"] = self._settings.enhancement + + if self._context_id: + msg["request_id"] = self._context_id + + return msg + + def _build_websocket_url(self) -> str: + """Build the WebSocket URL from base URL and model.""" + return f"{self._base_url}/api/v1/{self._settings.model}/get_speech/stream" + + async def start(self, frame: StartFrame): + """Start the Smallest TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the Smallest TTS service. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the Smallest TTS service. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]: + """Apply a settings delta, reconnecting if model changed. + + Per-message fields (speed, consistency, similarity, enhancement, voice, + language) apply automatically on the next ``_build_msg`` call. A model + change requires reconnecting because the model is part of the WebSocket URL. + """ + changed = await super()._update_settings(delta) + + if not changed: + return changed + + if "model" in changed: + await self._disconnect() + await self._connect() + + return changed + + async def _connect(self): + """Connect to Smallest WebSocket and start receive task.""" + await super()._connect() + + await self._connect_websocket() + + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + if self._websocket and not self._keepalive_task: + self._keepalive_task = self.create_task(self._keepalive_task_handler()) + + async def _disconnect(self): + """Disconnect from Smallest WebSocket and clean up tasks.""" + await super()._disconnect() + + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + if self._keepalive_task: + await self.cancel_task(self._keepalive_task) + self._keepalive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Establish WebSocket connection to the Smallest API.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + + logger.debug("Connecting to Smallest TTS") + + self._websocket = await websocket_connect( + self._build_websocket_url(), + additional_headers={"Authorization": f"Bearer {self._api_key}"}, + ) + + await self._call_event_handler("on_connected") + except Exception as e: + await self.push_error(error_msg=f"Smallest TTS connection error: {e}", exception=e) + self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") + + async def _disconnect_websocket(self): + """Close the WebSocket connection and clean up state.""" + try: + await self.stop_all_metrics() + + if self._websocket: + logger.debug("Disconnecting from Smallest TTS") + await self._websocket.close() + except Exception as e: + logger.error(f"{self} error closing websocket: {e}") + finally: + self._context_id = None + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + """Get the WebSocket connection if available. + + Returns: + The active WebSocket connection. + + Raises: + Exception: If no WebSocket connection is available. + """ + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _keepalive_task_handler(self): + """Send periodic keepalive messages to prevent idle timeout.""" + KEEPALIVE_INTERVAL = 30 + while True: + await asyncio.sleep(KEEPALIVE_INTERVAL) + await self._send_keepalive() + + async def _send_keepalive(self): + """Send a flush message to keep the connection alive.""" + if self._websocket and self._websocket.state is State.OPEN: + msg = {"flush": True} + await self._websocket.send(json.dumps(msg)) + + async def _receive_messages(self): + """Receive and process messages from the Smallest WebSocket API.""" + async for message in self._get_websocket(): + msg = json.loads(message) + status = msg.get("status") + + if status == "complete": + msg_request_id = msg.get("request_id") + if self._context_id and msg_request_id and msg_request_id == self._context_id: + await self.stop_all_metrics() + await self.push_frame(TTSStoppedFrame(context_id=self._context_id)) + self._context_id = None + elif status == "chunk": + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame( + audio=base64.b64decode(msg["data"]["audio"]), + sample_rate=self.sample_rate, + num_channels=1, + context_id=self._context_id, + ) + await self.push_frame(frame) + elif status == "error": + logger.error(f"{self} error: {msg}") + await self.push_frame(TTSStoppedFrame(context_id=self._context_id)) + await self.stop_all_metrics() + await self.push_error(error_msg=f"Smallest TTS error: {msg.get('error', msg)}") + self._context_id = None + else: + logger.warning(f"{self} unknown message status: {msg}") + + @traced_tts + async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Smallest's WebSocket streaming API. + + Args: + text: The text to synthesize into speech. + context_id: Unique identifier for this TTS context. + + Yields: + Frame: TTSStartedFrame to signal start; audio arrives via WebSocket. + """ + logger.debug(f"{self}: Generating TTS [{text}]") + + try: + if not self._websocket or self._websocket.state is State.CLOSED: + await self._connect() + + try: + self._context_id = context_id + yield TTSStartedFrame(context_id=context_id) + + msg = self._build_msg(text=text) + await self._get_websocket().send(json.dumps(msg)) + await self.start_tts_usage_metrics(text) + except Exception as e: + logger.error(f"{self} error sending message: {e}") + yield ErrorFrame(error=f"Smallest TTS send error: {e}") + yield TTSStoppedFrame(context_id=context_id) + await self._disconnect() + await self._connect() + return + yield None + except Exception as e: + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"Smallest TTS error: {e}") diff --git a/uv.lock b/uv.lock index 0089729ef..a6afce62c 100644 --- a/uv.lock +++ b/uv.lock @@ -8480,4 +8480,4 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, -] +] \ No newline at end of file From 51d28b4a9f13f13b45de14d90aa49de3dc5b1528 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 24 Mar 2026 11:18:55 -0400 Subject: [PATCH 2/3] Code review fixes --- README.md | 28 ++++++++++++++-------------- changelog/4092.added.md | 1 + env.example | 3 +++ pyproject.toml | 2 +- scripts/evals/run-release-evals.py | 1 + src/pipecat/services/smallest/tts.py | 24 ++++++++---------------- uv.lock | 8 ++++++-- 7 files changed, 34 insertions(+), 33 deletions(-) create mode 100644 changelog/4092.added.md diff --git a/README.md b/README.md index 5c8cee735..842863548 100644 --- a/README.md +++ b/README.md @@ -85,20 +85,20 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout ## 🧩 Available services -| Category | Services | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | -| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | -| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | -| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | -| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | -| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | -| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | -| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | -| Community | [Browse community integrations →](https://docs.pipecat.ai/server/services/community-integrations) | +| Category | Services | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [Novita](https://docs.pipecat.ai/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | +| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/server/services/tts/smallest), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | +| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | +| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | +| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/server/services/video/lemonslice), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | +| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | +| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | +| Community | [Browse community integrations →](https://docs.pipecat.ai/server/services/community-integrations) | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) diff --git a/changelog/4092.added.md b/changelog/4092.added.md new file mode 100644 index 000000000..aa4dcf269 --- /dev/null +++ b/changelog/4092.added.md @@ -0,0 +1 @@ +- Added `SmallestTTSService`, a WebSocket-based TTS service integration with Smallest AI's Waves API. Supports the Lightning v2 and v3.1 models with configurable voice, language, speed, consistency, similarity, and enhancement settings. diff --git a/env.example b/env.example index 8927a348f..00b5fa775 100644 --- a/env.example +++ b/env.example @@ -179,6 +179,9 @@ SENTRY_DSN=... SIMLI_API_KEY=... SIMLI_FACE_ID=... +# Smallest +SMALLEST_API_KEY=... + # Smart turn LOCAL_SMART_TURN_MODEL_PATH=... FAL_SMART_TURN_API_KEY=... diff --git a/pyproject.toml b/pyproject.toml index 57fe00105..b7fa5d824 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,10 +111,10 @@ runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0. sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] sarvam = [ "sarvamai==0.1.26", "pipecat-ai[websockets-base]" ] -smallest = [ "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] silero = [] simli = [ "simli-ai~=2.0.1"] +smallest = [ "pipecat-ai[websockets-base]" ] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-voice[smart]~=0.2.8" ] diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 0fe66d9a1..915112631 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -147,6 +147,7 @@ TESTS_07 = [ ("07zi-interruptible-piper.py", EVAL_SIMPLE_MATH), ("07zj-interruptible-kokoro.py", EVAL_SIMPLE_MATH), ("07zk-interruptible-resembleai.py", EVAL_SIMPLE_MATH), + ("07zl-interruptible-smallest.py", EVAL_SIMPLE_MATH), ("07-interruptible-openai-responses.py", EVAL_SIMPLE_MATH), # Needs a local XTTS docker instance running. # ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH), diff --git a/src/pipecat/services/smallest/tts.py b/src/pipecat/services/smallest/tts.py index ea2d03c66..eca73a8e0 100644 --- a/src/pipecat/services/smallest/tts.py +++ b/src/pipecat/services/smallest/tts.py @@ -31,7 +31,7 @@ from pipecat.frames.frames import ( ) from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven from pipecat.services.tts_service import InterruptibleTTSService -from pipecat.transcriptions.language import Language +from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts try: @@ -59,7 +59,7 @@ def language_to_smallest_tts_language(language: Language) -> Optional[str]: Returns: The Smallest language code string, or None if unsupported. """ - BASE_LANGUAGES = { + LANGUAGE_MAP = { Language.AR: "ar", Language.BN: "bn", Language.DE: "de", @@ -78,14 +78,7 @@ def language_to_smallest_tts_language(language: Language) -> Optional[str]: Language.TA: "ta", } - result = BASE_LANGUAGES.get(language) - - if not result: - lang_str = str(language.value) - base_code = lang_str.split("-")[0].lower() - result = base_code if base_code in BASE_LANGUAGES.values() else None - - return result + return resolve_language(language, LANGUAGE_MAP, use_base_code=True) @dataclass @@ -118,7 +111,7 @@ class SmallestTTSService(InterruptibleTTSService): api_key="your-api-key", settings=SmallestTTSService.Settings( voice="sophia", - language="en", + language=Language.EN, speed=1.0, ), ) @@ -148,7 +141,7 @@ class SmallestTTSService(InterruptibleTTSService): default_settings = self.Settings( model=SmallestTTSModel.LIGHTNING_V3_1.value, voice="sophia", - language=language_to_smallest_tts_language(Language.EN), + language=Language.EN, speed=None, consistency=None, similarity=None, @@ -325,7 +318,9 @@ class SmallestTTSService(InterruptibleTTSService): logger.debug("Disconnecting from Smallest TTS") await self._websocket.close() except Exception as e: - logger.error(f"{self} error closing websocket: {e}") + await self.push_error( + error_msg=f"Smallest TTS error closing websocket: {e}", exception=e + ) finally: self._context_id = None self._websocket = None @@ -379,7 +374,6 @@ class SmallestTTSService(InterruptibleTTSService): ) await self.push_frame(frame) elif status == "error": - logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame(context_id=self._context_id)) await self.stop_all_metrics() await self.push_error(error_msg=f"Smallest TTS error: {msg.get('error', msg)}") @@ -412,7 +406,6 @@ class SmallestTTSService(InterruptibleTTSService): await self._get_websocket().send(json.dumps(msg)) await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} error sending message: {e}") yield ErrorFrame(error=f"Smallest TTS send error: {e}") yield TTSStoppedFrame(context_id=context_id) await self._disconnect() @@ -420,5 +413,4 @@ class SmallestTTSService(InterruptibleTTSService): return yield None except Exception as e: - logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"Smallest TTS error: {e}") diff --git a/uv.lock b/uv.lock index a6afce62c..acfd5494d 100644 --- a/uv.lock +++ b/uv.lock @@ -4789,6 +4789,9 @@ sentry = [ simli = [ { name = "simli-ai" }, ] +smallest = [ + { name = "websockets" }, +] soniox = [ { name = "websockets" }, ] @@ -4925,6 +4928,7 @@ requires-dist = [ { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'resembleai'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'smallest'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'ultravox'" }, { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, @@ -4962,7 +4966,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1,<1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "camb", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "kokoro", "krisp", "langchain", "lemonslice", "livekit", "lmnt", "local", "local-smart-turn", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "noisereduce", "nvidia", "novita", "openai", "rnnoise", "openpipe", "openrouter", "perplexity", "piper", "qwen", "remote-smart-turn", "resembleai", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "camb", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "kokoro", "krisp", "langchain", "lemonslice", "livekit", "lmnt", "local", "local-smart-turn", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "noisereduce", "novita", "nvidia", "openai", "rnnoise", "openpipe", "openrouter", "perplexity", "piper", "qwen", "remote-smart-turn", "resembleai", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "smallest", "sentry", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] [package.metadata.requires-dev] dev = [ @@ -8480,4 +8484,4 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, -] \ No newline at end of file +] From f68b3222b323dc7fd2d94518a02ba2d4634ec8da Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 24 Mar 2026 11:46:28 -0400 Subject: [PATCH 3/3] Fix SmallestTTSService to use InterruptibleTTSService audio context system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Route audio through audio contexts (append_to_audio_context) instead of pushing frames directly, enabling proper turn management and interruptions - Add push_stop_frames and push_start_frame so the base class handles TTSStartedFrame/TTSStoppedFrame lifecycle - Remove manual context_id tracking (self._context_id) in favor of get_active_audio_context_id() - Don't call remove_audio_context on "complete" — Smallest sends one per request, not per turn; let the base class timeout handle cleanup - Guard v2-only params (consistency, similarity, enhancement) so they aren't sent to lightning-v3.1 - Remove request_id from request payload (not a documented request field) - Add flush_audio override to send flush to WebSocket --- src/pipecat/services/smallest/tts.py | 49 +++++++++++++--------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/src/pipecat/services/smallest/tts.py b/src/pipecat/services/smallest/tts.py index eca73a8e0..15d107ffd 100644 --- a/src/pipecat/services/smallest/tts.py +++ b/src/pipecat/services/smallest/tts.py @@ -26,7 +26,6 @@ from pipecat.frames.frames import ( Frame, StartFrame, TTSAudioRawFrame, - TTSStartedFrame, TTSStoppedFrame, ) from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven @@ -152,8 +151,8 @@ class SmallestTTSService(InterruptibleTTSService): default_settings.apply_update(settings) super().__init__( - aggregate_sentences=True, - push_text_frames=True, + push_stop_frames=True, + push_start_frame=True, pause_frame_processing=True, sample_rate=sample_rate, settings=default_settings, @@ -164,7 +163,6 @@ class SmallestTTSService(InterruptibleTTSService): self._base_url = base_url.rstrip("/") self._receive_task = None self._keepalive_task = None - self._context_id: Optional[str] = None def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -203,15 +201,15 @@ class SmallestTTSService(InterruptibleTTSService): if self._settings.speed is not None: msg["speed"] = self._settings.speed - if self._settings.consistency is not None: - msg["consistency"] = self._settings.consistency - if self._settings.similarity is not None: - msg["similarity"] = self._settings.similarity - if self._settings.enhancement is not None: - msg["enhancement"] = self._settings.enhancement - if self._context_id: - msg["request_id"] = self._context_id + # consistency, similarity, enhancement are only supported by lightning-v2 + if self._settings.model == SmallestTTSModel.LIGHTNING_V2.value: + if self._settings.consistency is not None: + msg["consistency"] = self._settings.consistency + if self._settings.similarity is not None: + msg["similarity"] = self._settings.similarity + if self._settings.enhancement is not None: + msg["enhancement"] = self._settings.enhancement return msg @@ -322,7 +320,6 @@ class SmallestTTSService(InterruptibleTTSService): error_msg=f"Smallest TTS error closing websocket: {e}", exception=e ) finally: - self._context_id = None self._websocket = None await self._call_event_handler("on_disconnected") @@ -352,6 +349,12 @@ class SmallestTTSService(InterruptibleTTSService): msg = {"flush": True} await self._websocket.send(json.dumps(msg)) + async def flush_audio(self, context_id: Optional[str] = None): + """Flush any pending audio synthesis.""" + if not self._websocket or self._websocket.state is State.CLOSED: + return + await self._get_websocket().send(json.dumps({"flush": True})) + async def _receive_messages(self): """Receive and process messages from the Smallest WebSocket API.""" async for message in self._get_websocket(): @@ -359,25 +362,22 @@ class SmallestTTSService(InterruptibleTTSService): status = msg.get("status") if status == "complete": - msg_request_id = msg.get("request_id") - if self._context_id and msg_request_id and msg_request_id == self._context_id: - await self.stop_all_metrics() - await self.push_frame(TTSStoppedFrame(context_id=self._context_id)) - self._context_id = None + await self.stop_all_metrics() elif status == "chunk": await self.stop_ttfb_metrics() + context_id = self.get_active_audio_context_id() frame = TTSAudioRawFrame( audio=base64.b64decode(msg["data"]["audio"]), sample_rate=self.sample_rate, num_channels=1, - context_id=self._context_id, + context_id=context_id, ) - await self.push_frame(frame) + await self.append_to_audio_context(context_id, frame) elif status == "error": - await self.push_frame(TTSStoppedFrame(context_id=self._context_id)) + context_id = self.get_active_audio_context_id() + await self.push_frame(TTSStoppedFrame(context_id=context_id)) await self.stop_all_metrics() await self.push_error(error_msg=f"Smallest TTS error: {msg.get('error', msg)}") - self._context_id = None else: logger.warning(f"{self} unknown message status: {msg}") @@ -390,7 +390,7 @@ class SmallestTTSService(InterruptibleTTSService): context_id: Unique identifier for this TTS context. Yields: - Frame: TTSStartedFrame to signal start; audio arrives via WebSocket. + Frame: Audio arrives via WebSocket receive task. """ logger.debug(f"{self}: Generating TTS [{text}]") @@ -399,9 +399,6 @@ class SmallestTTSService(InterruptibleTTSService): await self._connect() try: - self._context_id = context_id - yield TTSStartedFrame(context_id=context_id) - msg = self._build_msg(text=text) await self._get_websocket().send(json.dumps(msg)) await self.start_tts_usage_metrics(text)